squirrels 0.5.0b3__py3-none-any.whl → 0.6.0.post0__py3-none-any.whl

squirrels/_models.py

@@ -5,25 +5,27 @@ from abc import ABCMeta, abstractmethod
 from enum import Enum
 from pathlib import Path
 import asyncio, os, re, time, duckdb, sqlglot
-import polars as pl, pandas as pd, networkx as nx
+import polars as pl, pandas as pd
 
-from . import _constants as c, _utils as u, _py_module as pm, _model_queries as mq, _model_configs as mc, _sources as src, _api_response_models as arm
+from . import _constants as c, _utils as u, _py_module as pm, _model_queries as mq, _model_configs as mc, _sources as src
+from ._schemas import response_models as rm
 from ._exceptions import FileExecutionError, InvalidInputError
-from ._arguments._run_time_args import ContextArgs, ModelArgs, BuildModelArgs
-from ._auth import BaseUser
+from ._arguments.run_time_args import ContextArgs, ModelArgs, BuildModelArgs
+from ._auth import AbstractUser
 from ._connection_set import ConnectionsArgs, ConnectionSet, ConnectionProperties
-from ._manifest import DatasetConfig
+from ._manifest import DatasetConfig, ConnectionTypeEnum
 from ._parameter_sets import ParameterConfigsSet, ParametersArgs, ParameterSet
+from ._env_vars import SquirrelsEnvVars
 
 ContextFunc = Callable[[dict[str, Any], ContextArgs], None]
 
 
 class ModelType(Enum):
+    SEED = "seed"
     SOURCE = "source"
+    BUILD = "build"
     DBVIEW = "dbview"
     FEDERATE = "federate"
-    SEED = "seed"
-    BUILD = "build"
 
 
 @dataclass
@@ -42,7 +44,6 @@ class DataModel(metaclass=ABCMeta):
 
     _: KW_ONLY
     logger: u.Logger = field(default_factory=lambda: u.Logger(""))
-    env_vars: dict[str, str] = field(default_factory=dict)
     conn_set: ConnectionSet = field(default_factory=ConnectionSet)
 
     @property
@@ -78,15 +79,15 @@ class DataModel(metaclass=ABCMeta):
         self.confirmed_no_cycles = True
         return terminal_nodes
     
-    def _load_duckdb_view_to_python_df(self, conn: duckdb.DuckDBPyConnection, *, use_venv: bool = False) -> pl.LazyFrame:
-        table_name = ("venv." if use_venv else "") + self.name
+    def _load_duckdb_view_to_python_df(self, conn: duckdb.DuckDBPyConnection, *, use_datalake: bool = False) -> pl.LazyFrame:
+        table_name = ("vdl." if use_datalake else "") + self.name
         try:
             return conn.sql(f"FROM {table_name}").pl().lazy()
         except duckdb.CatalogException as e:
             raise u.ConfigurationError(f'Failed to load duckdb table or view "{self.name}" to python dataframe') from e
     
     def _run_sql_query_on_connection(self, connection_name: str, query: str, placeholders: dict = {}) -> pl.DataFrame:
-        self.logger.info(f"Running sql query on connection '{connection_name}': {query}")
+        self.logger.debug(f"Running SQL query on connection '{connection_name}':\n{query}")
         return self.conn_set.run_sql_query_from_conn_name(query, connection_name, placeholders)
     
     async def _trigger(self, conn: duckdb.DuckDBPyConnection, placeholders: dict = {}) -> None:
@@ -132,11 +133,13 @@ class DataModel(metaclass=ABCMeta):
     
     def _create_table_from_df(self, conn: duckdb.DuckDBPyConnection, query_result: pl.LazyFrame | pd.DataFrame):
         local_conn = conn.cursor()
+        # local_conn = conn
         try:
-            local_conn.register("df", query_result)
-            local_conn.execute(f"CREATE OR REPLACE TABLE {self.name} AS SELECT * FROM df")
+            assert query_result is not None
+            local_conn.execute(f"CREATE OR REPLACE TABLE {self.name} AS FROM query_result")
         finally:
             local_conn.close()
+            # pass
         
     def process_pass_through_columns(self, models_dict: dict[str, DataModel]) -> None:
         pass
@@ -171,19 +174,26 @@ class StaticModel(DataModel):
     def _get_result(self, conn: duckdb.DuckDBPyConnection) -> pl.LazyFrame:
         local_conn = conn.cursor()
         try:
-            return self._load_duckdb_view_to_python_df(local_conn, use_venv=True)
+            return self._load_duckdb_view_to_python_df(local_conn, use_datalake=True)
         except Exception as e:
-            raise InvalidInputError(61, f'Model "{self.name}" depends on static data models that cannot be found.')
+            raise InvalidInputError(409, f'dependent_data_model_not_found', f'Model "{self.name}" depends on static data models that cannot be found. Try building the Virtual Data Lake (VDL) first.')
         finally:
             local_conn.close()
     
     async def run_model(self, conn: duckdb.DuckDBPyConnection, placeholders: dict = {}) -> None:
-        start = time.time()
-
         if (self.needs_python_df or self.is_target) and self.result is None:
+            start = time.time()
+
             self.result = await asyncio.to_thread(self._get_result, conn)
-        
-        self.logger.log_activity_time(f"loading static model '{self.name}'", start)
+            
+            self.logger.log_activity_time(
+                f"loading {self.model_type.value} model '{self.name}' into memory", start, 
+                additional_data={
+                    "activity": "loading static data model into memory",
+                    "model_name": self.name, 
+                    "model_type": self.model_type.value
+                }
+            )
 
         await super().run_model(conn, placeholders)
 
@@ -224,10 +234,18 @@ class Seed(StaticModel):
         start = time.time()
 
         print(f"[{u.get_current_time()}] 🔨 BUILDING: seed model '{self.name}'")
-        await asyncio.to_thread(self._create_table_from_df, conn, self.result)
+        # await asyncio.to_thread(self._create_table_from_df, conn, self.result)
+        self._create_table_from_df(conn, self.result) # without threading
 
         print(f"[{u.get_current_time()}] ✅ FINISHED: seed model '{self.name}'")
-        self.logger.log_activity_time(f"building seed model '{self.name}' to venv", start)
+        self.logger.log_activity_time(
+            f"building seed model '{self.name}' into VDL", start, 
+            additional_data={
+                "activity": "building data model into VDL", 
+                "model_name": self.name, 
+                "model_type": self.model_type.value
+            }
+        )
 
         await super().build_model(conn, full_refresh)
 
@@ -239,24 +257,32 @@ class SourceModel(StaticModel):
     @property
     def model_type(self) -> ModelType:
         return ModelType.SOURCE
+
+    @property
+    def connection_props(self) -> ConnectionProperties:
+        conn_name = self.model_config.get_connection()
+        conn_props = self.conn_set.get_connection(conn_name)
+        if isinstance(conn_props, ConnectionProperties):
+            return conn_props
+        raise u.ConfigurationError(f'Unable to use connection "{conn_name}" for source "{self.name}". Connection "{conn_name}" must be a ConnectionProperties object')
     
     @property
     def is_queryable(self) -> bool:
-        return self.model_config.load_to_duckdb
+        connection_props = self.connection_props
+        return self.model_config.load_to_vdl or connection_props.type == ConnectionTypeEnum.DUCKDB
     
     def _build_source_model(self, conn: duckdb.DuckDBPyConnection, full_refresh: bool) -> None:
         local_conn = conn.cursor()
+        # local_conn = conn
+        
+        local_conn.begin()
         try:
             source = self.model_config
             conn_name = source.get_connection()
             
-            connection_props = self.conn_set.get_connection(conn_name)
-            if isinstance(connection_props, ConnectionProperties):
-                dialect = connection_props.dialect
-                attach_uri = connection_props.attach_uri_for_duckdb
-            else:
-                raise u.ConfigurationError(f'Unable to use connection "{conn_name}" for source "{self.name}". Connection "{conn_name}" must be a ConnectionProperties object')
-            
+            connection_props = self.connection_props
+            dialect = connection_props.dialect
+            attach_uri = connection_props.attach_uri_for_duckdb
             if attach_uri is None:
                 raise u.ConfigurationError(f'Loading to duckdb is not supported for source "{self.name}" since its connection "{conn_name}" uses an unsupported dialect')
             
@@ -268,8 +294,9 @@ class SourceModel(StaticModel):
             new_table_name = self.name
 
             if len(source.columns) == 0:
-                stmt = f"CREATE OR REPLACE TABLE {new_table_name} AS SELECT * FROM db_{conn_name}.{table_name}"
+                stmt = f"CREATE OR REPLACE TABLE {new_table_name} AS FROM db_{conn_name}.{table_name}"
                 u.run_duckdb_stmt(self.logger, local_conn, stmt)
+                local_conn.commit()
                 return
             
             increasing_column = source.update_hints.increasing_column
@@ -296,25 +323,44 @@ class SourceModel(StaticModel):
                 if max_val_of_incr_col is None:
                     recreate_table = True
 
-            insert_cols_clause = source.get_cols_for_insert_stmt()
-            insert_replace_clause = source.get_insert_replace_clause()
-            query = source.get_query_for_insert(dialect, conn_name, table_name, max_val_of_incr_col, full_refresh=recreate_table)
-            stmt = f"INSERT {insert_replace_clause} INTO {new_table_name} ({insert_cols_clause}) {query}"
+            query = source.get_query_for_upsert(dialect, conn_name, table_name, max_val_of_incr_col, full_refresh=recreate_table)
+
+            primary_keys = ", ".join(source.primary_key) if source.primary_key else ""
+            match_condition = f"USING ({primary_keys})" if primary_keys else "ON false"
+            stmt = (
+                f"MERGE INTO {new_table_name} "
+                f"USING ({query}) AS src "
+                f"{match_condition} "
+                f"WHEN MATCHED THEN UPDATE "
+                f"WHEN NOT MATCHED THEN INSERT BY NAME"
+            )
             u.run_duckdb_stmt(self.logger, local_conn, stmt)
+
+            local_conn.commit()
+        
         finally:
             local_conn.close()
+            # pass
 
     async def build_model(self, conn: duckdb.DuckDBPyConnection, full_refresh: bool) -> None:
-        if self.model_config.load_to_duckdb:
+        if self.model_config.load_to_vdl:
             start = time.time()
             print(f"[{u.get_current_time()}] 🔨 BUILDING: source model '{self.name}'")
 
-            await asyncio.to_thread(self._build_source_model, conn, full_refresh)
+            # await asyncio.to_thread(self._build_source_model, conn, full_refresh)
+            self._build_source_model(conn, full_refresh) # without threading
             
             print(f"[{u.get_current_time()}] ✅ FINISHED: source model '{self.name}'")
-            self.logger.log_activity_time(f"building source model '{self.name}' to venv", start)
+            self.logger.log_activity_time(
+                f"building source model '{self.name}' into VDL", start, 
+                additional_data={
+                    "activity": "building data model into VDL", 
+                    "model_name": self.name, 
+                    "model_type": self.model_type.value
+                }
+            )
 
-            await super().build_model(conn, full_refresh)
+        await super().build_model(conn, full_refresh)
         
 
 @dataclass
@@ -337,10 +383,16 @@ class QueryModel(DataModel):
             raise u.ConfigurationError(f'Model "{self.name}" references unknown model "{dependent_model_name}"')
         
         dep_model = models_dict[dependent_model_name]
-        if isinstance(dep_model, SourceModel) and not dep_model.model_config.load_to_duckdb:
-            raise u.ConfigurationError(
-                f'Model "{self.name}" cannot reference source model "{dependent_model_name}" which has load_to_duckdb=False'
-            )
+        if isinstance(dep_model, SourceModel) and not dep_model.model_config.load_to_vdl:
+            # Allow when caller is Build or Federate AND the source connection is duckdb; else error
+            conn_name = dep_model.model_config.get_connection()
+            conn_props = self.conn_set.get_connection(conn_name)
+            is_duckdb_conn = isinstance(conn_props, ConnectionProperties) and conn_props.type == ConnectionTypeEnum.DUCKDB
+            if not is_duckdb_conn:
+                raise u.ConfigurationError(
+                    f'Model "{self.name}" cannot reference source model "{dependent_model_name}". '
+                    'To be referenced by a build or federate model, the source must have load_to_vdl=True or a duckdb connection type.'
+                )
         
         self.model_config.depends_on.add(dependent_model_name)
         return dependent_model_name
@@ -355,10 +407,10 @@ class QueryModel(DataModel):
     def _get_compile_sql_model_args_from_ctx_args(
         self, ctx: dict[str, Any], ctx_args: ContextArgs
     ) -> dict[str, Any]:
-        is_placeholder = lambda placeholder: placeholder in ctx_args._placeholders_copy
+        is_placeholder = lambda placeholder: placeholder in ctx_args._placeholders
         kwargs = {
             "proj_vars": ctx_args.proj_vars, "env_vars": ctx_args.env_vars, "user": ctx_args.user, "prms": ctx_args.prms, 
-            "traits": ctx_args.traits, "ctx": ctx, "is_placeholder": is_placeholder, "set_placeholder": ctx_args.set_placeholder,
+            "configurables": ctx_args.configurables, "ctx": ctx, "is_placeholder": is_placeholder, "set_placeholder": ctx_args.set_placeholder,
             "param_exists": ctx_args.param_exists
         }
         return kwargs
@@ -417,7 +469,7 @@ class QueryModel(DataModel):
                 
                 # Copy metadata from upstream column
                 col.type = upstream_col.type if col.type == "" else col.type
-                col.condition = upstream_col.condition if col.condition == "" else col.condition
+                col.condition = upstream_col.condition if col.condition == [] else col.condition
                 col.description = upstream_col.description if col.description == "" else col.description
                 col.category = upstream_col.category if col.category == mc.ColumnCategory.MISC else col.category
 
@@ -432,7 +484,7 @@ class QueryModel(DataModel):
     def _log_sql_to_run(self, sql: str, placeholders: dict[str, Any]) -> None:
         log_msg = f"SQL to run for model '{self.name}':\n{sql}"
         log_msg += f"\n\n(with placeholders: {placeholders})"
-        self.logger.info(log_msg)
+        self.logger.debug(log_msg)
 
 
 @dataclass
@@ -457,11 +509,11 @@ class DbviewModel(QueryModel):
             if source_model.model_config.get_connection() != self.model_config.get_connection():
                 raise u.ConfigurationError(f'Dbview "{self.name}" references source "{source_name}" with different connection')
             
-            # Check if the source model has load_to_duckdb=False but this dbview has translate_to_duckdb=True
-            if not source_model.model_config.load_to_duckdb and self.model_config.translate_to_duckdb:
+            # Check if the source model has load_to_vdl=False but this dbview has translate_to_duckdb=True
+            if not source_model.model_config.load_to_vdl and self.model_config.translate_to_duckdb:
                 raise u.ConfigurationError(
                     f'Dbview "{self.name}" with translate_to_duckdb=True cannot reference source "{source_name}" '
-                    f'which has load_to_duckdb=False'
+                    f'which has load_to_vdl=False'
                 )
                 
             self.model_config.depends_on.add(source_name)
@@ -474,10 +526,11 @@ class DbviewModel(QueryModel):
 
     def _get_duckdb_query(self, read_dialect: str, query: str) -> str:
         kwargs = {
-            "source": lambda source_name: "venv." + source_name
+            "source": lambda source_name: "vdl." + source_name
         }
         compiled_query = self._get_compiled_sql_query_str(query, kwargs)
-        return sqlglot.transpile(compiled_query, read=read_dialect, write="duckdb")[0]
+        duckdb_query = sqlglot.transpile(compiled_query, read=read_dialect, write="duckdb", pretty=True)[0]
+        return "-- translated to duckdb\n" + duckdb_query
     
     def _compile_sql_model(self, kwargs: dict[str, Any]) -> mq.SqlModelQuery:
         compiled_query_str = self._get_compiled_sql_query_str(self.query_file.raw_query, kwargs)
@@ -486,15 +539,20 @@ class DbviewModel(QueryModel):
         connection_props = self.conn_set.get_connection(connection_name)
         
         if self.model_config.translate_to_duckdb and isinstance(connection_props, ConnectionProperties):
-            macros = { 
-                "source": lambda source_name: "venv." + source_name 
+            # Forbid translate_to_duckdb when dbview connection is duckdb
+            if connection_props.type == ConnectionTypeEnum.DUCKDB:
+                raise u.ConfigurationError(
+                    f'Dbview "{self.name}" has translate_to_duckdb=True but its connection is duckdb. Use a federate model instead.'
+                )
+            macros = {
+                "source": lambda source_name: "vdl." + source_name
             }
             compiled_query2 = self._get_compiled_sql_query_str(compiled_query_str, macros)
             compiled_query_str = self._get_duckdb_query(connection_props.dialect, compiled_query2)
             is_duckdb = True
         else:
-            macros = { 
-                "source": lambda source_name: self.sources[source_name].get_table() 
+            macros = {
+                "source": lambda source_name: self.sources[source_name].get_table()
             }
             compiled_query_str = self._get_compiled_sql_query_str(compiled_query_str, macros)
             is_duckdb = False
@@ -515,7 +573,14 @@ class DbviewModel(QueryModel):
         kwargs = self._get_compile_sql_model_args(ctx, ctx_args, models_dict)
         self.compiled_query = self._compile_sql_model(kwargs)
         
-        self.logger.log_activity_time(f"compiling dbview model '{self.name}'", start)
+        self.logger.log_activity_time(
+            f"compiling dbview model '{self.name}'", start, 
+            additional_data={
+                "activity": "compiling data model", 
+                "model_name": self.name, 
+                "model_type": self.model_type.value
+            }
+        )
     
     async def _run_sql_model(self, conn: duckdb.DuckDBPyConnection, placeholders: dict = {}) -> None:
         assert self.compiled_query is not None
@@ -531,7 +596,7 @@ class DbviewModel(QueryModel):
                         self.logger.info(f"Running dbview '{self.name}' on duckdb")
                         return local_conn.sql(query, params=placeholders).pl()
                     except duckdb.CatalogException as e:
-                        raise InvalidInputError(61, f'Model "{self.name}" depends on static data models that cannot be found.')
+                        raise InvalidInputError(409, f'dependent_data_model_not_found', f'Model "{self.name}" depends on static data models that cannot be found. Try building the Virtual Data Lake (VDL) first.')
                     except Exception as e:
                         raise RuntimeError(e)
                     finally:
@@ -551,7 +616,14 @@ class DbviewModel(QueryModel):
         
         await self._run_sql_model(conn, placeholders)
         
-        self.logger.log_activity_time(f"running dbview model '{self.name}'", start)
+        self.logger.log_activity_time(
+            f"running dbview model '{self.name}'", start, 
+            additional_data={
+                "activity": "running data model", 
+                "model_name": self.name, 
+                "model_type": self.model_type.value
+            }
+        )
         
         await super().run_model(conn, placeholders)
     
@@ -573,8 +645,16 @@ class FederateModel(QueryModel):
         
         def ref(dependent_model_name: str) -> str:
             dependent_model = self._ref_for_sql(dependent_model_name, models_dict)
-            prefix = "venv." if isinstance(models_dict[dependent_model], (SourceModel, BuildModel)) else ""
-            return prefix + dependent_model
+            dep = models_dict[dependent_model]
+            if isinstance(dep, BuildModel):
+                return "vdl." + dependent_model
+            if isinstance(dep, SourceModel):
+                if dep.model_config.load_to_vdl:
+                    return "vdl." + dependent_model
+                conn_name = dep.model_config.get_connection()
+                table_name = dep.model_config.get_table()
+                return f"db_{conn_name}.{table_name}"
+            return dependent_model
         
         kwargs["ref"] = ref
         return kwargs
@@ -591,12 +671,21 @@ class FederateModel(QueryModel):
         dependencies = self.model_config.depends_on
         connections = self.conn_set.get_connections_as_dict()
         
-        def run_external_sql(connection_name: str, sql_query: str) -> pl.DataFrame:
-            return self._run_sql_query_on_connection(connection_name, sql_query, ctx_args._placeholders_copy)
+        def _run_external_sql(connection_name: str, sql_query: str) -> pl.DataFrame:
+            return self._run_sql_query_on_connection(connection_name, sql_query, ctx_args._placeholders)
+        
+        build_model_args = BuildModelArgs(
+            **ctx_args._conn_args.__dict__,
+            connections=connections, dependencies=dependencies, 
+            _ref_func=self._ref_for_python, _run_external_sql_func=_run_external_sql
+        )
         
-        conn_args = ConnectionsArgs(ctx_args.project_path, ctx_args.proj_vars, ctx_args.env_vars)
-        build_model_args = BuildModelArgs(conn_args, connections, dependencies, self._ref_for_python, run_external_sql)
-        return ModelArgs(ctx_args, build_model_args, ctx)
+        # Instantiate ModelArgs with flattened arguments
+        combined_args = {
+            **ctx_args.__dict__, **build_model_args.__dict__, "ctx": ctx,
+        }
+        model_args = ModelArgs(**combined_args)
+        return model_args
 
     def _compile_python_model(
         self, query_file: mq.PyQueryFile, ctx: dict[str, Any], ctx_args: ContextArgs
@@ -628,7 +717,14 @@ class FederateModel(QueryModel):
         else:
             raise NotImplementedError(f"Query type not supported: {self.query_file.__class__.__name__}")
         
-        self.logger.log_activity_time(f"compiling federate model '{self.name}'", start)
+        self.logger.log_activity_time(
+            f"compiling federate model '{self.name}'", start, 
+            additional_data={
+                "activity": "compiling data model", 
+                "model_name": self.name, 
+                "model_type": self.model_type.value
+            }
+        )
         
         if not recurse:
             return 
@@ -657,10 +753,13 @@ class FederateModel(QueryModel):
                 try:
                     return local_conn.execute(create_query, existing_placeholders)
                 except duckdb.CatalogException as e:
-                    raise InvalidInputError(61, f'Model "{self.name}" depends on static data models that cannot be found.')
+                    if self.name == "__fake_target":
+                        raise InvalidInputError(409, "invalid_sql_query", f"Provided SQL query depends on static data models that cannot be found. Try building the Virtual Data Lake (VDL) first.")
+                    else:
+                        raise InvalidInputError(409, f'dependent_data_model_not_found', f'Model "{self.name}" depends on static data models that cannot be found. Try building the Virtual Data Lake (VDL) first.')
                 except Exception as e:
                     if self.name == "__fake_target":
-                        raise InvalidInputError(204, f"Failed to run provided SQL query")
+                        raise InvalidInputError(400, "invalid_sql_query", f"Failed to run provided SQL query")
                     else:
                         raise FileExecutionError(f'Failed to run federate sql model "{self.name}"', e) from e
             
@@ -687,7 +786,14 @@ class FederateModel(QueryModel):
         else:
             raise NotImplementedError(f"Query type not supported: {self.query_file.__class__.__name__}")
         
-        self.logger.log_activity_time(f"running federate model '{self.name}'", start)
+        self.logger.log_activity_time(
+            f"running federate model '{self.name}'", start, 
+            additional_data={
+                "activity": "running data model", 
+                "model_name": self.name, 
+                "model_type": self.model_type.value
+            }
+        )
         
         await super().run_model(conn, placeholders)
     
@@ -717,7 +823,12 @@ class BuildModel(StaticModel, QueryModel):
         }
         
         def ref_for_build(dependent_model_name: str) -> str:
-            dependent_model = self._ref_for_sql(dependent_model_name, dict(models_dict))
+            dependent_model = self._ref_for_sql(dependent_model_name, models_dict)
+            dep = models_dict[dependent_model]
+            if isinstance(dep, SourceModel) and not dep.model_config.load_to_vdl:
+                conn_name = dep.model_config.get_connection()
+                table_name = dep.model_config.get_table()
+                return f"db_{conn_name}.{table_name}"
             return dependent_model
         
         kwargs["ref"] = ref_for_build
@@ -740,11 +851,13 @@ class BuildModel(StaticModel, QueryModel):
     
     def _get_compile_python_model_args(self, conn_args: ConnectionsArgs) -> BuildModelArgs:
         
-        def run_external_sql(connection_name: str, sql_query: str):
+        def _run_external_sql(connection_name: str, sql_query: str):
             return self._run_sql_query_on_connection(connection_name, sql_query)
         
         return BuildModelArgs(
-            conn_args, self.conn_set.get_connections_as_dict(), self.model_config.depends_on, self._ref_for_python, run_external_sql
+            **conn_args.__dict__,
+            connections=self.conn_set.get_connections_as_dict(), dependencies=self.model_config.depends_on, 
+            _ref_func=self._ref_for_python, _run_external_sql_func=_run_external_sql
         )
 
     def _compile_python_model(
@@ -770,7 +883,14 @@ class BuildModel(StaticModel, QueryModel):
         else:
             raise NotImplementedError(f"Query type not supported: {self.query_file.__class__.__name__}")
         
-        self.logger.log_activity_time(f"compiling build model '{self.name}'", start)
+        self.logger.log_activity_time(
+            f"compiling build model '{self.name}'", start, 
+            additional_data={
+                "activity": "compiling data model", 
+                "model_name": self.name, 
+                "model_type": self.model_type.value
+            }
+        )
         
         dependencies = self.model_config.depends_on
         self.wait_count_for_build = len(dependencies)
@@ -785,14 +905,17 @@ class BuildModel(StaticModel, QueryModel):
         def create_table():
             create_query = self.model_config.get_sql_for_build(self.name, query)
             local_conn = conn.cursor()
+            # local_conn = conn
             try:
                 return u.run_duckdb_stmt(self.logger, local_conn, create_query, model_name=self.name)
             except Exception as e:
                 raise FileExecutionError(f'Failed to build static sql model "{self.name}"', e) from e
             finally:
                 local_conn.close()
+                # pass
         
-        await asyncio.to_thread(create_table)
+        # await asyncio.to_thread(create_table)
+        create_table() # without threading
 
     async def _build_python_model(self, compiled_query: mq.PyModelQuery, conn: duckdb.DuckDBPyConnection) -> None:
         query_result = await asyncio.to_thread(compiled_query.query)
@@ -800,7 +923,8 @@ class BuildModel(StaticModel, QueryModel):
             query_result = pl.from_pandas(query_result).lazy()
         if self.needs_python_df_for_build:
             self.result = query_result.lazy()
-        await asyncio.to_thread(self._create_table_from_df, conn, query_result)
+        # await asyncio.to_thread(self._create_table_from_df, conn, query_result)
+        self._create_table_from_df(conn, query_result) # without threading
 
     async def build_model(self, conn: duckdb.DuckDBPyConnection, full_refresh: bool) -> None:
         start = time.time()
@@ -813,24 +937,33 @@ class BuildModel(StaticModel, QueryModel):
             def load_df(conn: duckdb.DuckDBPyConnection, dep_model: DataModel):
                 if dep_model.result is None:
                     local_conn = conn.cursor()
+                    # local_conn = conn
                     try:
                         dep_model.result = dep_model._load_duckdb_view_to_python_df(local_conn)
                     finally:
                         local_conn.close()
+                        # pass
                 
             coroutines = []
             for dep_model in self.upstreams_for_build.values():
                 coro = asyncio.to_thread(load_df, conn, dep_model)
                 coroutines.append(coro)
             await u.asyncio_gather(coroutines)
-            
+
             # Then run the model's Python function to build the model
             await self._build_python_model(self.compiled_query, conn)
         else:
             raise NotImplementedError(f"Query type not supported: {self.query_file.__class__.__name__}")
         
         print(f"[{u.get_current_time()}] ✅ FINISHED: build model '{self.name}'")
-        self.logger.log_activity_time(f"building static build model '{self.name}'", start)
+        self.logger.log_activity_time(
+            f"building static build model '{self.name}' into VDL", start, 
+            additional_data={
+                "activity": "building data model into VDL", 
+                "model_name": self.name, 
+                "model_type": self.model_type.value
+            }
+        )
         
         await super().build_model(conn, full_refresh)
 
@@ -840,7 +973,7 @@ class DAG:
     dataset: DatasetConfig | None
     target_model: DataModel
     models_dict: dict[str, DataModel]
-    duckdb_filepath: str = field(default="")
+    datalake_db_path: str | None = field(default=None)
     logger: u.Logger = field(default_factory=lambda: u.Logger(""))
     parameter_set: ParameterSet | None = field(default=None, init=False) # set in apply_selections
     placeholders: dict[str, Any] = field(init=False, default_factory=dict)
@@ -849,36 +982,52 @@ class DAG:
         return f" for dataset '{self.dataset.name}'" if self.dataset else ""
     
     def compile_build_models(self, conn_args: ConnectionsArgs) -> None:
-        static_models: dict[str, StaticModel] = {k: v for k, v in self.models_dict.items() if isinstance(v, StaticModel)}
+        static_models: dict[str, StaticModel] = {
+            k: v for k, v in self.models_dict.items() if isinstance(v, StaticModel)
+        }
         for model in static_models.values():
             if isinstance(model, BuildModel):
                 model.compile_for_build(conn_args, static_models)
 
     def apply_selections(
-        self, param_cfg_set: ParameterConfigsSet, user: BaseUser | None, selections: dict[str, str]
+        self, param_cfg_set: ParameterConfigsSet, user: AbstractUser, selections: dict[str, str]
     ) -> None:
         start = time.time()
+        
         dataset_params = self.dataset.parameters if self.dataset else None
         parameter_set = param_cfg_set.apply_selections(dataset_params, selections, user)
         self.parameter_set = parameter_set
         msg_extension = self._get_msg_extension()
-        self.logger.log_activity_time("applying selections" + msg_extension, start)
+
+        dataset_name = self.dataset.name if self.dataset else None
+        self.logger.log_activity_time(
+            "applying selections" + msg_extension, start, 
+            additional_data={"activity": "applying selections", "dataset_name": dataset_name}
+        )
     
     def _compile_context(
-        self, param_args: ParametersArgs, context_func: ContextFunc, user: BaseUser | None, default_traits: dict[str, Any]
+        self, param_args: ParametersArgs, context_func: ContextFunc, user: AbstractUser, configurables: dict[str, str]
     ) -> tuple[dict[str, Any], ContextArgs]:
         start = time.time()
+
         context = {}
         assert isinstance(self.parameter_set, ParameterSet)
         prms = self.parameter_set.get_parameters_as_dict()
-        traits = self.dataset.traits if self.dataset else default_traits
-        args = ContextArgs(param_args, user, prms, traits)
+        args = ContextArgs(
+            **param_args.__dict__, user=user, prms=prms, configurables=configurables, _conn_args=param_args
+        )
         msg_extension = self._get_msg_extension()
+        
         try:
             context_func(context, args)
         except Exception as e:
             raise FileExecutionError(f'Failed to run {c.CONTEXT_FILE}' + msg_extension, e) from e
-        self.logger.log_activity_time("running context.py" + msg_extension, start)
+        
+        dataset_name = self.dataset.name if self.dataset else None
+        self.logger.log_activity_time(
+            "running context.py" + msg_extension, start, 
+            additional_data={"activity": "running context.py", "dataset_name": dataset_name}
+        )
         return context, args
     
     def _compile_models(self, context: dict[str, Any], ctx_args: ContextArgs, recurse: bool) -> None:
@@ -889,29 +1038,25 @@ class DAG:
         terminal_nodes = self.target_model.get_terminal_nodes(set())
         for model in self.models_dict.values():
             model.confirmed_no_cycles = False
-        self.logger.log_activity_time(f"validating no cycles in model dependencies", start)
+        self.logger.log_activity_time("validating no cycles in model dependencies", start)
         return terminal_nodes
 
+    def _attach_connections_with_type_duckdb(self, conn: duckdb.DuckDBPyConnection) -> None:
+        for conn_name, connection in self.target_model.conn_set.get_connections_as_dict().items():
+            if not isinstance(connection, ConnectionProperties):
+                continue
+            attach_uri = connection.attach_uri_for_duckdb
+            if attach_uri is None:
+                continue
+            attach_stmt = f"ATTACH IF NOT EXISTS '{attach_uri}' AS db_{conn_name} (READ_ONLY)"
+            u.run_duckdb_stmt(self.logger, conn, attach_stmt, redacted_values=[attach_uri])
+
     async def _run_models(self) -> None:
         terminal_nodes = self._get_terminal_nodes()
 
-        # create an empty duckdb venv file if it does not exist
+        conn = u.create_duckdb_connection(datalake_db_path=self.datalake_db_path)
         try:
-            conn = duckdb.connect(self.duckdb_filepath)
-            conn.close()
-        except duckdb.IOException as e:
-            # unable to create duckdb venv file means it's in use and already exists
-            # do not throw error here since attaching in read-only mode later may still work
-            pass
-        
-        conn = u.create_duckdb_connection()
-        try:
-            read_only = "(READ_ONLY)" if self.duckdb_filepath else ""
-            try:
-                conn.execute(f"ATTACH '{self.duckdb_filepath}' AS venv {read_only}")
-            except duckdb.IOException as e:
-                self.logger.warning(f"Unable to attach to duckdb venv file: {self.duckdb_filepath}")
-                raise e
+            self._attach_connections_with_type_duckdb(conn)
             
             coroutines = []
             for model_name in terminal_nodes:
@@ -923,18 +1068,18 @@ class DAG:
             conn.close()
     
     async def execute(
-        self, param_args: ParametersArgs, param_cfg_set: ParameterConfigsSet, context_func: ContextFunc, user: BaseUser | None, selections: dict[str, str], 
-        *, runquery: bool = True, recurse: bool = True, default_traits: dict[str, Any] = {}
+        self, param_args: ParametersArgs, param_cfg_set: ParameterConfigsSet, context_func: ContextFunc, user: AbstractUser, selections: dict[str, str], 
+        *, runquery: bool = True, recurse: bool = True, configurables: dict[str, str] = {}
     ) -> None:
         recurse = (recurse or runquery)
 
         self.apply_selections(param_cfg_set, user, selections)
 
-        context, ctx_args = self._compile_context(param_args, context_func, user, default_traits)
+        context, ctx_args = self._compile_context(param_args, context_func, user, configurables)
 
         self._compile_models(context, ctx_args, recurse)
         
-        self.placeholders = ctx_args._placeholders_copy
+        self.placeholders = dict(ctx_args._placeholders)
         if runquery:
             await self._run_models()
         
@@ -945,51 +1090,37 @@ class DAG:
         self.target_model.retrieve_dependent_query_models(all_model_names)
         return all_model_names
     
-    def to_networkx_graph(self) -> nx.DiGraph:
-        G = nx.DiGraph()
-
-        for model_name, model in self.models_dict.items():
-            level = model.get_max_path_length_to_target()
-            if level is not None:
-                G.add_node(model_name, layer=-level, model_type=model.model_type)
-        
-        for model_name in G.nodes:
-            model = self.models_dict[model_name]
-            for dep_model_name in model.downstreams:
-                G.add_edge(model_name, dep_model_name)
-        
-        return G
-    
-    def get_all_data_models(self) -> list[arm.DataModelItem]:
+    def get_all_data_models(self) -> list[rm.DataModelItem]:
         data_models = []
         for model_name, model in self.models_dict.items():
             is_queryable = model.is_queryable
-            data_model = arm.DataModelItem(name=model_name, model_type=model.model_type.value, config=model.model_config, is_queryable=is_queryable)
+            data_model = rm.DataModelItem(name=model_name, model_type=model.model_type.value, config=model.model_config, is_queryable=is_queryable)
             data_models.append(data_model)
         return data_models
     
-    def get_all_model_lineage(self) -> list[arm.LineageRelation]:
+    def get_all_model_lineage(self) -> list[rm.LineageRelation]:
         model_lineage = []
         for model_name, model in self.models_dict.items():
             if not isinstance(model, QueryModel):
                 continue
             for dep_model_name in model.model_config.depends_on:
                 edge_type = "buildtime" if isinstance(model, BuildModel) else "runtime"
-                source_model = arm.LineageNode(name=dep_model_name, type="model")
-                target_model = arm.LineageNode(name=model_name, type="model")
-                model_lineage.append(arm.LineageRelation(type=edge_type, source=source_model, target=target_model))
+                source_model = rm.LineageNode(name=dep_model_name, type="model")
+                target_model = rm.LineageNode(name=model_name, type="model")
+                model_lineage.append(rm.LineageRelation(type=edge_type, source=source_model, target=target_model))
         return model_lineage
 
 
 class ModelsIO:
 
     @classmethod
-    def _load_model_config(cls, filepath: Path, model_type: ModelType, env_vars: dict[str, str]) -> mc.ModelConfig:
+    def _load_model_config(cls, filepath: Path, model_type: ModelType, env_vars: SquirrelsEnvVars) -> mc.ModelConfig:
         yaml_path = filepath.with_suffix('.yml')
         config_dict = u.load_yaml_config(yaml_path) if yaml_path.exists() else {}
         
         if model_type == ModelType.DBVIEW:
-            config = mc.DbviewModelConfig(**config_dict).finalize_connection(env_vars)
+            default_conn_name = env_vars.connections_default_name_used
+            config = mc.DbviewModelConfig(**config_dict).finalize_connection(default_conn_name=default_conn_name)
             return config
         elif model_type == ModelType.FEDERATE:
             return mc.FederateModelConfig(**config_dict)
@@ -1000,13 +1131,13 @@ class ModelsIO:
 
     @classmethod
     def _populate_from_file(
-        cls, raw_queries_by_model: dict[str, mq.QueryFileWithConfig], dp: str, file: str, model_type: ModelType, env_vars: dict[str, str]
+        cls, raw_queries_by_model: dict[str, mq.QueryFileWithConfig], dp: str, file: str, model_type: ModelType, env_vars: SquirrelsEnvVars
     ) -> None:
         filepath = Path(dp, file)
         file_stem, extension = os.path.splitext(file)
         
         if extension == '.py':
-            module = pm.PyModule(filepath)
+            module = pm.PyModule(filepath, project_path=env_vars.project_path)
             raw_query = module.get_func_or_class(c.MAIN_FUNC)
             query_file = mq.PyQueryFile(filepath.as_posix(), raw_query)
         elif extension == '.sql':
@@ -1024,7 +1155,7 @@ class ModelsIO:
 
     @classmethod
     def _populate_raw_queries_for_type(
-        cls, folder_path: Path, model_type: ModelType, *, env_vars: dict[str, str] = {}
+        cls, folder_path: Path, model_type: ModelType, env_vars: SquirrelsEnvVars
     ) -> dict[str, mq.QueryFileWithConfig]:
         raw_queries_by_model: dict[str, mq.QueryFileWithConfig] = {}
         for dp, _, filenames in os.walk(folder_path):
@@ -1033,35 +1164,37 @@ class ModelsIO:
         return raw_queries_by_model
 
     @classmethod
-    def load_build_files(cls, logger: u.Logger, base_path: str) -> dict[str, mq.QueryFileWithConfig]:
+    def load_build_files(cls, logger: u.Logger, env_vars: SquirrelsEnvVars) -> dict[str, mq.QueryFileWithConfig]:
         start = time.time()
-        builds_path = u.Path(base_path, c.MODELS_FOLDER, c.BUILDS_FOLDER)
-        raw_queries_by_model = cls._populate_raw_queries_for_type(builds_path, ModelType.BUILD)
+        builds_path = u.Path(env_vars.project_path, c.MODELS_FOLDER, c.BUILDS_FOLDER)
+        raw_queries_by_model = cls._populate_raw_queries_for_type(builds_path, ModelType.BUILD, env_vars=env_vars)
         logger.log_activity_time("loading build files", start)
         return raw_queries_by_model
 
     @classmethod
-    def load_dbview_files(cls, logger: u.Logger, base_path: str, env_vars: dict[str, str]) -> dict[str, mq.QueryFileWithConfig]:
+    def load_dbview_files(cls, logger: u.Logger, env_vars: SquirrelsEnvVars) -> dict[str, mq.QueryFileWithConfig]:
         start = time.time()
-        dbviews_path = u.Path(base_path, c.MODELS_FOLDER, c.DBVIEWS_FOLDER)
+        dbviews_path = u.Path(env_vars.project_path, c.MODELS_FOLDER, c.DBVIEWS_FOLDER)
         raw_queries_by_model = cls._populate_raw_queries_for_type(dbviews_path, ModelType.DBVIEW, env_vars=env_vars)
         logger.log_activity_time("loading dbview files", start)
         return raw_queries_by_model
 
     @classmethod
-    def load_federate_files(cls, logger: u.Logger, base_path: str) -> dict[str, mq.QueryFileWithConfig]:
+    def load_federate_files(cls, logger: u.Logger, env_vars: SquirrelsEnvVars) -> dict[str, mq.QueryFileWithConfig]:
         start = time.time()
-        federates_path = u.Path(base_path, c.MODELS_FOLDER, c.FEDERATES_FOLDER)
-        raw_queries_by_model = cls._populate_raw_queries_for_type(federates_path, ModelType.FEDERATE)
+        federates_path = u.Path(env_vars.project_path, c.MODELS_FOLDER, c.FEDERATES_FOLDER)
+        raw_queries_by_model = cls._populate_raw_queries_for_type(federates_path, ModelType.FEDERATE, env_vars=env_vars)
         logger.log_activity_time("loading federate files", start)
         return raw_queries_by_model
 
     @classmethod
-    def load_context_func(cls, logger: u.Logger, base_path: str) -> ContextFunc:
+    def load_context_func(cls, logger: u.Logger, project_path: str) -> ContextFunc:
         start = time.time()
 
-        context_path = u.Path(base_path, c.PYCONFIGS_FOLDER, c.CONTEXT_FILE)
-        context_func: ContextFunc = pm.PyModule(context_path).get_func_or_class(c.MAIN_FUNC, default_attr=lambda ctx, sqrl: None)
+        context_path = u.Path(project_path, c.PYCONFIGS_FOLDER, c.CONTEXT_FILE)
+        context_func: ContextFunc = pm.PyModule(
+            context_path, project_path=project_path
+        ).get_func_or_class(c.MAIN_FUNC, default_attr=lambda ctx, sqrl: None)
 
         logger.log_activity_time("loading file for context.py", start)
         return context_func