squirrels 0.2.0rc1__py3-none-any.whl → 0.2.2__py3-none-any.whl

squirrels/__init__.py

@@ -1,8 +1,8 @@
-__version__ = '0.2.0'
+__version__ = '0.2.2'
 
 from .arguments.init_time_args import ConnectionsArgs, ParametersArgs
 from .arguments.run_time_args import AuthArgs, ContextArgs, ModelDepsArgs, ModelArgs
 from .parameter_options import SelectParameterOption, DateParameterOption, DateRangeParameterOption, NumberParameterOption, NumberRangeParameterOption
-from .parameters import Parameter, SingleSelectParameter, MultiSelectParameter, DateParameter, DateRangeParameter, NumberParameter, NumberRangeParameter
+from .parameters import SingleSelectParameter, MultiSelectParameter, DateParameter, DateRangeParameter, NumberParameter, NumberRangeParameter
 from .data_sources import SingleSelectDataSource, MultiSelectDataSource, DateDataSource, DateRangeDataSource, NumberDataSource, NumberRangeDataSource
 from .user_base import User, WrongPassword

squirrels/_api_server.py

@@ -1,5 +1,5 @@
-from typing import Iterable, Optional, Mapping, Callable, Coroutine, TypeVar, Any
-from fastapi import Depends, FastAPI, Request, HTTPException, status
+from typing import List, Iterable, Optional, Mapping, Callable, Coroutine, TypeVar, Any
+from fastapi import Depends, FastAPI, Request, HTTPException, Response, status
 from fastapi.responses import HTMLResponse, JSONResponse
 from fastapi.templating import Jinja2Templates
 from fastapi.staticfiles import StaticFiles
@@ -45,36 +45,35 @@ class ApiServer:
         start = time.time()
         app = FastAPI()
 
-        app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"])
+        @app.middleware("http")
+        async def catch_exceptions_middleware(request: Request, call_next):
+            try:
+                return await call_next(request)
+            except u.InvalidInputError as exc:
+                traceback.print_exc()
+                return JSONResponse(status_code=status.HTTP_400_BAD_REQUEST, 
+                                    content={"message": f"Invalid user input: {str(exc)}"})
+            except u.ConfigurationError as exc:
+                traceback.print_exc()
+                return JSONResponse(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 
+                                    content={"message": f"Squirrels configuration error: {str(exc)}"})
+            except NotImplementedError as exc:
+                traceback.print_exc()
+                return JSONResponse(status_code=status.HTTP_501_NOT_IMPLEMENTED, 
+                                    content={"message": f"Not implemented error: {str(exc)}"})
+            except Exception as exc:
+                traceback.print_exc()
+                return JSONResponse(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 
+                                    content={"message": f"Server error: {str(exc)}"})
+
+        app.add_middleware(
+            CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], 
+            expose_headers=["Applied-Username"]
+        )
 
         squirrels_version_path = f'/squirrels-v{sq_major_version}'
         partial_base_path = f'/{ManifestIO.obj.project_variables.get_name()}/v{ManifestIO.obj.project_variables.get_major_version()}'
         base_path = squirrels_version_path + u.normalize_name_for_api(partial_base_path)
-
-        static_dir = u.join_paths(os.path.dirname(__file__), c.PACKAGE_DATA_FOLDER, c.ASSETS_FOLDER)
-        app.mount('/'+c.ASSETS_FOLDER, StaticFiles(directory=static_dir), name=c.ASSETS_FOLDER)
-
-        templates_dir = u.join_paths(os.path.dirname(__file__), c.PACKAGE_DATA_FOLDER, c.TEMPLATES_FOLDER)
-        templates = Jinja2Templates(directory=templates_dir)
-
-        # Exception handlers
-        @app.exception_handler(u.InvalidInputError)
-        async def invalid_input_error_handler(request: Request, exc: u.InvalidInputError):
-            traceback.print_exc()
-            return JSONResponse(status_code=status.HTTP_400_BAD_REQUEST, 
-                                content={"message": f"Invalid user input: {str(exc)}"})
-
-        @app.exception_handler(u.ConfigurationError)
-        async def configuration_error_handler(request: Request, exc: u.InvalidInputError):
-            traceback.print_exc()
-            return JSONResponse(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 
-                                content={"message": f"Squirrels configuration error: {str(exc)}"})
-        
-        @app.exception_handler(NotImplementedError)
-        async def not_implemented_error_handler(request: Request, exc: u.InvalidInputError):
-            traceback.print_exc()
-            return JSONResponse(status_code=status.HTTP_501_NOT_IMPLEMENTED, 
-                                content={"message": f"Not implemented error: {str(exc)}"})
         
         # Helpers
         T = TypeVar('T')
@@ -127,13 +126,21 @@ class ApiServer:
             # Changing selections into a cachable "frozenset" that will later be converted to dictionary
             selections = set()
             for key, val in params.items():
-                if not isinstance(val, str):
+                if isinstance(val, List):
                     val = tuple(val)
                 selections.add((u.normalize_name(key), val))
             selections = frozenset(selections)
             
             return await api_function(user, dataset_normalized, selections, request_version)
 
+        async def do_cachable_action(cache: TTLCache, action: Callable[..., Coroutine[Any, Any, T]], *args) -> T:
+            cache_key = tuple(args)
+            result = cache.get(cache_key)
+            if result is None:
+                result = await action(*args)
+                cache[cache_key] = result
+            return result
+        
         # Login
         token_path = base_path + '/token'
 
@@ -154,20 +161,14 @@ class ApiServer:
                 "expiry_time": expiry
             }
         
-        async def get_current_user(token: str = Depends(oauth2_scheme)) -> Optional[User]:
+        async def get_current_user(response: Response, token: str = Depends(oauth2_scheme)) -> Optional[User]:
             user = self.authenticator.get_user_from_token(token)
+            username = "" if user is None else user.username
+            response.headers["Applied-Username"] = username
             return user
 
-        async def do_cachable_action(cache: TTLCache, action: Callable[..., Coroutine[Any, Any, T]], *args) -> T:
-            cache_key = tuple(args)
-            result = cache.get(cache_key)
-            if result is None:
-                result = await action(*args)
-                cache[cache_key] = result
-            return result
-
         # Parameters API
-        parameters_path = base_path + '/{dataset}/parameters'
+        parameters_path = base_path + '/dataset/{dataset}/parameters'
         
         parameters_cache_size = ManifestIO.obj.settings.get(c.PARAMETERS_CACHE_SIZE_SETTING, 1024)
         parameters_cache_ttl = ManifestIO.obj.settings.get(c.PARAMETERS_CACHE_TTL_SETTING, 0)
@@ -209,7 +210,7 @@ class ApiServer:
             return result
 
         # Results API
-        results_path = base_path + '/{dataset}'
+        results_path = base_path + '/dataset/{dataset}'
 
         results_cache_size = ManifestIO.obj.settings.get(c.RESULTS_CACHE_SIZE_SETTING, 128)
         results_cache_ttl = ManifestIO.obj.settings.get(c.RESULTS_CACHE_TTL_SETTING, 0)
@@ -248,8 +249,10 @@ class ApiServer:
             timer.add_activity_time("POST REQUEST total time for DATASET", start)
             return result
         
-        # Catalog API
-        def get_catalog0(user: Optional[User]):
+        # Datasets Catalog API
+        datasets_path = base_path + '/datasets'
+
+        def get_datasets0(user: Optional[User]):
             datasets_info = []
             for dataset_name, dataset_config in self.dataset_configs.items():
                 if can_user_access_dataset(user, dataset_name):
@@ -258,30 +261,44 @@ class ApiServer:
                         'name': dataset_name,
                         'label': dataset_config.label,
                         'parameters_path': parameters_path.format(dataset=dataset_normalized),
-                        'result_path': results_path.format(dataset=dataset_normalized),
-                        'first_minor_version': 0
+                        'result_path': results_path.format(dataset=dataset_normalized)
                     })
-            
+            return {"datasets": datasets_info}
+        
+        @app.get(datasets_path)
+        def get_datasets(request: Request, user: Optional[User] = Depends(get_current_user)):
+            return process_based_on_response_version_header(request.headers, {
+                0: lambda: get_datasets0(user)
+            })
+        
+        # Projects Catalog API
+        def get_catalog0():
             return {
                 'projects': [{
                     'name': ManifestIO.obj.project_variables.get_name(),
                     'label': ManifestIO.obj.project_variables.get_label(),
                     'versions': [{
                         'major_version': ManifestIO.obj.project_variables.get_major_version(),
-                        'latest_minor_version': ManifestIO.obj.project_variables.get_minor_version(),
+                        'minor_versions': [0],
                         'token_path': token_path,
-                        'datasets': datasets_info
+                        'datasets_path': datasets_path
                     }]
                 }]
             }
         
         @app.get(squirrels_version_path, response_class=JSONResponse)
-        async def get_catalog(request: Request, user: Optional[User] = Depends(get_current_user)):
+        async def get_catalog(request: Request):
             return process_based_on_response_version_header(request.headers, {
-                0: lambda: get_catalog0(user)
+                0: lambda: get_catalog0()
             })
         
         # Squirrels UI
+        static_dir = u.join_paths(os.path.dirname(__file__), c.PACKAGE_DATA_FOLDER, c.ASSETS_FOLDER)
+        app.mount('/'+c.ASSETS_FOLDER, StaticFiles(directory=static_dir), name=c.ASSETS_FOLDER)
+
+        templates_dir = u.join_paths(os.path.dirname(__file__), c.PACKAGE_DATA_FOLDER, c.TEMPLATES_FOLDER)
+        templates = Jinja2Templates(directory=templates_dir)
+
         @app.get('/', response_class=HTMLResponse)
         async def get_ui(request: Request):
             return templates.TemplateResponse('index.html', {

squirrels/_authenticator.py

@@ -34,7 +34,7 @@ class Authenticator:
         return AuthArgs(conn_args.proj_vars, conn_args.env_vars, conn_args._get_credential, connections, username, password)
 
     def authenticate_user(self, username: str, password: str) -> Optional[User]:
-        user_cls = self.auth_helper.get_func_or_class("User", default_attr=User)
+        user_cls: type[User] = self.auth_helper.get_func_or_class("User", default_attr=User)
         get_user = self.auth_helper.get_func_or_class(c.GET_USER_FUNC, is_required=False)
         try:
             real_user = get_user(self._get_auth_args(username, password)) if get_user is not None else None
@@ -48,9 +48,8 @@ class Authenticator:
             fake_users = EnvironConfigIO.obj.get_users()
             if username in fake_users and secrets.compare_digest(fake_users[username][c.USER_PWD_KEY], password):
                 is_internal = fake_users[username].get("is_internal", False)
-                user: User = user_cls(username, is_internal=is_internal)
                 try:
-                    return user.with_attributes(fake_users[username])
+                    return user_cls.Create(username, fake_users[username], is_internal=is_internal)
                 except Exception as e:
                     raise u.FileExecutionError(f'Failed to create user from User model in {c.AUTH_FILE}', e)
         

squirrels/_command_line.py

@@ -33,7 +33,7 @@ def main():
 
     compile_parser = subparsers.add_parser(c.COMPILE_CMD, help='Create files for rendered sql queries in the "target/compile" folder', add_help=False)
     compile_parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
-    compile_parser.add_argument('-d', '--dataset', type=str, help="Select dataset to use for dataset args. If not specified, all models for all datasets are compiled")
+    compile_parser.add_argument('-d', '--dataset', type=str, help="Select dataset to use for dataset traits. If not specified, all models for all datasets are compiled")
     compile_parser.add_argument('-a', '--all-test-sets', action="store_true", help="Compile models for all selection test sets")
     compile_parser.add_argument('-t', '--test-set', type=str, help="The selection test set to use. Default selections are used if not specified. Ignored if using --all-test-sets")
     compile_parser.add_argument('-s', '--select', type=str, help="Select single model to compile. If not specified, all models for the dataset are compiled. Also, ignored if --dataset is not specified")

squirrels/_constants.py

@@ -9,7 +9,6 @@ PROJ_VARS_KEY = 'project_variables'
 PROJECT_NAME_KEY = 'name'
 PROJECT_LABEL_KEY = 'label'
 MAJOR_VERSION_KEY = 'major_version'
-MINOR_VERSION_KEY = 'minor_version'
 
 PACKAGES_KEY = 'packages'
 PACKAGE_GIT_KEY = 'git'
@@ -48,7 +47,7 @@ DATASET_NAME_KEY = 'name'
 DATASET_LABEL_KEY = 'label'
 DATASET_MODEL_KEY = 'model'
 DATASET_PARAMETERS_KEY = 'parameters'
-DATASET_ARGS_KEY = 'args'
+DATASET_TRAITS_KEY = 'traits'
 
 DATASET_SCOPE_KEY = 'scope'
 PUBLIC_SCOPE = 'public'
@@ -109,14 +108,17 @@ PARAMETERS_OUTPUT = 'parameters.json'
 FINAL_VIEW_OUT_STEM = 'final_view'
 
 # Dataset setting names
-AUTH_TOKEN_EXPIRE_SETTING = 'auth.token.expire.minutes'
+AUTH_TOKEN_EXPIRE_SETTING = 'auth.token.expire_minutes'
 PARAMETERS_CACHE_SIZE_SETTING = 'parameters.cache.size'
-PARAMETERS_CACHE_TTL_SETTING = 'parameters.cache.ttl.minutes'
+PARAMETERS_CACHE_TTL_SETTING = 'parameters.cache.ttl_minutes'
 RESULTS_CACHE_SIZE_SETTING = 'results.cache.size'
-RESULTS_CACHE_TTL_SETTING = 'results.cache.ttl.minutes'
+RESULTS_CACHE_TTL_SETTING = 'results.cache.ttl_minutes'
 TEST_SET_DEFAULT_USED_SETTING = 'selection_test_sets.default_name_used'
 DB_CONN_DEFAULT_USED_SETTING = 'connections.default_name_used'
 DEFAULT_MATERIALIZE_SETTING = 'defaults.federates.materialized'
+IN_MEMORY_DB_SETTING = 'in_memory_database'
+SQLITE = 'sqlite'
+DUCKDB = 'duckdb'
 
 # Selection cfg sections
 USER_ATTRIBUTES_SECTION = 'user_attributes'

squirrels/_environcfg.py

@@ -43,7 +43,7 @@ class _EnvironConfig:
         return credential[c.USERNAME_KEY], credential[c.PASSWORD_KEY]
     
     def get_secret(self, key: str, *, default_factory: Optional[Callable[[],str]] = None) -> str:
-        if key not in self._secrets and default_factory is not None:
+        if not self._secrets.get(key) and default_factory is not None:
             self._secrets[key] = default_factory()
         return self._secrets.get(key)
 

squirrels/_initializer.py

@@ -40,7 +40,6 @@ class Initializer:
         options = ["core", "connections", "parameters", "dbview", "federate", "auth", "sample_db"]
         CORE, CONNECTIONS, PARAMETERS, DBVIEW, FEDERATE, AUTH, SAMPLE_DB = options
         TMP_FOLDER = "tmp"
-        IGNORES_FOLDER = "ignores"
 
         answers = { x: getattr(args, x) for x in options }
         if not any(answers.values()):
@@ -136,7 +135,7 @@ class Initializer:
 
             create_manifest_file()
             
-            self._copy_file(".gitignore", src_folder=IGNORES_FOLDER)
+            self._copy_file(".gitignore")
             self._copy_file(c.MANIFEST_FILE, src_folder=TMP_FOLDER)
             
             if connections_use_py:

squirrels/_manifest.py

@@ -26,10 +26,10 @@ class ProjectVarsConfig(ManifestComponentConfig):
     data: dict
 
     def __post_init__(self):
-        required_keys = [c.PROJECT_NAME_KEY, c.MAJOR_VERSION_KEY, c.MINOR_VERSION_KEY]
+        required_keys = [c.PROJECT_NAME_KEY, c.MAJOR_VERSION_KEY]
         self._validate_required(self.data, required_keys, c.PROJ_VARS_KEY)
         
-        integer_keys = [c.MAJOR_VERSION_KEY, c.MINOR_VERSION_KEY]
+        integer_keys = [c.MAJOR_VERSION_KEY]
         for key in integer_keys:
             if key in self.data and not isinstance(self.data[key], int):
                 raise u.ConfigurationError(f'Project variable "{key}" must be an integer')
@@ -46,9 +46,6 @@ class ProjectVarsConfig(ManifestComponentConfig):
     
     def get_major_version(self) -> int:
         return self.data[c.MAJOR_VERSION_KEY]
-    
-    def get_minor_version(self) -> int:
-        return self.data[c.MINOR_VERSION_KEY]
 
 
 @dataclass
@@ -84,14 +81,13 @@ class DbConnConfig(ManifestComponentConfig):
 
 @dataclass
 class ParametersConfig(ManifestComponentConfig):
-    name: str
     type: str
     factory: str
     arguments: dict
 
     @classmethod
     def from_dict(cls, kwargs: dict):
-        all_keys = [c.PARAMETER_NAME_KEY, c.PARAMETER_TYPE_KEY, c.PARAMETER_FACTORY_KEY, c.PARAMETER_ARGS_KEY]
+        all_keys = [c.PARAMETER_TYPE_KEY, c.PARAMETER_FACTORY_KEY, c.PARAMETER_ARGS_KEY]
         cls._validate_required(kwargs, all_keys, c.PARAMETERS_KEY)
         args = {key: kwargs[key] for key in all_keys}
         return cls(**args)
@@ -150,7 +146,7 @@ class DatasetsConfig(ManifestComponentConfig):
     model: str
     scope: DatasetScope
     parameters: Optional[list[str]]
-    args: dict
+    traits: dict
 
     @classmethod
     def from_dict(cls, kwargs: dict):
@@ -166,8 +162,8 @@ class DatasetsConfig(ManifestComponentConfig):
             raise u.ConfigurationError(f'Scope not found for dataset "{name}". Scope must be one of {scope_list}') from e
         
         parameters = kwargs.get(c.DATASET_PARAMETERS_KEY)
-        args = kwargs.get(c.DATASET_ARGS_KEY, {})
-        return cls(name, label, model, scope, parameters, args)
+        traits = kwargs.get(c.DATASET_TRAITS_KEY, {})
+        return cls(name, label, model, scope, parameters, traits)
 
 
 @dataclass
@@ -175,7 +171,7 @@ class _ManifestConfig:
     project_variables: ProjectVarsConfig
     packages: list[PackageConfig]
     connections: dict[str, DbConnConfig]
-    parameters: dict[str, ParametersConfig]
+    parameters: list[ParametersConfig]
     selection_test_sets: dict[str, TestSetsConfig]
     dbviews: dict[str, DbviewConfig]
     federates: dict[str, FederateConfig]
@@ -209,7 +205,7 @@ class _ManifestConfig:
             all_package_dirs.add(package.directory)
 
         db_conns = cls._create_configs_as_dict(DbConnConfig, kwargs, c.DB_CONNECTIONS_KEY, c.DB_CONN_NAME_KEY)
-        params = cls._create_configs_as_dict(ParametersConfig, kwargs, c.PARAMETERS_KEY, c.PARAMETER_NAME_KEY)
+        params = [ParametersConfig.from_dict(x) for x in kwargs.get(c.PARAMETERS_KEY, [])]
 
         test_sets = cls._create_configs_as_dict(TestSetsConfig, kwargs, c.TEST_SETS_KEY, c.TEST_SET_NAME_KEY)
         default_test_set: str = settings.get(c.TEST_SET_DEFAULT_USED_SETTING, c.DEFAULT_TEST_SET_NAME)

squirrels/_models.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
-from typing import Optional, Callable, Iterable, Any
+from typing import Union, Optional, Callable, Iterable, Any
 from dataclasses import dataclass, field
 from enum import Enum
 from pathlib import Path
@@ -7,7 +7,7 @@ import sqlite3, pandas as pd, asyncio, os, shutil
 
 from . import _constants as c, _utils as u, _py_module as pm
 from .arguments.run_time_args import ContextArgs, ModelDepsArgs, ModelArgs
-from .user_base import User
+from ._authenticator import User, Authenticator
 from ._connection_set import ConnectionSetIO
 from ._manifest import ManifestIO, DatasetsConfig
 from ._parameter_sets import ParameterConfigsSetIO, ParameterSet
@@ -146,7 +146,7 @@ class Model:
         configuration = SqlModelConfig(connection_name, materialized)
         kwargs = {
             "proj_vars": ctx_args.proj_vars, "env_vars": ctx_args.env_vars,
-            "user": ctx_args.user, "prms": ctx_args.prms, "args": ctx_args.args,
+            "user": ctx_args.user, "prms": ctx_args.prms, "traits": ctx_args.traits,
             "ctx": ctx, "config": configuration.set_attribute
         }
         dependencies = set()
@@ -166,7 +166,7 @@ class Model:
     
     async def _compile_python_model(self, ctx: dict[str, Any], ctx_args: ContextArgs) -> tuple[PyModelQuery, set]:
         assert(isinstance(self.query_file.raw_query, RawPyQuery))
-        sqrl_args = ModelDepsArgs(ctx_args.proj_vars, ctx_args.env_vars, ctx_args.user, ctx_args.prms, ctx_args.args, ctx)
+        sqrl_args = ModelDepsArgs(ctx_args.proj_vars, ctx_args.env_vars, ctx_args.user, ctx_args.prms, ctx_args.traits, ctx)
         try:
             dependencies = await asyncio.to_thread(self.query_file.raw_query.dependencies_func, sqrl_args)
         except Exception as e:
@@ -175,7 +175,7 @@ class Model:
         dbview_conn_name = self._get_dbview_conn_name()
         connections = ConnectionSetIO.obj.get_engines_as_dict()
         ref = lambda x: self.upstreams[x].result
-        sqrl_args = ModelArgs(ctx_args.proj_vars, ctx_args.env_vars, ctx_args.user, ctx_args.prms, ctx_args.args, 
+        sqrl_args = ModelArgs(ctx_args.proj_vars, ctx_args.env_vars, ctx_args.user, ctx_args.prms, ctx_args.traits, 
                               ctx, dbview_conn_name, connections, ref, set(dependencies))
             
         def compiled_query():
@@ -215,9 +215,9 @@ class Model:
             coroutines.append(coro)
         await asyncio.gather(*coroutines)
     
-    def validate_no_cycles(self, depencency_path: set[str]) -> set[str]:
+    def get_terminal_nodes(self, depencency_path: set[str]) -> set[str]:
         if self.confirmed_no_cycles:
-            return
+            return set()
         
         if self.name in depencency_path:
             raise u.ConfigurationError(f'Cycle found in model dependency graph')
@@ -229,11 +229,24 @@ class Model:
             new_path = set(depencency_path)
             new_path.add(self.name)
             for dep_model in self.upstreams.values():
-                terminal_nodes_under_dep = dep_model.validate_no_cycles(new_path)
+                terminal_nodes_under_dep = dep_model.get_terminal_nodes(new_path)
                 terminal_nodes = terminal_nodes.union(terminal_nodes_under_dep)
         
         self.confirmed_no_cycles = True
         return terminal_nodes
+        
+    def _load_pandas_to_table(self, df: pd.DataFrame, conn: sqlite3.Connection) -> None:
+        if u.use_duckdb():
+            conn.execute(f"CREATE TABLE {self.name} AS FROM df")
+        else:
+            df.to_sql(self.name, conn, index=False)
+            
+    def _load_table_to_pandas(self, conn: sqlite3.Connection) -> pd.DataFrame:
+        if u.use_duckdb():
+            return conn.execute(f"FROM {self.name}").df()
+        else:
+            query = f"SELECT * FROM {self.name}"
+            return pd.read_sql(query, conn)
 
     async def _run_sql_model(self, conn: sqlite3.Connection) -> None:
         assert(isinstance(self.compiled_query, SqlModelQuery))
@@ -248,7 +261,7 @@ class Model:
                     raise u.FileExecutionError(f'Failed to run dbview sql model "{self.name}"', e)
             
             df = await asyncio.to_thread(run_sql_query)
-            await asyncio.to_thread(df.to_sql, self.name, conn, index=False)
+            await asyncio.to_thread(self._load_pandas_to_table, df, conn)
             if self.needs_pandas or self.is_target:
                 self.result = df
         elif self.query_file.model_type == ModelType.FEDERATE:
@@ -261,15 +274,14 @@ class Model:
             
             await asyncio.to_thread(create_table)
             if self.needs_pandas or self.is_target:
-                query = f"SELECT * FROM {self.name}"
-                self.result = await asyncio.to_thread(pd.read_sql, query, conn)
+                self.result = await asyncio.to_thread(self._load_table_to_pandas, conn)
     
     async def _run_python_model(self, conn: sqlite3.Connection) -> None:
         assert(isinstance(self.compiled_query, PyModelQuery))
 
         df = await asyncio.to_thread(self.compiled_query.query)
         if self.needs_sql_table:
-            await asyncio.to_thread(df.to_sql, self.name, conn, index=False)
+            await asyncio.to_thread(self._load_pandas_to_table, df, conn)
         if self.needs_pandas or self.is_target:
             self.result = df
     
@@ -320,7 +332,7 @@ class DAG:
         context = {}
         param_args = ParameterConfigsSetIO.args
         prms = self.parameter_set.get_parameters_as_dict()
-        args = ContextArgs(param_args.proj_vars, param_args.env_vars, user, prms, self.dataset.args)
+        args = ContextArgs(param_args.proj_vars, param_args.env_vars, user, prms, self.dataset.traits)
         try:
             context_func(ctx=context, sqrl=args)
         except Exception as e:
@@ -331,14 +343,21 @@ class DAG:
     async def _compile_models(self, context: dict[str, Any], ctx_args: ContextArgs, recurse: bool) -> None:
         await self.target_model.compile(context, ctx_args, self.models_dict, recurse)
     
-    def _validate_no_cycles(self) -> set[str]:
+    def _get_terminal_nodes(self) -> set[str]:
         start = time.time()
-        terminal_nodes = self.target_model.validate_no_cycles(set())
+        terminal_nodes = self.target_model.get_terminal_nodes(set())
+        for model in self.models_dict.values():
+            model.confirmed_no_cycles = False
         timer.add_activity_time(f"validating no cycles in models dependencies", start)
         return terminal_nodes
 
     async def _run_models(self, terminal_nodes: set[str]) -> None:
-        conn = sqlite3.connect(":memory:", check_same_thread=False)
+        if u.use_duckdb():
+            import duckdb
+            conn = duckdb.connect()
+        else:
+            conn = sqlite3.connect(":memory:", check_same_thread=False)
+        
         try:
             coroutines = []
             for model_name in terminal_nodes:
@@ -360,7 +379,7 @@ class DAG:
 
         await self._compile_models(context, ctx_args, recurse)
         
-        terminal_nodes = self._validate_no_cycles()
+        terminal_nodes = self._get_terminal_nodes()
 
         if runquery:
             await self._run_models(terminal_nodes)
@@ -431,10 +450,13 @@ class ModelsIO:
     @classmethod
     async def WriteDatasetOutputsGivenTestSet(cls, dataset: str, select: str, test_set: str, runquery: bool, recurse: bool) -> Any:
         test_set_conf = ManifestIO.obj.selection_test_sets[test_set]
-        user = User("")
-        for key, val in test_set_conf.user_attributes.items():
-            setattr(user, key, val)
+        user_attributes = test_set_conf.user_attributes
         selections = test_set_conf.parameters
+        
+        username, is_internal = user_attributes.get("username", ""), user_attributes.get("is_internal", False)
+        user_cls: type[User] = Authenticator.get_auth_helper().get_func_or_class("User", default_attr=User)
+        user = user_cls.Create(username, test_set_conf.user_attributes, is_internal=is_internal)
+        
         dag = cls.GenerateDAG(dataset, target_model_name=select, always_pandas=True)
         await dag.execute(cls.context_func, user, selections, runquery=runquery, recurse=recurse)
         
@@ -445,7 +467,7 @@ class ModelsIO:
         def write_model_outputs(model: Model) -> None:
             subfolder = c.DBVIEWS_FOLDER if model.query_file.model_type == ModelType.DBVIEW else c.FEDERATES_FOLDER
             subpath = u.join_paths(output_folder, subfolder)
-            os.makedirs(subpath)
+            os.makedirs(subpath, exist_ok=True)
             if isinstance(model.compiled_query, SqlModelQuery):
                 output_filepath = u.join_paths(subpath, model.name+'.sql')
                 query = model.compiled_query.query

squirrels/_parameter_configs.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
-from typing import Type, Optional, Union, Sequence, Iterator
+from typing import Type, Optional, Union, Sequence, Iterator, Any
 from dataclasses import dataclass, field
 from abc import ABCMeta, abstractmethod
 from copy import copy
@@ -32,10 +32,10 @@ class ParameterConfigBase(metaclass=ABCMeta):
         self.user_attribute = user_attribute
         self.parent_name = parent_name
 
-    def _get_user_group(self, user: Optional[User]) -> Optional[str]:
+    def _get_user_group(self, user: Optional[User]) -> Any:
         if self.user_attribute is not None:
             if user is None:
-                raise u.ConfigurationError(f"Public datasets with non-authenticated users cannot use parameter named " +
+                raise u.ConfigurationError(f"Public datasets (which allows non-authenticated users) cannot use parameter " +
                                            f"'{self.name}' because 'user_attribute' is defined on this parameter.")
             return getattr(user, self.user_attribute)
         
@@ -130,7 +130,7 @@ class SelectionParameterConfig(ParameterConfig):
     def _get_default_ids_iterator(self, options: Sequence[po.SelectParameterOption]) -> Iterator[str]:
         return (x._identifier for x in options if x._is_default)
     
-    def copy(self) -> MultiSelectParameterConfig:
+    def copy(self) -> SelectionParameterConfig:
         """
         Use for unit testing only
         """

squirrels/_parameter_sets.py

@@ -164,7 +164,6 @@ class ParameterConfigsSetIO:
     
     @classmethod
     def _AddFromDict(cls, param_config: ParametersConfig) -> None:
-        param_config.arguments["name"] = param_config.name
         ptype = getattr(p, param_config.type)
         factory = getattr(ptype, param_config.factory)
         factory(**param_config.arguments)
@@ -174,8 +173,7 @@ class ParameterConfigsSetIO:
         start = time.time()
         cls.obj = _ParameterConfigsSet()
 
-        parameters_from_manifest = ManifestIO.obj.parameters.values()
-        for param_as_dict in parameters_from_manifest:
+        for param_as_dict in ManifestIO.obj.parameters:
             cls._AddFromDict(param_as_dict)
         
         conn_args = ConnectionSetIO.args

squirrels/_py_module.py

@@ -1,6 +1,6 @@
 from typing import Type, Optional, Any
 from types import ModuleType
-from importlib.machinery import SourceFileLoader
+import importlib.util
 
 from . import _constants as c, _utils as u
 
@@ -16,7 +16,9 @@ class PyModule:
         """
         self.filepath = str(filepath)
         try:
-            self.module: Optional[ModuleType] = SourceFileLoader(self.filepath, self.filepath).load_module()
+            spec = importlib.util.spec_from_file_location(self.filepath, self.filepath)
+            self.module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(self.module)
         except FileNotFoundError as e:
             if is_required:
                 raise u.ConfigurationError(f"Required file not found: '{self.filepath}'") from e

squirrels/_utils.py

@@ -3,6 +3,8 @@ from pathlib import Path
 from pandas.api import types as pd_types
 import json, jinja2 as j2, pandas as pd
 
+from . import _constants as c
+
 FilePath = Union[str, Path]
 
 
@@ -179,3 +181,8 @@ def process_if_not_none(input_val: Optional[X], processor: Callable[[X], Y]) ->
     if input_val is None:
         return None
     return processor(input_val)
+
+
+def use_duckdb():
+    from ._manifest import ManifestIO
+    return (ManifestIO.obj.settings.get(c.IN_MEMORY_DB_SETTING, c.SQLITE) == c.DUCKDB)