squirrels 0.5.0b3__py3-none-any.whl → 0.6.0.post0__py3-none-any.whl

squirrels/_parameter_configs.py

@@ -11,7 +11,7 @@ import polars as pl, re
 
 from . import _data_sources as d, _parameter_options as po, _parameters as p, _utils as u, _constants as c
 from ._exceptions import InvalidInputError
-from ._auth import BaseUser
+from ._schemas.auth_models import AbstractUser
 from ._connection_set import ConnectionSet
 from ._seeds import Seeds
 
@@ -56,12 +56,20 @@ class ParameterConfigBase(metaclass=ABCMeta):
     user_attribute: str | None = field(default=None, kw_only=True)
     parent_name: str | None = field(default=None, kw_only=True)
 
-    def _get_user_group(self, user: BaseUser | None) -> Any:
-        if self.user_attribute is not None:
-            if user is None:
-                raise u.ConfigurationError(f"Public datasets (accessible without authentication) cannot use parameter " +
-                                           f"'{self.name}' because 'user_attribute' is defined on this parameter.")
-            return getattr(user, self.user_attribute)
+    def _get_user_group(self, user: AbstractUser) -> Any:
+        if self.user_attribute is None:
+            return None
+        
+        final_object = user
+        attribute = self.user_attribute
+        try:
+            if "." in attribute:
+                parts = attribute.split(".", 1)
+                final_object = getattr(final_object, parts[0])
+                attribute = parts[1]
+            return getattr(final_object, attribute)
+        except AttributeError:
+            raise u.ConfigurationError(f"User attribute '{self.user_attribute}' is not valid")
         
     def copy(self):
         """
@@ -108,16 +116,16 @@ class ParameterConfig(Generic[ParamOptionType], ParameterConfigBase):
         pass
     
     def _invalid_input_error(self, selection: str, more_details: str = '') -> InvalidInputError:
-        return InvalidInputError(200, f'Selected value "{selection}" is not valid for parameter "{self.name}". ' + more_details)
+        return InvalidInputError(400, "invalid_parameter_selection", f'Selected value "{selection}" is not valid for parameter "{self.name}". ' + more_details)
     
     @abstractmethod
     def with_selection(
-        self, selection: str | None, user: BaseUser | None, parent_param: p._SelectionParameter | None
+        self, selection: str | None, user: AbstractUser, parent_param: p._SelectionParameter | None
     ) -> p.Parameter:
         pass
     
     def _get_options_iterator(
-        self, all_options: Sequence[ParamOptionType], user: BaseUser | None, parent_param: p._SelectionParameter | None
+        self, all_options: Sequence[ParamOptionType], user: AbstractUser, parent_param: p._SelectionParameter | None
     ) -> Iterator[ParamOptionType]:
         user_group = self._get_user_group(user)
         selected_parent_option_ids = frozenset(parent_param._get_selected_ids_as_list()) if parent_param else None
@@ -153,7 +161,7 @@ class SelectionParameterConfig(ParameterConfig[po.SelectParameterOption]):
         self.children[child.name] = child
         self.trigger_refresh = True
     
-    def _get_options(self, user: BaseUser | None, parent_param: p._SelectionParameter | None) -> Sequence[po.SelectParameterOption]:
+    def _get_options(self, user: AbstractUser, parent_param: p._SelectionParameter | None) -> Sequence[po.SelectParameterOption]:
         return tuple(self._get_options_iterator(self.all_options, user, parent_param))
     
     def _get_default_ids_iterator(self, options: Sequence[po.SelectParameterOption]) -> Iterator[str]:
@@ -189,7 +197,7 @@ class SingleSelectParameterConfig(SelectionParameterConfig):
         return d.SelectDataSource(*args, **kwargs)
     
     def with_selection(
-        self, selection: str | None, user: BaseUser | None, parent_param: p._SelectionParameter | None
+        self, selection: str | None, user: AbstractUser, parent_param: p._SelectionParameter | None
     ) -> p.SingleSelectParameter:
         options = self._get_options(user, parent_param)
         if selection is None:
@@ -237,7 +245,7 @@ class MultiSelectParameterConfig(SelectionParameterConfig):
         return d.SelectDataSource(*args, **kwargs)
     
     def with_selection(
-        self, selection: str | None, user: BaseUser | None, parent_param: p._SelectionParameter | None
+        self, selection: str | None, user: AbstractUser, parent_param: p._SelectionParameter | None
     ) -> p.MultiSelectParameter:
         options = self._get_options(user, parent_param)
         if selection is None:
@@ -293,7 +301,7 @@ class DateParameterConfig(_DateTypeParameterConfig[po.DateParameterOption]):
         return d.DateDataSource(*args, **kwargs)
     
     def with_selection(
-        self, selection: str | None, user: BaseUser | None, parent_param: p._SelectionParameter | None
+        self, selection: str | None, user: AbstractUser, parent_param: p._SelectionParameter | None
     ) -> p.DateParameter:
         curr_option: po.DateParameterOption | None = next(self._get_options_iterator(self.all_options, user, parent_param), None)
         selected_date = curr_option._default_date if selection is None and curr_option is not None else selection
@@ -332,7 +340,7 @@ class DateRangeParameterConfig(_DateTypeParameterConfig[po.DateRangeParameterOpt
         return d.DateRangeDataSource(*args, **kwargs)
     
     def with_selection(
-        self, selection: str | None, user: BaseUser | None, parent_param: p._SelectionParameter | None
+        self, selection: str | None, user: AbstractUser, parent_param: p._SelectionParameter | None
     ) -> p.DateRangeParameter:
         curr_option: po.DateRangeParameterOption | None = next(self._get_options_iterator(self.all_options, user, parent_param), None)
         if selection is None:
@@ -395,7 +403,7 @@ class NumberParameterConfig(_NumericParameterConfig[po.NumberParameterOption]):
         return d.NumberDataSource(*args, **kwargs)
     
     def with_selection(
-        self, selection: str | None, user: BaseUser | None, parent_param: p._SelectionParameter | None
+        self, selection: str | None, user: AbstractUser, parent_param: p._SelectionParameter | None
     ) -> p.NumberParameter:
         curr_option: po.NumberParameterOption | None = next(self._get_options_iterator(self.all_options, user, parent_param), None)
         selected_value = curr_option._default_value if selection is None and curr_option is not None else selection
@@ -434,7 +442,7 @@ class NumberRangeParameterConfig(_NumericParameterConfig[po.NumberRangeParameter
         return d.NumberRangeDataSource(*args, **kwargs)
     
     def with_selection(
-        self, selection: str | None, user: BaseUser | None, parent_param: p._SelectionParameter | None
+        self, selection: str | None, user: AbstractUser, parent_param: p._SelectionParameter | None
     ) -> p.NumberRangeParameter:
         curr_option: po.NumberRangeParameterOption | None = next(self._get_options_iterator(self.all_options, user, parent_param), None)
         if selection is None:
@@ -524,7 +532,7 @@ class TextParameterConfig(ParameterConfig[po.TextParameterOption]):
         return d.TextDataSource(*args, **kwargs)
     
     def with_selection(
-        self, selection: str | None, user: BaseUser | None, parent_param: p._SelectionParameter | None
+        self, selection: str | None, user: AbstractUser, parent_param: p._SelectionParameter | None
     ) -> p.TextParameter:
         curr_option: po.TextParameterOption | None = next(self._get_options_iterator(self.all_options, user, parent_param), None)
         entered_text = curr_option._default_text if selection is None and curr_option is not None else selection
@@ -550,6 +558,8 @@ class DataSourceParameterConfig(Generic[ParamConfigType], ParameterConfigBase):
         super().__init__(name, label, description=description, user_attribute=user_attribute, parent_name=parent_name)
         self.parameter_type = parameter_type
         if isinstance(data_source, dict):
+            if "source" in data_source:
+                data_source["source"] = d.SourceEnum(data_source["source"])
             data_source = parameter_type.DataSource(**data_source)
         self.data_source = data_source
         self.extra_args = extra_args
@@ -557,15 +567,26 @@ class DataSourceParameterConfig(Generic[ParamConfigType], ParameterConfigBase):
     def convert(self, df: pl.DataFrame) -> ParamConfigType:
         return self.data_source._convert(self, df)
     
-    def get_dataframe(self, default_conn_name: str, conn_set: ConnectionSet, seeds: Seeds) -> pl.DataFrame:
+    def get_dataframe(self, default_conn_name: str, conn_set: ConnectionSet, seeds: Seeds, datalake_db_path: str = "") -> pl.DataFrame:
         datasource = self.data_source
         query = datasource._get_query()
-        if datasource._is_from_seeds:
+        if datasource._source == d.SourceEnum.SEEDS:
             df = seeds.run_query(query)
-        else:
+        elif datasource._source == d.SourceEnum.VDL:
+            vdl_conn = u.create_duckdb_connection(datalake_db_path)
+            try:
+                # Query the VDL database
+                df = vdl_conn.sql(query).pl()
+            except Exception as e:
+                raise u.ConfigurationError(f'Error executing query for datasource parameter "{self.name}" from VDL') from e
+            finally:
+                vdl_conn.close()
+        else:  # source == "connection"
+            conn_name = None
             try:
                 conn_name = datasource._get_connection_name(default_conn_name)
                 df = conn_set.run_sql_query_from_conn_name(query, conn_name)
             except RuntimeError as e:
-                raise u.ConfigurationError(f'Error executing query for datasource parameter "{self.name}"') from e
+                ending = f' "{conn_name}"' if conn_name is not None else ""
+                raise u.ConfigurationError(f'Error executing query for datasource parameter "{self.name}" from connection{ending}') from e
         return df

squirrels/_parameter_options.py

@@ -35,7 +35,7 @@ class ParameterOption(metaclass=ABCMeta):
         
         Arguments:
             user_group: The value of the user's "user group attribute". Only None when "user_attribute" is not specified
-                for the Parameter factory. Note that when user is None but "user_attribute" is specified, an error is thrown
+                for the Parameter factory.
             selected_parent_option_ids: List of selected option ids from the parent parameter. Only None when the Parameter
                 object has no parent parameter.
         

squirrels/_parameter_sets.py

@@ -1,15 +1,17 @@
 from __future__ import annotations
-from typing import Optional, Sequence, Any
+from typing import Optional, Sequence, Callable, Any
 from dataclasses import dataclass, field
 from collections import OrderedDict
 import time, concurrent.futures, polars as pl
 
-from . import _parameters as p, _utils as u, _constants as c, _parameter_configs as pc, _py_module as pm, _api_response_models as arm
-from ._arguments._init_time_args import ParametersArgs
+from . import _parameters as p, _utils as u, _constants as c, _parameter_configs as pc, _py_module as pm
+from ._schemas import response_models as rm
+from ._arguments.init_time_args import ParametersArgs
 from ._manifest import ParametersConfig, ManifestConfig
-from ._connection_set import ConnectionSet, ConnectionsArgs
+from ._connection_set import ConnectionSet
 from ._seeds import Seeds
-from ._auth import BaseUser
+from ._schemas.auth_models import AbstractUser
+from ._env_vars import SquirrelsEnvVars
 
 
 @dataclass
@@ -22,11 +24,11 @@ class ParameterSet:
     def get_parameters_as_dict(self) -> dict[str, p.Parameter]:
         return self._parameters_dict.copy()
 
-    def to_api_response_model0(self) -> arm.ParametersModel:
+    def to_api_response_model0(self) -> rm.ParametersModel:
         parameters = []
         for x in self._parameters_dict.values():
             parameters.append(x._to_api_response_model0())
-        return arm.ParametersModel(parameters=parameters)
+        return rm.ParametersModel(parameters=parameters)
 
 
 @dataclass
@@ -103,13 +105,10 @@ class ParameterConfigsSet:
         self.__validate_param_relationships()
     
     def apply_selections(
-        self, dataset_params: Optional[Sequence[str]], selections: dict[str, Any], user: BaseUser | None, *, parent_param: str | None = None
+        self, dataset_params: Optional[Sequence[str]], selections: dict[str, Any], user: AbstractUser, *, parent_param: str | None = None
     ) -> ParameterSet:
         if dataset_params is None:
-            if user is None:
-                dataset_params = [k for k, v in self._data.items() if v.user_attribute is None]
-            else:
-                dataset_params = list(self._data.keys())
+            dataset_params = list(self._data.keys())
         
         parameters_by_name: dict[str, p.Parameter] = {}
         params_to_process = [parent_param] if parent_param else dataset_params
@@ -153,44 +152,56 @@ class ParameterConfigsSetIO:
     """
     Static class for the singleton object of ParameterConfigsSet
     """
-    obj: ParameterConfigsSet # this is static (set in load_from_file) to simplify development experience for pyconfigs/parameters.py
+    param_factories: list[Callable[[ParametersArgs], pc.ParameterConfigBase]] = []  # this is static (set in load_from_file) to stage the functions from pyconfigs/parameters.py before using them
     
     @classmethod
-    def _get_df_dict_from_data_sources(cls, default_conn_name: str, seeds: Seeds, conn_set: ConnectionSet) -> dict[str, pl.DataFrame]:
+    def _get_df_dict_from_data_sources(
+        cls, param_configs_set: ParameterConfigsSet, default_conn_name: str, seeds: Seeds, conn_set: ConnectionSet, datalake_db_path: str
+    ) -> dict[str, pl.DataFrame]:
+        
         def get_dataframe(ds_param_config: pc.DataSourceParameterConfig) -> tuple[str, pl.DataFrame]:
-            return ds_param_config.name, ds_param_config.get_dataframe(default_conn_name, conn_set, seeds)
+            return ds_param_config.name, ds_param_config.get_dataframe(default_conn_name, conn_set, seeds, datalake_db_path)
         
-        ds_param_configs = cls.obj._get_all_ds_param_configs()
+        ds_param_configs = param_configs_set._get_all_ds_param_configs()
         with concurrent.futures.ThreadPoolExecutor() as executor:
             df_dict = dict(executor.map(get_dataframe, ds_param_configs))
         
         return df_dict
     
     @classmethod
-    def _add_from_dict(cls, param_config: ParametersConfig) -> None:
+    def _add_from_dict(cls, param_configs_set: ParameterConfigsSet, param_config: ParametersConfig) -> None:
         ptype = getattr(p, param_config.type)
         factory = getattr(ptype, param_config.factory)
-        factory(**param_config.arguments)
-    
-    @classmethod
-    def get_param_args(cls, conn_args: ConnectionsArgs) -> ParametersArgs:
-        return ParametersArgs(conn_args.project_path, conn_args.proj_vars, conn_args.env_vars)
+        obj = factory(**param_config.arguments)
+        param_configs_set.add(obj)
     
     @classmethod
     def load_from_file(
-        cls, logger: u.Logger, base_path: str, manifest_cfg: ManifestConfig, seeds: Seeds, conn_set: ConnectionSet, param_args: ParametersArgs
+        cls, logger: u.Logger, env_vars: SquirrelsEnvVars, manifest_cfg: ManifestConfig, seeds: Seeds, conn_set: ConnectionSet, 
+        param_args: ParametersArgs
     ) -> ParameterConfigsSet:
         start = time.time()
-        cls.obj = ParameterConfigsSet()
+        param_configs_set = ParameterConfigsSet()
 
         for param_as_dict in manifest_cfg.parameters:
-            cls._add_from_dict(param_as_dict)
+            cls._add_from_dict(param_configs_set, param_as_dict)
+        
+        # adds to cls.param_factories as side effect
+        main_result = pm.run_pyconfig_main(env_vars.project_path, c.PARAMETERS_FILE, {"sqrl": param_args})  
+        param_factories = cls.param_factories
+        cls.param_factories = []
+
+        for param_factory in param_factories:
+            param_configs_set.add(param_factory(param_args))
         
-        pm.run_pyconfig_main(base_path, c.PARAMETERS_FILE, {"sqrl": param_args})
+        if isinstance(main_result, list):
+            for param_config in main_result:
+                param_configs_set.add(param_config)
         
-        default_conn_name = manifest_cfg.env_vars.get(c.SQRL_CONNECTIONS_DEFAULT_NAME_USED, "default")
-        df_dict = cls._get_df_dict_from_data_sources(default_conn_name, seeds, conn_set)
-        cls.obj._post_process_params(df_dict)
+        default_conn_name = env_vars.connections_default_name_used
+        datalake_db_path = env_vars.vdl_catalog_db_path
+        df_dict = cls._get_df_dict_from_data_sources(param_configs_set, default_conn_name, seeds, conn_set, datalake_db_path)
+        param_configs_set._post_process_params(df_dict)
         
         logger.log_activity_time("loading parameters", start)
-        return cls.obj
+        return param_configs_set