squirrels 0.1.1.post1__py3-none-any.whl → 0.2.0.dev0__py3-none-any.whl

squirrels/_parameter_configs.py

@@ -0,0 +1,401 @@
+from __future__ import annotations
+from typing import Type, Optional, Union, Sequence, Iterator
+from dataclasses import dataclass, field
+from abc import ABCMeta, abstractmethod
+from copy import copy
+import pandas as pd
+
+from . import parameter_options as po, parameters as p, data_sources as d, _utils as u
+from ._authenticator import User
+
+
+@dataclass
+class ParameterConfigBase(metaclass=ABCMeta):
+    """
+    Abstract class for all parameter classes
+    """
+    name: str
+    label: str
+    is_hidden: bool # = field(default=False, kw_only=True)
+    user_attribute: Optional[str] # = field(default=None, kw_only=True)
+    parent_name: Optional[str] # = field(default=None, kw_only=True)
+
+    @abstractmethod
+    def __init__(
+        self, widget_type: str, name: str, label: str, *, is_hidden: bool = False, user_attribute: Optional[str] = None, 
+        parent_name: Optional[str] = None, **kwargs
+    ) -> None:
+        self.widget_type = widget_type
+        self.name = name
+        self.label = label
+        self.is_hidden = is_hidden
+        self.user_attribute = user_attribute
+        self.parent_name = parent_name
+
+    def _get_user_group(self, user: Optional[User]) -> Optional[str]:
+        if self.user_attribute is not None:
+            if user is None:
+                raise u.ConfigurationError(f"Public datasets with non-authenticated users cannot use parameter named " +
+                                           f"'{self.name}' because 'user_attribute' is defined on this parameter.")
+            return getattr(user, self.user_attribute)
+        
+    def copy(self):
+        """
+        Use for unit testing only
+        """
+        return copy(self)
+
+    def to_json_dict0(self) -> dict:
+        return {
+            'widget_type': self.widget_type,
+            'name': self.name,
+            'label': self.label
+        }
+
+
+@dataclass
+class ParameterConfig(ParameterConfigBase):
+    """
+    Abstract class for all parameter classes (except DataSourceParameters)
+    """
+    all_options: Sequence[po.ParameterOption] = field(repr=False)
+
+    @abstractmethod
+    def __init__(
+        self, widget_type: str, name: str, label: str, all_options: Sequence[Union[po.ParameterOption, dict]], *, is_hidden: bool = False, 
+        user_attribute: Optional[str] = None, parent_name: Optional[str] = None, **kwargs
+    ) -> None:
+        super().__init__(widget_type, name, label, is_hidden=is_hidden, user_attribute=user_attribute, 
+                         parent_name=parent_name)
+        self.all_options = tuple(self.__to_param_option(x) for x in all_options)
+
+    def __to_param_option(self, option: Union[po.ParameterOption, dict]) -> po.ParameterOption:
+        return self.__class__.ParameterOption(**option) if isinstance(option, dict) else option
+
+    @staticmethod
+    @abstractmethod
+    def ParameterOption(*args, **kwargs) -> po.ParameterOption:
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def DataSource(*args, **kwargs) -> d.DataSource:
+        pass
+    
+    def _raise_invalid_input_error(self, selection: str, more_details: str = '', e: Exception = None) -> None:
+        raise u.InvalidInputError(f'Selected value "{selection}" is not valid for parameter "{self.name}". ' + more_details) from e
+    
+    @abstractmethod
+    def with_selection(
+        self, selection: Optional[str], user: Optional[User], parent_param: Optional[p._SelectionParameter], 
+        *, request_version: Optional[int] = None
+    ) -> p.Parameter:
+        pass
+    
+    def _get_options_iterator(self, user: Optional[User], parent_param: Optional[p._SelectionParameter]) -> Iterator[po.ParameterOption]:
+        user_group = self._get_user_group(user)
+        selected_parent_option_ids = frozenset(parent_param._get_selected_ids_as_list()) if parent_param else None
+        return (x for x in self.all_options if x._is_valid(user_group, selected_parent_option_ids))
+
+
+@dataclass
+class SelectionParameterConfig(ParameterConfig):
+    """
+    Abstract class for select parameter classes (single-select, multi-select, etc)
+    """
+    children: dict[str, ParameterConfigBase] = field(default_factory=dict, init=False, repr=False)
+    trigger_refresh: bool = field(default=False, init=False)
+
+    @abstractmethod
+    def __init__(
+        self, widget_type: str, name: str, label: str, all_options: Sequence[Union[po.SelectParameterOption, dict]], *, is_hidden: bool = False, 
+        user_attribute: Optional[str] = None, parent_name: Optional[str] = None, **kwargs
+    ) -> None:
+        super().__init__(widget_type, name, label, all_options, is_hidden=is_hidden, user_attribute=user_attribute, 
+                         parent_name=parent_name)
+        self.children: dict[str, ParameterConfigBase] = dict()
+        self.trigger_refresh = False
+
+    @staticmethod
+    def ParameterOption(*args, **kwargs):
+        return po.SelectParameterOption(*args, **kwargs)
+    
+    def _add_child_mutate(self, child: ParameterConfigBase):
+        self.children[child.name] = child
+        self.trigger_refresh = True
+    
+    def _get_options(self, user: Optional[User], parent_param: Optional[p._SelectionParameter]) -> Sequence[po.SelectParameterOption]:
+        return tuple(self._get_options_iterator(user, parent_param))
+    
+    def _get_default_ids_iterator(self, options: Sequence[po.SelectParameterOption]) -> Iterator[str]:
+        return (x._identifier for x in options if x._is_default)
+    
+    def copy(self) -> MultiSelectParameterConfig:
+        """
+        Use for unit testing only
+        """
+        other = super().copy()
+        other.children = self.children.copy()
+        return other
+
+    def to_json_dict0(self) -> dict:
+        output = super().to_json_dict0()
+        output['trigger_refresh'] = self.trigger_refresh
+        return output
+
+
+@dataclass
+class SingleSelectParameterConfig(SelectionParameterConfig):
+    """
+    Class to define configurations for single-select parameter widgets.
+    """
+    
+    def __init__(
+        self, name: str, label: str, all_options: Sequence[Union[po.SelectParameterOption, dict]], *, is_hidden: bool = False, 
+        user_attribute: Optional[str] = None, parent_name: Optional[str] = None, **kwargs
+    ) -> None:
+        super().__init__("single_select", name, label, all_options, is_hidden=is_hidden, user_attribute=user_attribute, 
+                         parent_name=parent_name)
+     
+    @staticmethod
+    def DataSource(*args, **kwargs):
+        return d.SingleSelectDataSource(*args, **kwargs)
+    
+    def with_selection(
+        self, selection: Optional[str], user: Optional[User], parent_param: Optional[p._SelectionParameter],
+        *, request_version: Optional[int] = None
+    ) -> p.SingleSelectParameter:
+        options = self._get_options(user, parent_param)
+        if selection is None:
+            selected_id = next(self._get_default_ids_iterator(options), None)
+            if selected_id is None and len(options) > 0:
+                selected_id = options[0]._identifier
+        else:
+            selected_id = selection
+        return p.SingleSelectParameter(self, options, selected_id)
+
+
+@dataclass
+class MultiSelectParameterConfig(SelectionParameterConfig):
+    """
+    Class to define configurations for multi-select parameter widgets.
+    """
+    include_all: bool # = field(default=True, kw_only=True)
+    order_matters: bool # = field(default=False, kw_only=True)
+
+    def __init__(
+        self, name: str, label: str, all_options: Sequence[Union[po.SelectParameterOption, dict]], *, include_all: bool = True, 
+        order_matters: bool = False, is_hidden: bool = False, user_attribute: Optional[str] = None, parent_name: Optional[str] = None, 
+        **kwargs
+    ) -> None:
+        super().__init__("multi_select", name, label, all_options, is_hidden=is_hidden, user_attribute=user_attribute, 
+                         parent_name=parent_name)
+        self.include_all = include_all
+        self.order_matters = order_matters
+    
+    @staticmethod
+    def DataSource(*args, **kwargs):
+        return d.MultiSelectDataSource(*args, **kwargs)
+    
+    def with_selection(
+        self, selection: Optional[str], user: Optional[User], parent_param: Optional[p._SelectionParameter],
+        *, request_version: Optional[int] = None
+    ) -> p.MultiSelectParameter:
+        options = self._get_options(user, parent_param)
+        if selection is None:
+            selected_ids = tuple(self._get_default_ids_iterator(options))
+        else:
+            selected_ids = u.load_json_or_comma_delimited_str_as_list(selection)
+        return p.MultiSelectParameter(self, options, selected_ids)
+
+    def to_json_dict0(self) -> dict:
+        output = super().to_json_dict0()
+        output['include_all'] = self.include_all
+        output['order_matters'] = self.order_matters
+        return output
+
+
+@dataclass
+class _DateTypeParameterConfig(ParameterConfig):
+    """
+    Abstract class for date and date range parameter configs
+    """
+
+    @abstractmethod
+    def __init__(
+        self, widget_type: str, name: str, label: str, all_options: Sequence[Union[po.ParameterOption, dict]], *, is_hidden: bool = False, 
+        user_attribute: Optional[str] = None, parent_name: Optional[str] = None, **kwargs
+    ) -> None:
+        super().__init__(widget_type, name, label, all_options, is_hidden=is_hidden, user_attribute=user_attribute, 
+                         parent_name=parent_name)
+
+
+@dataclass
+class DateParameterConfig(_DateTypeParameterConfig):
+    """
+    Class to define configurations for single-select parameter widgets.
+    """
+    
+    def __init__(
+        self, name: str, label: str, all_options: Sequence[Union[po.DateParameterOption, dict]], *, is_hidden: bool = False, 
+        user_attribute: Optional[str] = None, parent_name: Optional[str] = None, **kwargs
+    ) -> None:
+        super().__init__("date", name, label, all_options, is_hidden=is_hidden, user_attribute=user_attribute, 
+                         parent_name=parent_name)
+
+    @staticmethod
+    def ParameterOption(*args, **kwargs):
+        return po.DateParameterOption(*args, **kwargs)
+    
+    @staticmethod
+    def DataSource(*args, **kwargs):
+        return d.DateDataSource(*args, **kwargs)
+    
+    def with_selection(
+        self, selection: Optional[str], user: Optional[User], parent_param: Optional[p._SelectionParameter],
+        *, request_version: Optional[int] = None
+    ) -> p.DateParameter:
+        curr_option: po.DateParameterOption = next(self._get_options_iterator(user, parent_param))
+        selected_date = curr_option._default_date if selection is None else selection
+        return p.DateParameter(self, curr_option, selected_date)
+
+
+@dataclass
+class DateRangeParameterConfig(_DateTypeParameterConfig):
+    """
+    Class to define configurations for single-select parameter widgets.
+    """
+    
+    def __init__(
+        self, name: str, label: str, all_options: Sequence[Union[po.DateRangeParameterOption, dict]], *, is_hidden: bool = False, 
+        user_attribute: Optional[str] = None, parent_name: Optional[str] = None, **kwargs
+    ) -> None:
+        super().__init__("date_range", name, label, all_options, is_hidden=is_hidden, user_attribute=user_attribute, 
+                         parent_name=parent_name)
+
+    @staticmethod
+    def ParameterOption(*args, **kwargs):
+        return po.DateRangeParameterOption(*args, **kwargs)
+    
+    @staticmethod
+    def DataSource(*args, **kwargs):
+        return d.DateRangeDataSource(*args, **kwargs)
+    
+    def with_selection(
+        self, selection: Optional[str], user: Optional[User], parent_param: Optional[p._SelectionParameter],
+        *, request_version: Optional[int] = None
+    ) -> p.DateParameter:
+        curr_option: po.DateRangeParameterOption = next(self._get_options_iterator(user, parent_param))
+        if selection is None:
+            selected_start_date = curr_option._default_start_date
+            selected_end_date = curr_option._default_end_date
+        else:
+            try:
+                selected_start_date, selected_end_date = u.load_json_or_comma_delimited_str_as_list(selection)
+            except ValueError as e:
+                self._raise_invalid_input_error(selection, "Date range parameter selection must be two dates joined by comma.", e)
+        return p.DateRangeParameter(self, curr_option, selected_start_date, selected_end_date)
+
+
+@dataclass
+class _NumericParameterConfig(ParameterConfig):
+    """
+    Abstract class for number and number range parameter configs
+    """
+
+    @abstractmethod
+    def __init__(
+        self, widget_type: str, name: str, label: str, all_options: Sequence[Union[po.ParameterOption, dict]], *, is_hidden: bool = False, 
+        user_attribute: Optional[str] = None, parent_name: Optional[str] = None, **kwargs
+    ) -> None:
+        super().__init__(widget_type, name, label, all_options, is_hidden=is_hidden, user_attribute=user_attribute, 
+                         parent_name=parent_name)
+
+
+@dataclass
+class NumberParameterConfig(_NumericParameterConfig):
+    """
+    Class to define configurations for single-select parameter widgets.
+    """
+    
+    def __init__(
+        self, name: str, label: str, all_options: Sequence[Union[po.NumberParameterOption, dict]], *, is_hidden: bool = False, 
+        user_attribute: Optional[str] = None, parent_name: Optional[str] = None, **kwargs
+    ) -> None:
+        super().__init__("number", name, label, all_options, is_hidden=is_hidden, user_attribute=user_attribute, 
+                         parent_name=parent_name)
+
+    @staticmethod
+    def ParameterOption(*args, **kwargs):
+        return po.NumberParameterOption(*args, **kwargs)
+    
+    @staticmethod
+    def DataSource(*args, **kwargs):
+        return d.NumberDataSource(*args, **kwargs)
+    
+    def with_selection(
+        self, selection: Optional[str], user: Optional[User], parent_param: Optional[p._SelectionParameter],
+        *, request_version: Optional[int] = None
+    ) -> p.NumberParameter:
+        curr_option: po.NumberParameterOption = next(self._get_options_iterator(user, parent_param))
+        selected_value = curr_option._default_value if selection is None else selection
+        return p.NumberParameter(self, curr_option, selected_value)
+
+
+@dataclass
+class NumberRangeParameterConfig(_NumericParameterConfig):
+    """
+    Class to define configurations for single-select parameter widgets.
+    """
+    
+    def __init__(
+        self, name: str, label: str, all_options: Sequence[Union[po.NumberRangeParameterOption, dict]], *, is_hidden: bool = False, 
+        user_attribute: Optional[str] = None, parent_name: Optional[str] = None, **kwargs
+    ) -> None:
+        super().__init__("number_range", name, label, all_options, is_hidden=is_hidden, user_attribute=user_attribute, 
+                         parent_name=parent_name)
+
+    @staticmethod
+    def ParameterOption(*args, **kwargs):
+        return po.NumberRangeParameterOption(*args, **kwargs)
+    
+    @staticmethod
+    def DataSource(*args, **kwargs):
+        return d.NumberRangeDataSource(*args, **kwargs)
+    
+    def with_selection(
+        self, selection: Optional[str], user: Optional[User], parent_param: Optional[p._SelectionParameter],
+        *, request_version: Optional[int] = None
+    ) -> p.NumberRangeParameter:
+        curr_option: po.NumberRangeParameterOption = next(self._get_options_iterator(user, parent_param))
+        if selection is None:
+            selected_lower_value = curr_option._default_lower_value
+            selected_upper_value = curr_option._default_upper_value
+        else:
+            try:
+                selected_lower_value, selected_upper_value = u.load_json_or_comma_delimited_str_as_list(selection)
+            except ValueError as e:
+                self._raise_invalid_input_error(selection, "Number range parameter selection must be two numbers joined by comma.", e)
+        return p.NumberRangeParameter(self, curr_option, selected_lower_value, selected_upper_value)
+
+
+@dataclass
+class DataSourceParameterConfig(ParameterConfigBase):
+    """
+    Class to define configurations for parameter widgets whose options come from lookup tables
+    """
+    parameter_type: Type[ParameterConfig]
+    data_source: d.DataSource
+
+    def __init__(
+        self, parameter_type: Type[ParameterConfig], name: str, label: str, data_source: Union[d.DataSource, dict], *, 
+        is_hidden: bool = False, user_attribute: Optional[str] = None, parent_name: Optional[str] = None, **kwargs
+    ) -> None:
+        super().__init__("data_source", name, label, is_hidden=is_hidden, user_attribute=user_attribute, parent_name=parent_name)
+        self.parameter_type = parameter_type
+        if isinstance(data_source, dict):
+            data_source = parameter_type.DataSource(**data_source)
+        self.data_source = data_source
+
+    def convert(self, df: pd.DataFrame) -> ParameterConfig:
+        return self.data_source._convert(self, df)

squirrels/_parameter_sets.py

@@ -0,0 +1,188 @@
+from __future__ import annotations
+from typing import Optional, Sequence
+from dataclasses import dataclass, field
+from collections import OrderedDict
+import concurrent.futures, pandas as pd
+
+from . import _utils as u, _constants as c, parameters as p, _parameter_configs as pc, _py_module as pm
+from .arguments.init_time_args import ParametersArgs
+from ._manifest import ManifestIO, ParametersConfig
+from ._connection_set import ConnectionSetIO
+from ._authenticator import User
+from ._timer import timer, time
+
+
+@dataclass
+class ParameterSet:
+    """
+    A wrapper class for a sequence of parameters with the selections applied as well
+    """
+    _parameters_dict: OrderedDict[str, p.Parameter]
+
+    def get_parameters_as_dict(self) -> dict[str, p.Parameter]:
+        return self._parameters_dict.copy()
+
+    def to_json_dict0(self, *, debug: bool = False) -> dict:
+        parameters = []
+        for x in self._parameters_dict.values():
+            if not x._config.is_hidden or debug:
+                parameters.append(x.to_json_dict0())
+        return {"parameters": parameters}
+
+
+@dataclass
+class _ParameterConfigsSet:
+    """
+    Pool of parameter configs, can create multiple for unit testing purposes
+    """
+    _data: dict[str, pc.ParameterConfig] = field(default_factory=OrderedDict)
+    _data_source_params: dict[str, pc.DataSourceParameterConfig] = field(default_factory=dict)
+        
+    def get(self, name: Optional[str]) -> Optional[pc.ParameterConfig]:
+        try:
+            return self._data[name] if name is not None else None
+        except KeyError as e:
+            raise u.ConfigurationError(f'Unable to find parameter named "{name}"') from e
+
+    def add(self, param_config: pc.ParameterConfigBase) -> None:
+        self._data[param_config.name] = param_config
+        if isinstance(param_config, pc.DataSourceParameterConfig):
+            self._data_source_params[param_config.name] = param_config
+    
+    def _get_all_ds_param_configs(self) -> Sequence[pc.DataSourceParameterConfig]:
+        return list(self._data_source_params.values())
+
+    def __convert_datasource_params(self, df_dict: dict[str, pd.DataFrame]) -> None:
+        done = set()
+        for curr_name in self._data_source_params:
+            stack = [curr_name] # Note: parents must be converted first before children
+            while stack:
+                name = stack[-1]
+                if name not in done:
+                    param = self._data_source_params.get(name, self.get(name))
+                    parent_name = param.parent_name
+                    if parent_name is not None and parent_name not in done:
+                        stack.append(parent_name)
+                        continue
+                    if isinstance(param, pc.DataSourceParameterConfig):
+                        if name not in df_dict:
+                            raise u.ConfigurationError(f'No reference data found for parameter "{name}"')
+                        self._data[name] = param.convert(df_dict[name])
+                    done.add(name)
+                stack.pop()
+    
+    def __validate_param_relationships(self) -> None:
+        for param_config in self._data.values():
+            assert isinstance(param_config, pc.ParameterConfig)
+            parent_name = param_config.parent_name
+            parent = self.get(parent_name)
+            if parent:
+                if not isinstance(param_config, pc.SelectionParameterConfig):
+                    if not isinstance(parent, pc.SingleSelectParameterConfig):
+                        raise u.ConfigurationError(f'Only single-select parameters can be parents of non-select parameters. ' +
+                                                   f'Parameter "{parent_name}" is the parent of non-select parameter ' +
+                                                   f'"{param_config.name}" but "{parent_name}" is not a single-select parameter.')
+                    seen = set()
+                    for option in param_config.all_options:
+                        lookup_keys = option._parent_option_ids
+                        if len(option._user_groups) > 0:
+                            lookup_keys = set((x, y) for x in option._parent_option_ids for y in option._user_groups)
+                        if not seen.isdisjoint(lookup_keys):
+                            raise u.ConfigurationError(f'Each distinct value of "parent option id" can only appear once (per user group)' +
+                                                       f'among the options of non-select parameter "{param_config.name}".')
+                        seen.update(lookup_keys)
+                
+                if not isinstance(parent, pc.SelectionParameterConfig):
+                    raise u.ConfigurationError(f'Only selection parameters can be parents. Parameter "{parent_name}" is the parent of ' +
+                                               f'"{param_config.name}" but "{parent_name}" is not a selection parameter.')
+                
+                parent._add_child_mutate(param_config)
+    
+    def _post_process_params(self, df_dict: dict[str, pd.DataFrame]) -> None:
+        self.__convert_datasource_params(df_dict)
+        self.__validate_param_relationships()
+    
+    def apply_selections(
+        self, dataset_params: Optional[Sequence[str]], selections: dict[str, str], user: Optional[User], 
+        *, updates_only: bool = False, request_version: Optional[int] = None
+    ) -> ParameterSet:
+        if dataset_params is None:
+            dataset_params = self._data.keys()
+        
+        parameters_by_name: dict[str, p.Parameter] = {}
+        params_to_process = selections.keys() if selections and updates_only else dataset_params
+        params_to_process_set = set(params_to_process)
+        for some_name in params_to_process:
+            stack = [some_name] # Note: process parent selections first (if applicable) before children
+            while stack:
+                curr_name = stack[-1]
+                children = []
+                if curr_name not in parameters_by_name:
+                    param_conf = self.get(curr_name)
+                    parent_name = param_conf.parent_name
+                    if parent_name is None:
+                        parent = None
+                    elif parent_name in params_to_process_set and parent_name not in parameters_by_name:
+                        stack.append(parent_name)
+                        continue
+                    else:
+                        parent = parameters_by_name.get(parent_name)
+                    param = param_conf.with_selection(selections.get(curr_name), user, parent)
+                    parameters_by_name[curr_name] = param
+                    if isinstance(param_conf, pc.SelectionParameterConfig):
+                        children = list(param_conf.children.keys())
+                stack.pop()
+                stack.extend(children)
+        
+        ordered_parameters = OrderedDict((key, parameters_by_name[key]) for key in dataset_params if key in parameters_by_name)
+        return ParameterSet(ordered_parameters)
+
+
+class ParameterConfigsSetIO:
+    """
+    Static class for the singleton object of __ParameterConfigsPoolData
+    """
+    args: ParametersArgs
+    obj: _ParameterConfigsSet
+    
+    @classmethod
+    def _GetDfDict(cls) -> dict[str, pd.DataFrame]:
+        def get_dataframe_from_query(ds_param_config: pc.DataSourceParameterConfig) -> pd.DataFrame:
+            key, datasource = ds_param_config.name, ds_param_config.data_source
+            try:
+                query, conn_name = datasource._get_query(), datasource._get_connection_name()
+                df = ConnectionSetIO.obj.run_sql_query_from_conn_name(query, conn_name)
+            except RuntimeError as e:
+                raise u.ConfigurationError(f'Error executing query for datasource parameter "{key}"') from e
+            return key, df
+        
+        ds_param_configs = cls.obj._get_all_ds_param_configs()
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            df_dict = dict(executor.map(get_dataframe_from_query, ds_param_configs))
+        
+        return df_dict
+    
+    @classmethod
+    def _AddFromDict(cls, param_config: ParametersConfig) -> None:
+        param_config.arguments["name"] = param_config.name
+        ptype = getattr(p, param_config.type)
+        factory = getattr(ptype, param_config.factory)
+        factory(**param_config.arguments)
+    
+    @classmethod
+    def LoadFromFile(cls) -> None:
+        start = time.time()
+        cls.obj = _ParameterConfigsSet()
+
+        parameters_from_manifest = ManifestIO.obj.parameters
+        for param_as_dict in parameters_from_manifest:
+            cls._AddFromDict(param_as_dict)
+        
+        conn_args = ConnectionSetIO.args
+        cls.args = ParametersArgs(conn_args.proj_vars, conn_args.env_vars)
+        pm.run_pyconfig_main(c.PARAMETERS_FILE, {"sqrl": cls.args})
+        
+        df_dict = cls._GetDfDict()
+        cls.obj._post_process_params(df_dict)
+        
+        timer.add_activity_time("loading parameters", start)

squirrels/_py_module.py

@@ -0,0 +1,60 @@
+from typing import Type, Optional, Any
+from types import ModuleType
+from importlib.machinery import SourceFileLoader
+
+from . import _constants as c, _utils as u
+
+
+class PyModule:
+    def __init__(self, filepath: u.FilePath, *, default_class: Optional[Type] = None, is_required: bool = False) -> None:
+        """
+        Constructor for PyModule, an abstract module for a file that may or may not exist
+        
+        Parameters:
+            filepath (str | pathlib.Path): The file path to the python module
+            is_required: If true, throw an error if the file path doesn't exist
+        """
+        self.filepath = str(filepath)
+        try:
+            self.module: Optional[ModuleType] = SourceFileLoader(self.filepath, self.filepath).load_module()
+        except FileNotFoundError as e:
+            if is_required:
+                raise u.ConfigurationError(f"Required file not found: '{self.filepath}'") from e
+            self.module: Optional[ModuleType] = default_class
+    
+    def get_func_or_class(self, attr_name: str, *, default_attr: Any = None, is_required: bool = True) -> Any:
+        """
+        Get an attribute of the module. Usually a python function or class.
+
+        Parameters:
+            attr_name: The attribute name
+            default_attr: The default function or class to use if the attribute cannot be found
+            is_required: If true, throw an error if the attribute cannot be found, unless default_attr is not None
+        
+        Returns:
+            The attribute of the module
+        """
+        func_or_class = default_attr
+        if self.module is not None and hasattr(self.module, attr_name):
+            func_or_class = getattr(self.module, attr_name)
+        if func_or_class is None and is_required:
+            raise u.ConfigurationError(f"Module '{self.filepath}' missing required attribute '{attr_name}'")
+        return func_or_class
+
+
+def run_pyconfig_main(filename: str, kwargs: dict[str, Any] = {}) -> None:
+    """
+    Given a python file in the 'pyconfigs' folder, run its main function
+    
+    Parameters:
+        filename: The name of the file to run main function
+        kwargs: Dictionary of the main function arguments
+    """
+    filepath = u.join_paths(c.PYCONFIG_FOLDER, filename)
+    module = PyModule(filepath)
+    main_function = module.get_func_or_class(c.MAIN_FUNC, is_required=False)
+    if main_function:
+        try:
+            main_function(**kwargs)
+        except Exception as e:
+            raise u.FileExecutionError(f'Failed to run python file "{filepath}"', e)

squirrels/_timer.py

@@ -0,0 +1,36 @@
+from datetime import datetime
+import time
+
+
+class Timer:
+    def __init__(self):
+        # self.times: dict[str, list[float]] = dict()
+        self.verbose = False
+    
+    def _get_dt_from_timestamp(self, timestamp) -> str:
+        return datetime.fromtimestamp(timestamp).strftime('%H:%M:%S.%f')
+    
+    def add_activity_time(self, activity: str, start_timestamp: float) -> None:
+        if self.verbose:
+            end_timestamp = time.time()
+            time_taken = round((end_timestamp-start_timestamp) * 10**3, 3)
+            # times_list = self.times.setdefault(activity, list())
+            # times_list.append(time_taken)
+            print(f'Time taken for "{activity}": {time_taken}ms')
+
+            start_datetime = self._get_dt_from_timestamp(start_timestamp)
+            end_datetime = self._get_dt_from_timestamp(end_timestamp)
+            print(f'--> start time: "{start_datetime}", end time: "{end_datetime}"')
+            print()
+    
+    # def report_times(self):
+    #     if self.verbose:
+    #         for activity, times_list in self.times.items():
+    #             total_time = sum(times_list)
+    #             avg_time = total_time / len(times_list)
+    #             print()
+    #             print(f'Time statistics for "{activity}":')
+    #             print(f'  Total time: {total_time}ms')
+    #             print(f'  Average time: {avg_time}ms')
+
+timer = Timer()