squirrels 0.3.3__py3-none-any.whl → 0.4.1__py3-none-any.whl

squirrels/_authenticator.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Type, Optional
 from datetime import datetime, timedelta, timezone
 from jwt.exceptions import InvalidTokenError
 import secrets, jwt
@@ -7,34 +7,36 @@ from . import _utils as u, _constants as c
 from .arguments.run_time_args import AuthArgs
 from ._py_module import PyModule
 from .user_base import User, WrongPassword
-from ._environcfg import EnvironConfigIO
+from ._environcfg import EnvironConfig
 from ._manifest import DatasetScope
-from ._connection_set import ConnectionSetIO
+from ._connection_set import ConnectionsArgs, ConnectionSet
 
 
 class Authenticator:
-    
-    @classmethod
-    def get_auth_helper(cls, default_auth_helper = None):
-        auth_module_path = u.join_paths(c.PYCONFIGS_FOLDER, c.AUTH_FILE)
-        return PyModule(auth_module_path, default_class=default_auth_helper)
 
-    def __init__(self, token_expiry_minutes: int, auth_helper = None) -> None:
+    def __init__(self, base_path: str, env_cfg: EnvironConfig, conn_args: ConnectionsArgs, conn_set: ConnectionSet, token_expiry_minutes: int, *, auth_helper = None) -> None:
+        self.env_cfg = env_cfg
+        self.conn_args = conn_args
+        self.conn_set = conn_set
         self.token_expiry_minutes = token_expiry_minutes
-        self.auth_helper = self.get_auth_helper(auth_helper)
+        self.auth_helper = self._get_auth_helper(base_path, default_auth_helper=auth_helper)
+        self.user_cls: Type[User] = self.auth_helper.get_func_or_class("User", default_attr=User)
         self.secret_key = self._get_secret_key()
         self.algorithm = "HS256"
+    
+    def _get_auth_helper(self, base_path: str, *, default_auth_helper = None):
+        auth_module_path = u.Path(base_path, c.PYCONFIGS_FOLDER, c.AUTH_FILE)
+        return PyModule(auth_module_path, default_class=default_auth_helper)
 
-    def _get_secret_key(self):
-        secret_key = EnvironConfigIO.obj.get_secret(c.JWT_SECRET_KEY, default_factory=lambda: secrets.token_hex(32))
-        return secret_key
+    def _get_secret_key(self) -> str:
+        secret_key = self.env_cfg.get_secret(c.JWT_SECRET_KEY, default_factory=lambda: secrets.token_hex(32))
+        return str(secret_key)
     
     def _get_auth_args(self, username: str, password: str):
-        conn_args, connections = ConnectionSetIO.args, ConnectionSetIO.obj.get_engines_as_dict()
-        return AuthArgs(conn_args.proj_vars, conn_args.env_vars, conn_args._get_credential, connections, username, password)
+        connections = self.conn_set.get_engines_as_dict()
+        return AuthArgs(self.conn_args.proj_vars, self.conn_args.env_vars, self.conn_args._get_credential, connections, username, password)
 
     def authenticate_user(self, username: str, password: str) -> Optional[User]:
-        user_cls: type[User] = self.auth_helper.get_func_or_class("User", default_attr=User)
         get_user = self.auth_helper.get_func_or_class(c.GET_USER_FUNC, is_required=False)
         try:
             real_user = get_user(self._get_auth_args(username, password)) if get_user is not None else None
@@ -45,13 +47,13 @@ class Authenticator:
             return real_user
         
         if not isinstance(real_user, WrongPassword):
-            fake_users = EnvironConfigIO.obj.get_users()
-            if username in fake_users and secrets.compare_digest(fake_users[username][c.USER_PWD_KEY], password):
-                fake_user = fake_users[username].copy()
+            fake_users = self.env_cfg.get_users()
+            if username in fake_users and secrets.compare_digest(fake_users[username].password, password):
+                fake_user = fake_users[username].model_dump()
                 fake_user.pop("username", "")
                 is_internal = fake_user.pop("is_internal", False)
                 try:
-                    return user_cls.Create(username, is_internal=is_internal, **fake_user)
+                    return self.user_cls.Create(username, is_internal=is_internal, **fake_user)
                 except Exception as e:
                     raise u.FileExecutionError(f'Failed to create user from User model in {c.AUTH_FILE}', e) from e
         
@@ -68,8 +70,7 @@ class Authenticator:
             try:
                 payload: dict = jwt.decode(token, self.secret_key, algorithms=[self.algorithm])
                 payload.pop("exp")
-                user_cls: User = self.auth_helper.get_func_or_class("User", default_attr=User)
-                return user_cls._FromDict(payload)
+                return self.user_cls._FromDict(payload)
             except InvalidTokenError:
                 return None
 

squirrels/_command_line.py

@@ -1,93 +1,117 @@
-from argparse import ArgumentParser
-import sys, time, asyncio
+from argparse import ArgumentParser, _SubParsersAction
+import sys, asyncio, traceback, io
+
 sys.path.append('.')
 
+from ._version import __version__
+from ._api_server import ApiServer
+from ._initializer import Initializer
+from ._package_loader import PackageLoaderIO
+from .project import SquirrelsProject
 from . import _constants as c
-from ._timer import timer
 
 
 def main():
     """
     Main entry point for the squirrels command line utilities.
     """
-    start = time.time()
-    parser = ArgumentParser(description="Command line utilities from the squirrels python package", add_help=False)
-    parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
+    def with_help(parser: ArgumentParser):
+        parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
+        return parser
+
+    parser = with_help(ArgumentParser(description="Command line utilities from the squirrels python package", add_help=False))
     parser.add_argument('-V', '--version', action='store_true', help='Show the version and exit')
-    parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose output')
+    parser.add_argument('--log-level', type=str, choices=["DEBUG", "INFO", "WARNING"], default="INFO", help='Level of logging to use')
+    parser.add_argument('--log-format', type=str, choices=["text", "json"], default="text", help='Format of the log records')
+    parser.add_argument('--log-file', type=str, default=c.LOGS_FILE, help=f'Name of log file to write to in the "logs/" folder. Default is {c.LOGS_FILE}. If name is empty, then file logging is disabled')
     subparsers = parser.add_subparsers(title='commands', dest='command')
 
-    init_parser = subparsers.add_parser(c.INIT_CMD, help='Initialize a squirrels project', add_help=False)
-    init_parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
+    def add_subparser(subparsers: _SubParsersAction, cmd: str, help_text: str):
+        subparser = with_help(subparsers.add_parser(cmd, description=help_text, help=help_text, add_help=False))
+        return subparser
+
+    init_parser = add_subparser(subparsers, c.INIT_CMD, 'Initialize a squirrels project')
     init_parser.add_argument('-o', '--overwrite', action='store_true', help="Overwrite files that already exist")
     init_parser.add_argument('--core', action='store_true', help='Include all core files')
-    init_parser.add_argument('--connections', type=str, choices=c.CONF_FORMAT_CHOICES, help=f'Configure database connections as yaml (default) or python. Ignored if "--core" is not specified')
-    init_parser.add_argument('--parameters', type=str, choices=c.CONF_FORMAT_CHOICES, help=f'Configure parameters as python (default) or yaml. Ignored if "--core" is not specified')
-    init_parser.add_argument('--dbview', type=str, choices=c.FILE_TYPE_CHOICES, help='Create database view model as sql (default) or python file. Ignored if "--core" is not specified')
-    init_parser.add_argument('--federate', type=str, choices=c.FILE_TYPE_CHOICES, help='Create federated model as sql (default) or python file. Ignored if "--core" is not specified')
+    init_parser.add_argument('--connections', type=str, choices=c.CONF_FORMAT_CHOICES, help=f'Configure database connections as yaml (default) or python')
+    init_parser.add_argument('--parameters', type=str, choices=c.CONF_FORMAT_CHOICES, help=f'Configure parameters as python (default) or yaml')
+    init_parser.add_argument('--dbview', type=str, choices=c.FILE_TYPE_CHOICES, help='Create database view model as sql (default) or python file')
+    init_parser.add_argument('--federate', type=str, choices=c.FILE_TYPE_CHOICES, help='Create federated model as sql (default) or python file')
+    init_parser.add_argument('--dashboard', action='store_true', help=f'Include a sample dashboard file')
     init_parser.add_argument('--auth', action='store_true', help=f'Include the {c.AUTH_FILE} file')
-    init_parser.add_argument('--sample-db', type=str, choices=c.DATABASE_CHOICES, help='Sample sqlite database to include')
 
-    module_parser = subparsers.add_parser(c.DEPS_CMD, help=f'Load all packages specified in {c.MANIFEST_FILE} (from git)', add_help=False)
-    module_parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
+    def with_file_format_options(parser: ArgumentParser):
+        help_text = "Create model as sql (default) or python file"
+        parser.add_argument('--format', type=str, choices=c.FILE_TYPE_CHOICES, default=c.SQL_FILE_TYPE, help=help_text)
+        return parser
+    
+    get_file_help_text = "Get a sample file for the squirrels project. If the file name already exists, it will be prefixed with a timestamp."
+    get_file_parser = add_subparser(subparsers, c.GET_FILE_CMD, get_file_help_text)
+    get_file_subparsers = get_file_parser.add_subparsers(title='file_name', dest='file_name')
+    add_subparser(get_file_subparsers, c.ENV_CONFIG_FILE, f'Get a sample {c.ENV_CONFIG_FILE} file')
+    manifest_parser = add_subparser(get_file_subparsers, c.MANIFEST_FILE, f'Get a sample {c.MANIFEST_FILE} file')
+    manifest_parser.add_argument("--no-connections", action='store_true', help=f'Exclude the connections section')
+    manifest_parser.add_argument("--parameters", action='store_true', help=f'Include the parameters section')
+    manifest_parser.add_argument("--dashboards", action='store_true', help=f'Include the dashboards section')
+    add_subparser(get_file_subparsers, c.AUTH_FILE, f'Get a sample {c.AUTH_FILE} file')
+    add_subparser(get_file_subparsers, c.CONNECTIONS_FILE, f'Get a sample {c.CONNECTIONS_FILE} file')
+    add_subparser(get_file_subparsers, c.PARAMETERS_FILE, f'Get a sample {c.PARAMETERS_FILE} file')
+    add_subparser(get_file_subparsers, c.CONTEXT_FILE, f'Get a sample {c.CONTEXT_FILE} file')
+    with_file_format_options(add_subparser(get_file_subparsers, c.DBVIEW_FILE_STEM, f'Get a sample dbview model file'))
+    with_file_format_options(add_subparser(get_file_subparsers, c.FEDERATE_FILE_STEM, f'Get a sample federate model file'))
+    add_subparser(get_file_subparsers, c.DASHBOARD_FILE_STEM, f'Get a sample dashboard file')
+    add_subparser(get_file_subparsers, c.EXPENSES_DB, f'Get the sample SQLite database on expenses')
+    add_subparser(get_file_subparsers, c.WEATHER_DB, f'Get the sample SQLite database on weather')
+    
+    add_subparser(subparsers, c.DEPS_CMD, f'Load all packages specified in {c.MANIFEST_FILE} (from git)')
 
-    compile_parser = subparsers.add_parser(c.COMPILE_CMD, help='Create rendered SQL files in the folder "./target/compile"', add_help=False)
+    compile_parser = add_subparser(subparsers, c.COMPILE_CMD, 'Create rendered SQL files in the folder "./target/compile"')
     compile_dataset_group = compile_parser.add_mutually_exclusive_group(required=True)
-    compile_test_set_group = compile_parser.add_mutually_exclusive_group(required=False)
-    compile_parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
-    
     compile_dataset_group.add_argument('-d', '--dataset', type=str, help="Select dataset to use for dataset traits. Is required, unless using --all-datasets")
     compile_dataset_group.add_argument('-D', '--all-datasets', action="store_true", help="Compile models for all datasets. Only required if --dataset is not specified")
+    compile_test_set_group = compile_parser.add_mutually_exclusive_group(required=False)
     compile_test_set_group.add_argument('-t', '--test-set', type=str, help="The selection test set to use. If not specified, default selections are used, unless using --all-test-sets")
     compile_test_set_group.add_argument('-T', '--all-test-sets', action="store_true", help="Compile models for all selection test sets")
-    
     compile_parser.add_argument('-s', '--select', type=str, help="Select single model to compile. If not specified, all models for the dataset are compiled. Ignored if using --all-datasets")
     compile_parser.add_argument('-r', '--runquery', action='store_true', help='Runs all target models, and produce the results as csv files')
 
-    run_parser = subparsers.add_parser(c.RUN_CMD, help='Run the API server', add_help=False)
-    run_parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
+    run_parser = add_subparser(subparsers, c.RUN_CMD, 'Run the API server')
     run_parser.add_argument('--no-cache', action='store_true', help='Do not cache any api results')
     run_parser.add_argument('--host', type=str, default='127.0.0.1', help="The host to run on")
     run_parser.add_argument('--port', type=int, default=4465, help="The port to run on")
 
     args, _ = parser.parse_known_args()
-    timer.verbose = args.verbose
-    timer.add_activity_time('parsing arguments', start)
+    project = SquirrelsProject(log_level=args.log_level, log_format=args.log_format, log_file=args.log_file)
     
-    from . import __version__
-    from ._api_server import ApiServer
-    from ._models import ModelsIO
-    from ._initializer import Initializer
-    from ._manifest import ManifestIO
-    from ._package_loader import PackageLoaderIO
-    from ._connection_set import ConnectionSetIO
-    from ._parameter_sets import ParameterConfigsSetIO
-    from ._seeds import SeedsIO
-
     if args.version:
         print(__version__)
     elif args.command == c.INIT_CMD:
-        Initializer(args.overwrite).init_project(args)
+        Initializer(overwrite=args.overwrite).init_project(args)
+    elif args.command == c.GET_FILE_CMD:
+        Initializer().get_file(args)
     elif args.command == c.DEPS_CMD:
-        ManifestIO.LoadFromFile()
-        PackageLoaderIO.LoadPackages(reload=True)
+        PackageLoaderIO.load_packages(project._logger, project._manifest_cfg, reload=True)
     elif args.command in [c.RUN_CMD, c.COMPILE_CMD]:
-        ManifestIO.LoadFromFile()
-        SeedsIO.LoadFiles()
-        ConnectionSetIO.LoadFromFile()
         try:
-            ParameterConfigsSetIO.LoadFromFile()
-            ModelsIO.LoadFiles()
             if args.command == c.RUN_CMD:
-                server = ApiServer(args.no_cache)
+                server = ApiServer(args.no_cache, project)
                 server.run(args)
             elif args.command == c.COMPILE_CMD:
-                task = ModelsIO.WriteOutputs(args.dataset, args.all_datasets, args.select, args.test_set, args.all_test_sets, args.runquery)
+                task = project.compile(
+                    dataset=args.dataset, do_all_datasets=args.all_datasets, selected_model=args.select, test_set=args.test_set, 
+                    do_all_test_sets=args.all_test_sets, runquery=args.runquery
+                )
                 asyncio.run(task)
         except KeyboardInterrupt:
             pass
+        except Exception as e:
+            buffer = io.StringIO()
+            traceback.print_exception(e, file=buffer)
+            err_msg = buffer.getvalue()
+            print(err_msg)
+            project._logger.error(err_msg)
         finally:
-            ConnectionSetIO.Dispose()
+            project.close()
     elif args.command is None:
         print(f'Command is missing. Enter "squirrels -h" for help.')
     else:

squirrels/_connection_set.py

@@ -1,12 +1,11 @@
 from dataclasses import dataclass
 from sqlalchemy import Engine, create_engine
-import pandas as pd
+import time, pandas as pd
 
 from . import _utils as u, _constants as c, _py_module as pm
 from .arguments.init_time_args import ConnectionsArgs
-from ._environcfg import EnvironConfigIO
-from ._manifest import ManifestIO
-from ._timer import timer, time
+from ._environcfg import EnvironConfig
+from ._manifest import ManifestConfig
 
 
 @dataclass
@@ -37,7 +36,7 @@ class ConnectionSet:
         except Exception as e:
             raise RuntimeError(e) from e
 
-    def _dispose(self) -> None:
+    def dispose(self) -> None:
         """
         Disposes of all the engines in this ConnectionSet
         """
@@ -47,11 +46,20 @@ class ConnectionSet:
 
 
 class ConnectionSetIO:
-    args: ConnectionsArgs
-    obj: ConnectionSet
 
     @classmethod
-    def LoadFromFile(cls):
+    def load_conn_py_args(cls, logger: u.Logger, env_cfg: EnvironConfig, manifest_cfg: ManifestConfig) -> ConnectionsArgs:
+        start = time.time()
+        
+        proj_vars = manifest_cfg.project_variables.model_dump()
+        env_vars = env_cfg.get_all_env_vars()
+        conn_args = ConnectionsArgs(proj_vars, env_vars, env_cfg.get_credential)
+        
+        logger.log_activity_time("setting up arguments for connections.py", start)
+        return conn_args
+
+    @classmethod
+    def load_from_file(cls, logger: u.Logger, base_path: str, manifest_cfg: ManifestConfig, conn_args: ConnectionsArgs) -> ConnectionSet:
         """
         Takes the DB connection engines from both the squirrels.yml and connections.py files and merges them
         into a single ConnectionSet
@@ -61,22 +69,12 @@ class ConnectionSetIO:
         """
         start = time.time()
         engines: dict[str, Engine] = {}
-        cls.obj = ConnectionSet(engines)
-        try:
-            for config in ManifestIO.obj.connections.values():
-                engines[config.name] = create_engine(config.url)
-            
-            proj_vars = ManifestIO.obj.project_variables
-            env_vars = EnvironConfigIO.obj.get_all_env_vars()
-            get_credential = EnvironConfigIO.obj.get_credential
-            cls.args = ConnectionsArgs(proj_vars, env_vars, get_credential)
-            pm.run_pyconfig_main(c.CONNECTIONS_FILE, {"connections": engines, "sqrl": cls.args})
-        except Exception as e:
-            cls.Dispose()
-            raise e
         
-        timer.add_activity_time("creating sqlalchemy engines", start)
+        for config in manifest_cfg.connections.values():
+            engines[config.name] = create_engine(config.url)
 
-    @classmethod
-    def Dispose(cls):
-        cls.obj._dispose()
+        pm.run_pyconfig_main(base_path, c.CONNECTIONS_FILE, {"connections": engines, "sqrl": conn_args})
+        conn_set = ConnectionSet(engines)
+
+        logger.log_activity_time("creating sqlalchemy engines", start)
+        return conn_set

squirrels/_constants.py

@@ -1,75 +1,11 @@
 # Squirrels CLI commands
 INIT_CMD = 'init'
+GET_FILE_CMD = 'get-file'
 DEPS_CMD = 'deps'
 COMPILE_CMD = 'compile'
 RUN_CMD = 'run'
 
-# Manifest file keys
-PROJ_VARS_KEY = 'project_variables'
-PROJECT_NAME_KEY = 'name'
-PROJECT_LABEL_KEY = 'label'
-MAJOR_VERSION_KEY = 'major_version'
-
-PACKAGES_KEY = 'packages'
-PACKAGE_GIT_KEY = 'git'
-PACKAGE_DIRECTORY_KEY = 'directory'
-PACKAGE_REVISION_KEY = 'revision'
-
-DB_CONNECTIONS_KEY = 'connections'
-DB_CONN_NAME_KEY = 'name'
-DB_CONN_CRED_KEY = 'credential'
-DB_CONN_URL_KEY = 'url'
-
-DBVIEWS_KEY = 'dbviews'
-DBVIEW_NAME_KEY = 'name'
-DBVIEW_CONN_KEY = 'connection_name'
-DEFAULT_DB_CONN = 'default'
-
-FEDERATES_KEY = 'federates'
-FEDERATE_NAME_KEY = 'name'
-MATERIALIZED_KEY = 'materialized'
-DEFAULT_TABLE_MATERIALIZE = 'table'
-
-PARAMETERS_KEY = 'parameters'
-PARAMETER_NAME_KEY = 'name'
-PARAMETER_TYPE_KEY = 'type'
-PARAMETER_FACTORY_KEY = 'factory'
-PARAMETER_ARGS_KEY = 'arguments'
-
-TEST_SETS_KEY = 'selection_test_sets'
-TEST_SET_NAME_KEY = 'name'
-DEFAULT_TEST_SET_NAME = 'default'
-TEST_SET_DATASETS_KEY = 'datasets'
-TEST_SET_USER_ATTR_KEY = 'user_attributes'
-TEST_SET_PARAMETERS_KEY = 'parameters'
-
-DATASETS_KEY = 'datasets'
-DATASET_NAME_KEY = 'name'
-DATASET_LABEL_KEY = 'label'
-DATASET_MODEL_KEY = 'model'
-DATASET_PARAMETERS_KEY = 'parameters'
-DATASET_TRAITS_KEY = 'traits'
-DATASET_DEFAULT_TEST_SET_KEY = 'default_test_set'
-
-DATASET_SCOPE_KEY = 'scope'
-PUBLIC_SCOPE = 'public'
-PROTECTED_SCOPE = 'protected'
-PRIVATE_SCOPE = 'private'
-
-SETTINGS_KEY = 'settings'
-
 # Environment config keys
-USERS_KEY = 'users'
-USER_NAME_KEY = 'username'
-USER_PWD_KEY = 'password'
-
-ENV_VARS_KEY = 'env_vars'
-
-CREDENTIALS_KEY = 'credentials'
-USERNAME_KEY = 'username'
-PASSWORD_KEY = 'password'
-
-SECRETS_KEY = 'secrets'
 JWT_SECRET_KEY = 'jwt_secret'
 
 # Folder/File names
@@ -78,24 +14,29 @@ BASE_PROJECT_FOLDER = 'base_project'
 ASSETS_FOLDER = 'assets'
 TEMPLATES_FOLDER = 'templates'
 
-ENVIRON_CONFIG_FILE = 'environcfg.yml'
 ENV_CONFIG_FILE = 'env.yml'
 MANIFEST_JINJA_FILE = 'squirrels.yml.j2'
 CONNECTIONS_YML_FILE = 'connections.yml'
 PARAMETERS_YML_FILE = 'parameters.yml'
+DASHBOARDS_YML_FILE = 'dashboards.yml'
 MANIFEST_FILE = 'squirrels.yml'
-LU_DATA_FILE = 'lu_data.xlsx'
+
+LOGS_FOLDER = 'logs'
+LOGS_FILE = 'squirrels.log'
 
 DATABASE_FOLDER = 'assets'
 PACKAGES_FOLDER = 'sqrl_packages'
 
+MACROS_FOLDER = 'macros'
+
 MODELS_FOLDER = 'models'
 DBVIEWS_FOLDER = 'dbviews'
-DATABASE_VIEW_SQL_FILE = 'database_view1.sql'
-DATABASE_VIEW_PY_FILE = 'database_view1.py'
+DBVIEW_FILE_STEM = 'dbview_example'
 FEDERATES_FOLDER = 'federates'
-FEDERATE_SQL_NAME = 'dataset_example.sql'
-FEDERATE_PY_NAME = 'dataset_example.py'
+FEDERATE_FILE_STEM = 'federate_example'
+
+DASHBOARDS_FOLDER = 'dashboards'
+DASHBOARD_FILE_STEM = 'dashboard_example'
 
 PYCONFIGS_FOLDER = 'pyconfigs'
 AUTH_FILE = 'auth.py'
@@ -114,11 +55,18 @@ SUBCATEGORY_SEED_FILE = 'seed_subcategories.csv'
 AUTH_TOKEN_EXPIRE_SETTING = 'auth.token.expire_minutes'
 PARAMETERS_CACHE_SIZE_SETTING = 'parameters.cache.size'
 PARAMETERS_CACHE_TTL_SETTING = 'parameters.cache.ttl_minutes'
-RESULTS_CACHE_SIZE_SETTING = 'results.cache.size'
-RESULTS_CACHE_TTL_SETTING = 'results.cache.ttl_minutes'
+RESULTS_CACHE_SIZE_SETTING = 'results.cache.size' # deprecated
+RESULTS_CACHE_TTL_SETTING = 'results.cache.ttl_minutes' # deprecated
+DATASETS_CACHE_SIZE_SETTING = 'datasets.cache.size'
+DATASETS_CACHE_TTL_SETTING = 'datasets.cache.ttl_minutes'
+DASHBOARDS_CACHE_SIZE_SETTING = 'dashboards.cache.size'
+DASHBOARDS_CACHE_TTL_SETTING = 'dashboards.cache.ttl_minutes'
 TEST_SET_DEFAULT_USED_SETTING = 'selection_test_sets.default_name_used'
+DEFAULT_TEST_SET_NAME = 'default'
 DB_CONN_DEFAULT_USED_SETTING = 'connections.default_name_used'
+DEFAULT_DB_CONN = 'default'
 DEFAULT_MATERIALIZE_SETTING = 'defaults.federates.materialized'
+DEFAULT_MATERIALIZE = 'table'
 SEEDS_INFER_SCHEMA_SETTING = 'seeds.infer_schema'
 SEEDS_NA_VALUES_SETTING = 'seeds.na_values'
 IN_MEMORY_DB_SETTING = 'in_memory_database'
@@ -141,10 +89,12 @@ CONF_FORMAT_CHOICES = [YML_FORMAT, PYTHON_FORMAT]
 PYTHON_FORMAT2 = 'py (recommended)'
 CONF_FORMAT_CHOICES2 = [(PYTHON_FORMAT2, PYTHON_FORMAT), YML_FORMAT]
 
-EXPENSES_DB_NAME = 'expenses'
-WEATHER_DB_NAME = 'weather'
-NO_DB = 'none'
-DATABASE_CHOICES = [EXPENSES_DB_NAME, WEATHER_DB_NAME, NO_DB]
+EXPENSES_DB = 'expenses.db'
+WEATHER_DB = 'weather.db'
+
+# Dashboard formats
+PNG = "png"
+HTML = "html"
 
 # Function names
 GET_USER_FUNC = "get_user_if_valid"
@@ -152,4 +102,5 @@ DEP_FUNC = "dependencies"
 MAIN_FUNC = "main"
 
 # Regex
-date_regex = r'^\d{4}\-\d{2}\-\d{2}$'
+date_regex = r"^\d{4}\-\d{2}\-\d{2}$"
+color_regex = r"^#[0-9a-fA-F]{6}$"

squirrels/_dashboards_io.py

@@ -0,0 +1,61 @@
+from typing import Type, TypeVar, Callable, Coroutine, Any
+from dataclasses import dataclass
+import inspect, os, time
+
+from .arguments.run_time_args import DashboardArgs
+from ._py_module import PyModule
+from . import _constants as c, _utils as u, dashboards as d
+
+T = TypeVar('T', bound=d.Dashboard)
+
+
+@dataclass
+class DashboardFunction:
+    dashboard_name: str
+    filepath: str
+
+    @property
+    def dashboard_func(self) -> Callable[[DashboardArgs], Coroutine[Any, Any, d.Dashboard]]:
+        if not hasattr(self, '_dashboard_func'):
+            module = PyModule(self.filepath)
+            self._dashboard_func = module.get_func_or_class(c.MAIN_FUNC)
+        return self._dashboard_func
+
+    def get_dashboard_format(self) -> str:
+        return_type = inspect.signature(self.dashboard_func).return_annotation
+        assert issubclass(return_type, d.Dashboard), f"Function must return Dashboard type"
+        if return_type == d.PngDashboard:
+            return c.PNG
+        elif return_type == d.HtmlDashboard:
+            return c.HTML
+        else:
+            raise NotImplementedError(f"Dashboard format {return_type} not supported")
+    
+    async def get_dashboard(self, args: DashboardArgs, *, dashboard_type: Type[T] = d.Dashboard) -> T:
+        try:
+            dashboard = await self.dashboard_func(args)
+            assert isinstance(dashboard, dashboard_type), f"Function does not return expected Dashboard type: {dashboard_type}"
+        except (u.InvalidInputError, u.ConfigurationError, u.FileExecutionError) as e:
+            raise e
+        except Exception as e:
+            raise u.FileExecutionError(f'Failed to run "{c.MAIN_FUNC}" function for dashboard "{self.dashboard_name}"', e) from e
+
+        return dashboard
+
+
+class DashboardsIO:
+
+    @classmethod
+    def load_files(cls, logger: u.Logger, base_path: str) -> dict[str, DashboardFunction]:
+        start = time.time()
+        
+        dashboards_by_name = {}
+        for dp, _, filenames in os.walk(u.Path(base_path, c.DASHBOARDS_FOLDER)):
+            for file in filenames:
+                filepath = os.path.join(dp, file)
+                file_stem, extension = os.path.splitext(file)
+                if extension == '.py':
+                    dashboards_by_name[file_stem] = DashboardFunction(file_stem, filepath)
+        
+        logger.log_activity_time("loading files for dashboards", start)
+        return dashboards_by_name