squirrels 0.3.3__py3-none-any.whl → 0.4.0__py3-none-any.whl

squirrels/_environcfg.py

@@ -1,81 +1,84 @@
-from typing import Any, Optional, Callable
-from dataclasses import dataclass
-import os, yaml
+from pathlib import Path
+from typing import Any, Callable
+from pydantic import BaseModel, Field, field_validator, ValidationError
+import os, yaml, time
 
 from . import _constants as c, _utils as u
-from ._timer import timer, time
 
-_GLOBAL_SQUIRRELS_CFG_FILE1 = u.join_paths(os.path.expanduser('~'), '.squirrels', c.ENVIRON_CONFIG_FILE)
-_GLOBAL_SQUIRRELS_CFG_FILE2 = u.join_paths(os.path.expanduser('~'), '.squirrels', c.ENV_CONFIG_FILE)
+_GLOBAL_SQUIRRELS_CFG_FILE = u.Path(os.path.expanduser('~'), '.squirrels', c.ENV_CONFIG_FILE)
 
 
-@dataclass
-class _EnvironConfig:
-    _users: dict[str, dict[str, Any]]
-    _env_vars: dict[str, str]
-    _credentials: dict[str, dict[str, str]]
-    _secrets: dict[str, str]
-    
-    def __post_init__(self):
-        for username, user in self._users.items():
-            user[c.USER_NAME_KEY] = username
-            if c.USER_PWD_KEY not in user:
-                raise u.ConfigurationError(f"All users must have password in environcfg.yml")
-        
-        for key, cred in self._credentials.items():
-            if c.USERNAME_KEY not in cred or c.PASSWORD_KEY not in cred:
-                raise u.ConfigurationError(f'Either "{c.USERNAME_KEY}" or "{c.PASSWORD_KEY}" was not specified for credential "{key}"')
+class _UserConfig(BaseModel, extra="allow"):
+    username: str
+    password: str
+    is_internal: bool = False
+
+
+class _CredentialsConfig(BaseModel):
+    username: str
+    password: str
+
+
+class EnvironConfig(BaseModel):
+    users: dict[str, _UserConfig] = Field(default_factory=dict)
+    env_vars: dict[str, str] = Field(default_factory=dict)
+    credentials: dict[str, _CredentialsConfig] = Field(default_factory=dict)
+    secrets: dict[str, Any | None] = Field(default_factory=dict)
+
+    @field_validator("users", mode="before")
+    @classmethod
+    def inject_username(cls, users: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:
+        processed_users = {}
+        for username, user in users.items():
+            processed_users[username] = {"username": username, **user}
+        return processed_users
     
-    def get_users(self) -> dict[str, dict[str, Any]]:
-        return self._users.copy()
+    def get_users(self) -> dict[str, _UserConfig]:
+        return self.users.copy()
     
     def get_all_env_vars(self) -> dict[str, str]:
-        return self._env_vars.copy()
+        return self.env_vars.copy()
     
-    def get_credential(self, key: Optional[str]) -> tuple[str, str]:
+    def get_credential(self, key: str | None) -> tuple[str, str]:
         if not key:
             return "", ""
 
         try:
-            credential = self._credentials[key]
+            credential = self.credentials[key]
         except KeyError as e:
             raise u.ConfigurationError(f'No credentials configured for "{key}"') from e
     
-        return credential[c.USERNAME_KEY], credential[c.PASSWORD_KEY]
+        return credential.username, credential.password
     
-    def get_secret(self, key: str, *, default_factory: Optional[Callable[[],str]] = None) -> str:
-        if not self._secrets.get(key) and default_factory is not None:
-            self._secrets[key] = default_factory()
-        return self._secrets.get(key)
+    def get_secret(self, key: str, default_factory: Callable[[], Any]) -> Any:
+        if self.secrets.get(key) is None:
+            self.secrets[key] = default_factory()
+        return self.secrets[key]
 
 
 class EnvironConfigIO:
-    obj: _EnvironConfig
     
     @classmethod
-    def LoadFromFile(cls):
+    def load_from_file(cls, logger: u.Logger, base_path: str) -> EnvironConfig:
         start = time.time()
-        def load_yaml(filename: str) -> dict[str, dict]:
+        def load_yaml(filename: str | Path) -> dict[str, dict]:
             try:
                 with open(filename, 'r') as f:
                     return yaml.safe_load(f)
             except FileNotFoundError:
                 return {}
         
-        master_env_config1 = load_yaml(_GLOBAL_SQUIRRELS_CFG_FILE1)
-        master_env_config2 = load_yaml(_GLOBAL_SQUIRRELS_CFG_FILE2)
-        proj_env_config1 = load_yaml(c.ENVIRON_CONFIG_FILE)
-        proj_env_config2 = load_yaml(c.ENV_CONFIG_FILE)
+        master_env_config = load_yaml(_GLOBAL_SQUIRRELS_CFG_FILE)
+        proj_env_config = load_yaml(Path(base_path, c.ENV_CONFIG_FILE))
 
-        for project_config in [master_env_config2, proj_env_config1, proj_env_config2]:
-            for key in project_config:
-                master_env_config1.setdefault(key, {})
-                master_env_config1[key].update(project_config[key])
+        for key in proj_env_config:
+            master_env_config.setdefault(key, {})
+            master_env_config[key].update(proj_env_config[key])
         
-        users = master_env_config1.get(c.USERS_KEY, {})
-        env_vars = master_env_config1.get(c.ENV_VARS_KEY, {})
-        credentials = master_env_config1.get(c.CREDENTIALS_KEY, {})
-        secrets = master_env_config1.get(c.SECRETS_KEY, {})
-
-        cls.obj = _EnvironConfig(users, env_vars, credentials, secrets)
-        timer.add_activity_time(f"loading {c.ENV_CONFIG_FILE} file", start)
+        try:
+            env_cfg = EnvironConfig(**master_env_config)
+        except ValidationError as e:
+            raise u.ConfigurationError(f"Failed to process {c.ENV_CONFIG_FILE} file. " + str(e)) from e
+        
+        logger.log_activity_time(f"loading {c.ENV_CONFIG_FILE} file", start)
+        return env_cfg

squirrels/_initializer.py

@@ -1,178 +1,221 @@
 from typing import Optional
+from datetime import datetime
 import inquirer, os, shutil
 
 from . import _constants as c, _utils as u
 
-base_proj_dir = u.join_paths(os.path.dirname(__file__), c.PACKAGE_DATA_FOLDER, c.BASE_PROJECT_FOLDER)
+base_proj_dir = u.Path(os.path.dirname(__file__), c.PACKAGE_DATA_FOLDER, c.BASE_PROJECT_FOLDER)
 
 
 class Initializer:
-    def __init__(self, overwrite: bool):
+    def __init__(self, *, overwrite: bool = False):
         self.overwrite = overwrite
 
-    def _path_exists(self, filepath: str) -> bool:
-        if not self.overwrite and os.path.exists(filepath):
-            print(f'File "{filepath}" already exists. Creation skipped.')
-            return True
-        return False
+    def _path_exists(self, filepath: u.Path) -> bool:
+        return os.path.exists(filepath)
     
-    def _copy_file(self, filepath: str, *, src_folder: str = ""):
-        if not self._path_exists(filepath):
-            dest_dir = os.path.dirname(filepath)
-            if dest_dir != "":
-                os.makedirs(dest_dir, exist_ok=True)
-            src_path = u.join_paths(base_proj_dir, src_folder, filepath)
+    def _files_have_same_content(self, file1: u.Path, file2: u.Path) -> bool:
+        with open(file1, 'rb') as f1, open(file2, 'rb') as f2:
+            return f1.read() == f2.read()
+    
+    def _add_timestamp_to_filename(self, path: u.Path) -> u.Path:
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        new_filename = f"{path.stem}_{timestamp}{path.suffix}"
+        return path.with_name(new_filename)
+    
+    def _copy_file(self, filepath: u.Path, *, src_folder: str = ""):
+        src_path = u.Path(base_proj_dir, src_folder, filepath)
+        
+        dest_dir = os.path.dirname(filepath)
+        if dest_dir != "":
+            os.makedirs(dest_dir, exist_ok=True)
+        
+        perform_copy = True
+        if self._path_exists(filepath):
+            old_filepath = filepath
+            if self._files_have_same_content(src_path, filepath):
+                perform_copy = False
+                extra_msg = "Skipping... file contents is same as source"
+            elif self.overwrite:
+                extra_msg = "Overwriting file..."
+            else:
+                filepath = self._add_timestamp_to_filename(old_filepath)
+                extra_msg = f'Creating file as "{filepath}" instead...'
+            print(f'File "{old_filepath}" already exists.', extra_msg)
+        else:
+            print(f'Creating file "{filepath}"...')
+        
+        if perform_copy:
             shutil.copy(src_path, filepath)
 
     def _copy_dbview_file(self, filepath: str):
-        self._copy_file(u.join_paths(c.MODELS_FOLDER, c.DBVIEWS_FOLDER, filepath))
+        self._copy_file(u.Path(c.MODELS_FOLDER, c.DBVIEWS_FOLDER, filepath))
 
     def _copy_federate_file(self, filepath: str):
-        self._copy_file(u.join_paths(c.MODELS_FOLDER, c.FEDERATES_FOLDER, filepath))
+        self._copy_file(u.Path(c.MODELS_FOLDER, c.FEDERATES_FOLDER, filepath))
 
     def _copy_database_file(self, filepath: str):
-        self._copy_file(u.join_paths(c.DATABASE_FOLDER, filepath))
+        self._copy_file(u.Path(c.DATABASE_FOLDER, filepath))
     
     def _copy_pyconfig_file(self, filepath: str):
-        self._copy_file(u.join_paths(c.PYCONFIGS_FOLDER, filepath))
+        self._copy_file(u.Path(c.PYCONFIGS_FOLDER, filepath))
     
     def _copy_seed_file(self, filepath: str):
-        self._copy_file(u.join_paths(c.SEEDS_FOLDER, filepath))
+        self._copy_file(u.Path(c.SEEDS_FOLDER, filepath))
+    
+    def _copy_dashboard_file(self, filepath: str):
+        self._copy_file(u.Path(c.DASHBOARDS_FOLDER, filepath))
 
-    def init_project(self, args):
-        options = ["core", "connections", "parameters", "dbview", "federate", "auth", "sample_db"]
-        CORE, CONNECTIONS, PARAMETERS, DBVIEW, FEDERATE, AUTH, SAMPLE_DB = options
+    def _create_manifest_file(self, has_connections: bool, has_parameters: bool, has_dashboards: bool):
         TMP_FOLDER = "tmp"
 
+        def get_content(file_name: Optional[str]) -> str:
+            if file_name is None:
+                return ""
+            
+            yaml_path = u.Path(base_proj_dir, file_name)
+            return u.read_file(yaml_path)
+        
+        file_name_dict = {
+            "parameters": c.PARAMETERS_YML_FILE if has_parameters else None, 
+            "connections": c.CONNECTIONS_YML_FILE if has_connections else None,
+            "dashboards": c.DASHBOARDS_YML_FILE if has_dashboards else None
+        }
+        substitutions = {key: get_content(val) for key, val in file_name_dict.items()}
+        
+        manifest_template = get_content(c.MANIFEST_JINJA_FILE)
+        manifest_content = u.render_string(manifest_template, **substitutions)
+        output_path = u.Path(base_proj_dir, TMP_FOLDER, c.MANIFEST_FILE)
+        with open(u.Path(output_path), "w") as f:
+            f.write(manifest_content)
+        
+        self._copy_file(u.Path(c.MANIFEST_FILE), src_folder=TMP_FOLDER)
+
+    def init_project(self, args):
+        options = ["core", "connections", "parameters", "dbview", "federate", "dashboard", "auth"]
+        _, CONNECTIONS, PARAMETERS, DBVIEW, FEDERATE, DASHBOARD, AUTH = options
+
         answers = { x: getattr(args, x) for x in options }
         if not any(answers.values()):
-            core_questions = [
-                inquirer.Confirm(CORE, 
-                                 message="Include all core project files?",
-                                 default=True)
-            ]
-            answers = inquirer.prompt(core_questions)
-            
-            if answers.get(CORE, False):
-                conditional_questions = [
-                    inquirer.List(CONNECTIONS,
-                                  message=f"How would you like to configure the database connections?" ,
-                                  choices=c.CONF_FORMAT_CHOICES),
-                    inquirer.List(PARAMETERS,
-                                  message=f"How would you like to configure the parameters?" ,
-                                  choices=c.CONF_FORMAT_CHOICES2),
-                    inquirer.List(DBVIEW, 
-                                  message="What's the file format for the database view model?",
-                                  choices=c.FILE_TYPE_CHOICES),
-                    inquirer.List(FEDERATE, 
-                                  message="What's the file format for the federated model?",
-                                  choices=c.FILE_TYPE_CHOICES),
-                ]
-                answers.update(inquirer.prompt(conditional_questions))
-
-            remaining_questions = [
-                inquirer.Confirm(AUTH,
-                                 message=f"Do you want to add the '{c.AUTH_FILE}' file to enable custom API authentication?" ,
-                                 default=False),
-                inquirer.List(SAMPLE_DB, 
-                              message="What sample sqlite database do you wish to use (if any)?",
-                              choices= c.DATABASE_CHOICES)
+            questions = [
+                inquirer.List(
+                    CONNECTIONS, message=f"How would you like to configure the database connections?", choices=c.CONF_FORMAT_CHOICES
+                ),
+                inquirer.List(
+                    PARAMETERS, message=f"How would you like to configure the parameters?", choices=c.CONF_FORMAT_CHOICES2
+                ),
+                inquirer.List(
+                    DBVIEW, message="What's the file format for the database view model?", choices=c.FILE_TYPE_CHOICES
+                ),
+                inquirer.List(
+                    FEDERATE, message="What's the file format for the federated model?", choices=c.FILE_TYPE_CHOICES
+                ),
+                inquirer.Confirm(
+                    DASHBOARD, message=f"Do you want to include a dashboard example?", default=False
+                ),
+                inquirer.Confirm(
+                    AUTH, message=f"Do you want to add the '{c.AUTH_FILE}' file to enable custom API authentication?", default=False
+                ),
             ]
-            answers.update(inquirer.prompt(remaining_questions))
+            answers = inquirer.prompt(questions)
+            assert isinstance(answers, dict)
         
-        if answers.get(CONNECTIONS) is None:
-            answers[CONNECTIONS] = c.YML_FORMAT
-        if answers.get(PARAMETERS) is None:
-            answers[PARAMETERS] = c.PYTHON_FORMAT
-        if answers.get(DBVIEW) is None:
-            answers[DBVIEW] = c.SQL_FILE_TYPE
-        if answers.get(FEDERATE) is None:
-            answers[FEDERATE] = c.SQL_FILE_TYPE
-
-        if answers.get(CORE, False):
-            connections_format = answers.get(CONNECTIONS)
-            connections_use_yaml = (connections_format == c.YML_FORMAT)
-            connections_use_py = (connections_format == c.PYTHON_FORMAT)
-
-            parameters_format = answers.get(PARAMETERS)
-            parameters_use_yaml = (parameters_format == c.YML_FORMAT)
-            parameters_use_py = (parameters_format == c.PYTHON_FORMAT)
-
-            db_view_format = answers.get(DBVIEW)
-            if db_view_format == c.SQL_FILE_TYPE:
-                db_view_file = c.DATABASE_VIEW_SQL_FILE
-            elif db_view_format == c.PYTHON_FILE_TYPE:
-                db_view_file = c.DATABASE_VIEW_PY_FILE
-            else:
-                raise NotImplementedError(f"Database view format '{db_view_format}' not supported")
+        def get_answer(key, default):
+            """
+            If key is in answers dict as None, using `.get` on a dictionary will return None even if a default is provided.
+
+            For instance, the following prints None.
+            >>> test_dict = {"key": None}
+            >>> print(test_dict.get("key", "default"))
+
+            This function will return the default value if the key is in the dict with value None.
+            """
+            answer = answers.get(key)
+            return answer if answer is not None else default
         
-            federate_format = answers.get(FEDERATE)
-            if federate_format == c.SQL_FILE_TYPE:
-                federate_file = c.FEDERATE_SQL_NAME
-            elif federate_format == c.PYTHON_FILE_TYPE:
-                federate_file = c.FEDERATE_PY_NAME
-            else:
-                raise NotImplementedError(f"Dataset format '{federate_format}' not supported")
+        connections_format = get_answer(CONNECTIONS, c.YML_FORMAT)
+        connections_use_yaml = (connections_format == c.YML_FORMAT)
+        connections_use_py = (connections_format == c.PYTHON_FORMAT)
+
+        parameters_format = get_answer(PARAMETERS, c.PYTHON_FORMAT)
+        parameters_use_yaml = (parameters_format == c.YML_FORMAT)
+        parameters_use_py = (parameters_format == c.PYTHON_FORMAT)
+
+        dbview_format = get_answer(DBVIEW, c.SQL_FILE_TYPE)
+        if dbview_format == c.SQL_FILE_TYPE:
+            db_view_file = c.DBVIEW_FILE_STEM + ".sql"
+        elif dbview_format == c.PYTHON_FILE_TYPE:
+            db_view_file = c.DBVIEW_FILE_STEM + ".py"
+        else:
+            raise NotImplementedError(f"Dbview model format '{dbview_format}' not supported")
     
-            def create_manifest_file():
-                def get_content(file_name: Optional[str]) -> str:
-                    if file_name is None:
-                        return ""
-                    
-                    yaml_path = u.join_paths(base_proj_dir, file_name)
-                    return u.read_file(yaml_path)
-                
-                file_name_dict = {
-                    "parameters": c.PARAMETERS_YML_FILE if parameters_use_yaml else None, 
-                    "connections": c.CONNECTIONS_YML_FILE if connections_use_yaml else None
-                }
-                substitutions = {key: get_content(val) for key, val in file_name_dict.items()}
-                
-                manifest_template = get_content(c.MANIFEST_JINJA_FILE)
-                manifest_content = u.render_string(manifest_template, **substitutions)
-                output_path = u.join_paths(base_proj_dir, TMP_FOLDER, c.MANIFEST_FILE)
-                with open(u.join_paths(output_path), "w") as f:
-                    f.write(manifest_content)
-
-            create_manifest_file()
-            
-            self._copy_file(".gitignore")
-            self._copy_file(c.MANIFEST_FILE, src_folder=TMP_FOLDER)
-            
-            if connections_use_py:
-                self._copy_pyconfig_file(c.CONNECTIONS_FILE)
-            elif connections_use_yaml:
-                pass # already included in squirrels.yml
-            else:
-                raise NotImplementedError(f"Format '{connections_format}' not supported for configuring database connections")
-            
-            if parameters_use_py:
-                self._copy_pyconfig_file(c.PARAMETERS_FILE)
-            elif parameters_use_yaml:
-                pass # already included in squirrels.yml
-            else:
-                raise NotImplementedError(f"Format '{parameters_format}' not supported for configuring widget parameters")
-            
-            self._copy_pyconfig_file(c.CONTEXT_FILE)
-            self._copy_file(c.ENV_CONFIG_FILE)
-            self._copy_seed_file(c.CATEGORY_SEED_FILE)
-            self._copy_seed_file(c.SUBCATEGORY_SEED_FILE)
+        federate_format = get_answer(FEDERATE, c.SQL_FILE_TYPE)
+        if federate_format == c.SQL_FILE_TYPE:
+            federate_file = c.FEDERATE_FILE_STEM + ".sql"
+        elif federate_format == c.PYTHON_FILE_TYPE:
+            federate_file = c.FEDERATE_FILE_STEM + ".py"
+        else:
+            raise NotImplementedError(f"Federate model format '{federate_format}' not supported")
 
-            self._copy_dbview_file(db_view_file)
-            self._copy_federate_file(federate_file)
+        dashboards_enabled = get_answer(DASHBOARD, False)
+
+        self._create_manifest_file(connections_use_yaml, parameters_use_yaml, dashboards_enabled)
+        
+        self._copy_file(u.Path(".gitignore"))
+        self._copy_file(u.Path(c.ENV_CONFIG_FILE))
+        
+        if connections_use_py:
+            self._copy_pyconfig_file(c.CONNECTIONS_FILE)
+        elif connections_use_yaml:
+            pass # already included in squirrels.yml
+        else:
+            raise NotImplementedError(f"Format '{connections_format}' not supported for configuring database connections")
+        
+        if parameters_use_py:
+            self._copy_pyconfig_file(c.PARAMETERS_FILE)
+        elif parameters_use_yaml:
+            pass # already included in squirrels.yml
+        else:
+            raise NotImplementedError(f"Format '{parameters_format}' not supported for configuring widget parameters")
         
-        if answers.get(AUTH, False):
+        self._copy_pyconfig_file(c.CONTEXT_FILE)
+        self._copy_seed_file(c.CATEGORY_SEED_FILE)
+        self._copy_seed_file(c.SUBCATEGORY_SEED_FILE)
+
+        self._copy_dbview_file(db_view_file)
+        self._copy_federate_file(federate_file)
+        
+        if dashboards_enabled:
+            self._copy_dashboard_file(c.DASHBOARD_FILE_STEM + ".py")
+        
+        if get_answer(AUTH, False):
             self._copy_pyconfig_file(c.AUTH_FILE)
 
-        sample_db = answers.get(SAMPLE_DB)
-        if sample_db is None: # None if not prompt mode and '--sample-db' option was not specified
-            sample_db = c.EXPENSES_DB_NAME
+        self._copy_database_file(c.EXPENSES_DB)
         
-        if sample_db != c.NO_DB:
-            if sample_db == c.EXPENSES_DB_NAME:
-                self._copy_database_file(c.EXPENSES_DB_NAME+".db")
-            elif sample_db == c.WEATHER_DB_NAME:
-                self._copy_database_file(c.WEATHER_DB_NAME+".db")
+        print(f"\nSuccessfully created new Squirrels project in current directory!\n")
+    
+    def get_file(self, args):
+        if args.file_name == c.ENV_CONFIG_FILE:
+            self._copy_file(u.Path(c.ENV_CONFIG_FILE))
+            print("PLEASE ENSURE THE FILE IS INCLUDED IN .gitignore")
+        elif args.file_name == c.MANIFEST_FILE:
+            self._create_manifest_file(not args.no_connections, args.parameters, args.dashboards)
+        elif args.file_name in (c.AUTH_FILE, c.CONNECTIONS_FILE, c.PARAMETERS_FILE, c.CONTEXT_FILE):
+            self._copy_pyconfig_file(args.file_name)
+        elif args.file_name in (c.DBVIEW_FILE_STEM, c.FEDERATE_FILE_STEM):
+            if args.format == c.SQL_FILE_TYPE:
+                extension = ".sql"
+            elif args.format == c.PYTHON_FILE_TYPE:
+                extension = ".py"
             else:
-                raise NotImplementedError(f"No database found called '{sample_db}'")
-    
+                raise NotImplementedError(f"Format '{args.format}' not supported for {args.file_name}")
+            copy_method = self._copy_dbview_file if args.file_name == c.DBVIEW_FILE_STEM else self._copy_federate_file
+            copy_method(args.file_name + extension)
+        elif args.file_name == c.DASHBOARD_FILE_STEM:
+            self._copy_dashboard_file(args.file_name + ".py")
+        elif args.file_name in (c.EXPENSES_DB, c.WEATHER_DB):
+            self._copy_database_file(args.file_name)
+        else:
+            raise NotImplementedError(f"File '{args.file_name}' not supported")
+