squirrels 0.1.1.post1__py3-none-any.whl → 0.2.0.dev0__py3-none-any.whl

squirrels/_command_line.py

@@ -1,24 +1,9 @@
-from typing import List, Tuple, Optional
 from argparse import ArgumentParser
-import sys, time, pwinput
+import sys, time, asyncio
 sys.path.append('.')
 
-from squirrels import _constants as c, _credentials_manager as cm, _manifest as mf, _module_loader as ml
-from squirrels import connection_set as cs
-from squirrels._version import __version__
-from squirrels._api_server import ApiServer
-from squirrels._renderer import RendererIOWrapper
-from squirrels._initializer import Initializer
-from squirrels._timed_imports import timer
-
-
-def _prompt_user_pw(args_values: Optional[List[str]]) -> Tuple[str, str]:
-    if args_values is not None:
-        user, pw = args_values
-    else:
-        user = input("Enter username: ")
-        pw = pwinput.pwinput("Enter password: ")
-    return user, pw
+from . import _constants as c
+from ._timer import timer
 
 
 def main():
@@ -26,81 +11,84 @@ def main():
     Main entry point for the squirrels command line utilities.
     """
     start = time.time()
-    parser = ArgumentParser(description="Command line utilities from the squirrels python package")
-    parser.add_argument('-v', '--version', action='store_true', help='Show the version and exit')
-    parser.add_argument('--verbose', action='store_true', help='Enable verbose output')
+    parser = ArgumentParser(description="Command line utilities from the squirrels python package", add_help=False)
+    parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
+    parser.add_argument('-V', '--version', action='store_true', help='Show the version and exit')
+    parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose output')
     subparsers = parser.add_subparsers(title='commands', dest='command')
 
-    init_parser = subparsers.add_parser(c.INIT_CMD, help='Initialize a squirrels project')
-    init_parser.add_argument('--overwrite', action='store_true', help="Overwrite files that already exist")
-    init_parser.add_argument('--core', action='store_true', help='Include all core files (squirrels.yaml, parameters.py, database view, etc.)')
-    init_parser.add_argument('--db-view', type=str, choices=c.FILE_TYPE_CHOICES, help='Create database view as sql (default) or python file. Ignored if "--core" is not specified')
-    init_parser.add_argument('--connections', action='store_true', help=f'Include the {c.CONNECTIONS_FILE} file')
-    init_parser.add_argument('--context', action='store_true', help=f'Include the {c.CONTEXT_FILE} file')
-    init_parser.add_argument('--selections-cfg', action='store_true', help=f'Include the {c.SELECTIONS_CFG_FILE} file')
-    init_parser.add_argument('--final-view', type=str, choices=c.FILE_TYPE_CHOICES, help='Include final view as sql or python file')
+    init_parser = subparsers.add_parser(c.INIT_CMD, help='Initialize a squirrels project', add_help=False)
+    init_parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
+    init_parser.add_argument('-o', '--overwrite', action='store_true', help="Overwrite files that already exist")
+    init_parser.add_argument('--core', action='store_true', help='Include all core files')
+    init_parser.add_argument('--connections', type=str, choices=c.CONF_FORMAT_CHOICES, help=f'Configure database connections as yaml (default) or python. Ignored if "--core" is not specified')
+    init_parser.add_argument('--parameters', type=str, choices=c.CONF_FORMAT_CHOICES, help=f'Configure parameters as python (default) or yaml. Ignored if "--core" is not specified')
+    init_parser.add_argument('--dbview', type=str, choices=c.FILE_TYPE_CHOICES, help='Create database view model as sql (default) or python file. Ignored if "--core" is not specified')
+    init_parser.add_argument('--federate', type=str, choices=c.FILE_TYPE_CHOICES, help='Create federated model as sql (default) or python file. Ignored if "--core" is not specified')
+    init_parser.add_argument('--auth', action='store_true', help=f'Include the {c.AUTH_FILE} file')
     init_parser.add_argument('--sample-db', type=str, choices=c.DATABASE_CHOICES, help='Sample sqlite database to include')
 
-    subparsers.add_parser(c.LOAD_MODULES_CMD, help='Load all the modules specified in squirrels.yaml from git')
-
-    def _add_profile_argument(parser: ArgumentParser):
-        parser.add_argument('key', type=str, help='Key to the database connection credential')
+    module_parser = subparsers.add_parser(c.DEPS_CMD, help=f'Load all packages specified in {c.MANIFEST_FILE} (from git)', add_help=False)
+    module_parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
 
-    set_cred_parser = subparsers.add_parser(c.SET_CRED_CMD, help='Set a database connection credential key')
-    _add_profile_argument(set_cred_parser)
-    set_cred_parser.add_argument('--values', type=str, nargs=2, help='The username and password')
+    compile_parser = subparsers.add_parser(c.COMPILE_CMD, help='Create files for rendered sql queries in the "target/compile" folder', add_help=False)
+    compile_parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
+    compile_parser.add_argument('-d', '--dataset', type=str, help="Select dataset to use for dataset args. If not specified, all models for all datasets are compiled")
+    compile_parser.add_argument('-a', '--all-test-sets', action="store_true", help="Compile models for all test sets")
+    compile_parser.add_argument('-t', '--test-set', type=str, help="The selection test set to use. Default selections are used if not specified. Ignored if using --all-test-sets")
+    compile_parser.add_argument('-s', '--select', type=str, help="Select single model to compile. If not specified, all models for the dataset are compiled. Also, ignored if --dataset is not specified")
+    compile_parser.add_argument('-r', '--runquery', action='store_true', help='Runs all target models, and produce the results as csv files')
 
-    subparsers.add_parser(c.GET_CREDS_CMD, help='Get all database connection credential keys')
-
-    delete_cred_parser = subparsers.add_parser(c.DELETE_CRED_CMD, help='Delete a database connection credential key')
-    _add_profile_argument(delete_cred_parser)
-
-    test_parser = subparsers.add_parser(c.TEST_CMD, help='For a given dataset, create outputs for parameter API response and rendered sql queries')
-    test_parser.add_argument('dataset', type=str, help='Name of dataset (provided in squirrels.yaml) to test. Results are written in an "outputs" folder')
-    test_parser.add_argument('-c', '--cfg', type=str, help="Configuration file for parameter selections. Path is relative to the dataset's folder")
-    test_parser.add_argument('-d', '--data', type=str, help="Excel file with lookup data to avoid making a database connection. Path is relative to the dataset's folder")
-    test_parser.add_argument('-r', '--runquery', action='store_true', help='Runs all database queries and final view, and produce the results as csv files')
-
-    run_parser = subparsers.add_parser(c.RUN_CMD, help='Run the builtin API server')
+    run_parser = subparsers.add_parser(c.RUN_CMD, help='Run the builtin API server', add_help=False)
+    run_parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
     run_parser.add_argument('--no-cache', action='store_true', help='Do not cache any api results')
-    run_parser.add_argument('--debug', action='store_true', help='In debug mode, all "hidden parameters" show in the parameters response')
-    run_parser.add_argument('--host', type=str, default='127.0.0.1')
-    run_parser.add_argument('--port', type=int, default=8000)
+    run_parser.add_argument('--debug', action='store_true', help='Show all "hidden parameters" in the parameters response')
+    run_parser.add_argument('--host', type=str, default='127.0.0.1', help="The host to run on")
+    run_parser.add_argument('--port', type=int, default=4465, help="The port to run on")
 
     args, _ = parser.parse_known_args()
     timer.verbose = args.verbose
     timer.add_activity_time('parsing arguments', start)
+    
+    from . import __version__
+    from ._api_server import ApiServer
+    from ._models import ModelsIO
+    from ._initializer import Initializer
+    from ._manifest import ManifestIO
+    from ._package_loader import PackageLoaderIO
+    from ._connection_set import ConnectionSetIO
+    from ._parameter_sets import ParameterConfigsSetIO
 
     if args.version:
         print(__version__)
     elif args.command == c.INIT_CMD:
         Initializer(args.overwrite).init_project(args)
-    elif args.command == c.LOAD_MODULES_CMD:
-        manifest = mf._from_file()
-        ml.load_modules(manifest)
-    elif args.command == c.SET_CRED_CMD:
-        user, pw = _prompt_user_pw(args.values)
-        cm.squirrels_config_io.set_credential(args.key, user, pw)
-    elif args.command == c.GET_CREDS_CMD: 
-        cm.squirrels_config_io.print_all_credentials()
-    elif args.command == c.DELETE_CRED_CMD:
-        cm.squirrels_config_io.delete_credential(args.key)
-    elif args.command in [c.RUN_CMD, c.TEST_CMD]:
-        manifest = mf._from_file()
-        conn_set = cs._from_file(manifest)
-        if args.command == c.RUN_CMD:
-            server = ApiServer(manifest, conn_set, args.no_cache, args.debug)
-            server.run(args)
-        elif args.command == c.TEST_CMD:
-            rendererIO = RendererIOWrapper(args.dataset, manifest, conn_set, args.data)
-            rendererIO.write_outputs(args.cfg, args.runquery)
-        conn_set._dispose()
+    elif args.command == c.DEPS_CMD:
+        ManifestIO.LoadFromFile()
+        PackageLoaderIO.LoadPackages(reload=True)
+    elif args.command in [c.RUN_CMD, c.COMPILE_CMD]:
+        ManifestIO.LoadFromFile()
+        PackageLoaderIO.LoadPackages()
+        ConnectionSetIO.LoadFromFile()
+        try:
+            ParameterConfigsSetIO.LoadFromFile()
+            ModelsIO.LoadFiles()
+            
+            if args.command == c.RUN_CMD:
+                server = ApiServer(args.no_cache, args.debug)
+                server.run(args)
+                pass
+            elif args.command == c.COMPILE_CMD:
+                task = ModelsIO.WriteOutputs(args.dataset, args.select, args.all_test_sets, args.test_set, args.runquery)
+                asyncio.run(task)
+        finally:
+            ConnectionSetIO.Dispose()
     elif args.command is None:
         print(f'Command is missing. Enter "squirrels -h" for help.')
     else:
         print(f'Error: No such command "{args.command}". Enter "squirrels -h" for help.')
     
-    timer.report_times()
+    # timer.report_times()
 
 
 if __name__ == '__main__':

squirrels/_connection_set.py

@@ -0,0 +1,96 @@
+from typing import Union
+from dataclasses import dataclass
+from sqlalchemy import Engine, Pool, create_engine
+import pandas as pd
+
+from . import _utils as u, _constants as c, _py_module as pm
+from .arguments.init_time_args import ConnectionsArgs
+from ._environcfg import EnvironConfigIO
+from ._manifest import ManifestIO
+from ._timer import timer, time
+
+
+@dataclass
+class ConnectionSet:
+    """
+    A wrapper class around a collection of Connection Pools or Sqlalchemy Engines
+
+    Attributes:
+        conn_pools: A dictionary of connection pool name to the corresponding Pool or Engine from sqlalchemy
+    """
+    _conn_pools: dict[str, Union[Engine, Pool]]
+
+    def get_connections_as_dict(self):
+        return self._conn_pools
+    
+    def get_connection_pool(self, conn_name: str) -> Union[Engine, Pool]:
+        try:
+            connection_pool = self._conn_pools[conn_name]
+        except KeyError as e:
+            raise u.ConfigurationError(f'Connection name "{conn_name}" was not configured') from e
+        return connection_pool
+    
+    def run_sql_query_from_conn_name(self, query: str, conn_name: str) -> pd.DataFrame:
+        connection_pool = self.get_connection_pool(conn_name)
+        if isinstance(connection_pool, Pool):
+            conn = connection_pool.connect()
+        elif isinstance(connection_pool, Engine):
+            conn = connection_pool.raw_connection()
+        else:
+            raise u.ConfigurationError(f'Value type not supported for connection name "{conn_name}". Must be sqlalchemy engine or pool')
+        
+        try:
+            cur = conn.cursor()
+            try:
+                cur.execute(query)
+            except Exception as e:
+                raise RuntimeError(e)
+            df = pd.DataFrame(data=cur.fetchall(), columns=[x[0] for x in cur.description])
+        finally:
+            conn.close()
+
+        return df
+
+    def _dispose(self) -> None:
+        """
+        Disposes of all the connection pools in this ConnectionSet
+        """
+        for pool in self._conn_pools.values():
+            if isinstance(pool, (Engine, Pool)):
+                pool.dispose()
+
+
+class ConnectionSetIO:
+    args: ConnectionsArgs
+    obj: ConnectionSet
+
+    @classmethod
+    def LoadFromFile(cls):
+        """
+        Takes the DB Connections from both the squirrels.yml and connections.py files and merges them
+        into a single ConnectionSet
+        
+        Returns:
+            A ConnectionSet with the DB connections from both squirrels.yml and connections.py
+        """
+        start = time.time()
+        connections: dict[str, Union[Engine, Pool]] = {}
+        cls.obj = ConnectionSet(connections)
+        try:
+            for config in ManifestIO.obj.db_connections:
+                connections[config.connection_name] = create_engine(config.url)
+            
+            proj_vars = ManifestIO.obj.project_variables
+            env_vars = EnvironConfigIO.obj.get_all_env_vars()
+            get_credential = EnvironConfigIO.obj.get_credential
+            cls.args = ConnectionsArgs(proj_vars, env_vars, get_credential)
+            pm.run_pyconfig_main(c.CONNECTIONS_FILE, {"connections": connections, "sqrl": cls.args})
+        except Exception as e:
+            cls.Dispose()
+            raise e
+        
+        timer.add_activity_time("creating sqlalchemy engines or pools", start)
+
+    @classmethod
+    def Dispose(cls):
+        cls.obj._dispose()

squirrels/_constants.py

@@ -1,64 +1,145 @@
 # Squirrels CLI commands
 INIT_CMD = 'init'
-LOAD_MODULES_CMD = 'load-modules'
-GET_CREDS_CMD = 'get-all-credentials'
-SET_CRED_CMD = 'set-credential'
-DELETE_CRED_CMD = 'delete-credential'
-TEST_CMD = 'test'
+DEPS_CMD = 'deps'
+COMPILE_CMD = 'compile'
 RUN_CMD = 'run'
 
 # Manifest file keys
 PROJ_VARS_KEY = 'project_variables'
+PROJECT_NAME_KEY = 'name'
+PROJECT_LABEL_KEY = 'label'
+MAJOR_VERSION_KEY = 'major_version'
+MINOR_VERSION_KEY = 'minor_version'
+
+PACKAGES_KEY = 'packages'
+PACKAGE_GIT_KEY = 'git'
+PACKAGE_DIRECTORY_KEY = 'directory'
+PACKAGE_REVISION_KEY = 'revision'
+
 DB_CONNECTIONS_KEY = 'db_connections'
-DB_CREDENTIALS_KEY = 'credential_key'
-URL_KEY = 'url'
-DB_CONNECTION_KEY = 'db_connection'
-MODULES_KEY = 'modules'
-DATASET_LABEL_KEY = 'label'
+DB_CONN_NAME_KEY = 'connection_name'
+DB_CONN_CRED_KEY = 'credential_key'
+DB_CONN_URL_KEY = 'url'
+
+DBVIEWS_KEY = 'dbviews'
+DBVIEW_NAME_KEY = 'name'
+DBVIEW_CONN_KEY = 'connection_name'
+DEFAULT_DB_CONN = 'default'
+
+FEDERATES_KEY = 'federates'
+FEDERATE_NAME_KEY = 'name'
+MATERIALIZED_KEY = 'materialized'
+DEFAULT_TABLE_MATERIALIZE = 'table'
+
+PARAMETERS_KEY = 'parameters'
+PARAMETER_NAME_KEY = 'name'
+PARAMETER_TYPE_KEY = 'type'
+PARAMETER_FACTORY_KEY = 'factory'
+PARAMETER_ARGS_KEY = 'arguments'
+
+TEST_SETS_KEY = 'selection_test_sets'
+TEST_SET_NAME_KEY = 'name'
+DEFAULT_TEST_SET_NAME = 'default'
+TEST_SET_USER_ATTR_KEY = 'user_attributes'
+TEST_SET_PARAMETERS_KEY = 'parameters'
+
 DATASETS_KEY = 'datasets'
-HEADERS_KEY = 'headers'
+DATASET_NAME_KEY = 'name'
+DATASET_LABEL_KEY = 'label'
+DATASET_MODEL_KEY = 'model'
+DATASET_PARAMETERS_KEY = 'parameters'
+DATASET_ARGS_KEY = 'args'
+
+# deprecate these later
 DATABASE_VIEWS_KEY = 'database_views'
 FILE_KEY = 'file'
+DB_CONNECTION_KEY = 'db_connection'
 FINAL_VIEW_KEY = 'final_view'
-BASE_PATH_KEY = 'base_path'
-SETTINGS_KEY = 'settings'
 
-# Project variable keys
-PRODUCT_KEY = 'product'
-MAJOR_VERSION_KEY = 'major_version'
-MINOR_VERSION_KEY = 'minor_version'
+DATASET_SCOPE_KEY = 'scope'
+PUBLIC_SCOPE = 'public'
+PROTECTED_SCOPE = 'protected'
+PRIVATE_SCOPE = 'private'
 
-# Database credentials keys
+SETTINGS_KEY = 'settings'
+
+# Environment config keys
 CREDENTIALS_KEY = 'credentials'
 USERNAME_KEY = 'username'
 PASSWORD_KEY = 'password'
-DEFAULT_DB_CONN = 'default'
+USER_NAME_KEY = 'username'
+USER_PWD_KEY = 'password'
+SECRETS_KEY = 'secrets'
+JWT_SECRET_KEY = 'jwt_secret'
 
 # Folder/File names
-MANIFEST_FILE = 'squirrels.yaml'
+PACKAGE_DATA_FOLDER = 'package_data'
+BASE_PROJECT_FOLDER = 'base_project'
+STATIC_FOLDER = 'static'
+TEMPLATES_FOLDER = 'templates'
+
+ENVIRON_CONFIG_FILE = 'environcfg.yml'
+MANIFEST_JINJA_FILE = 'squirrels.yml.j2'
+CONNECTIONS_YML_FILE = 'connections.yml'
+PARAMETERS_YML_FILE = 'parameters.yml'
+MANIFEST_FILE = 'squirrels.yml'
+LU_DATA_FILE = 'lu_data.xlsx'
+
+DATABASE_FOLDER = 'database'
+PACKAGES_FOLDER = 'sqrl_packages'
+
+MODELS_FOLDER = 'models'
+DBVIEWS_FOLDER = 'dbviews'
+DATABASE_VIEW_SQL_FILE = 'database_view1.sql'
+DATABASE_VIEW_PY_FILE = 'database_view1.py'
+FEDERATES_FOLDER = 'federates'
+FEDERATE_SQL_NAME = 'dataset_example.sql'
+FEDERATE_PY_NAME = 'dataset_example.py'
+
+PYCONFIG_FOLDER = 'pyconfigs'
+AUTH_FILE = 'auth.py'
 CONNECTIONS_FILE = 'connections.py'
-OUTPUTS_FOLDER = 'outputs'
-MODULES_FOLDER = 'modules'
-DATASETS_FOLDER = 'datasets'
+CONTEXT_FILE = 'context.py'
 PARAMETERS_FILE = 'parameters.py'
+
+TARGET_FOLDER = 'target'
+COMPILE_FOLDER = 'compile'
+
+OUTPUTS_FOLDER = 'outputs'
 PARAMETERS_OUTPUT = 'parameters.json'
-DATABASE_VIEW_SQL_FILE = 'database_view1.sql.j2'
-DATABASE_VIEW_PY_FILE = 'database_view1.py'
-FINAL_VIEW_SQL_NAME = 'final_view.sql.j2'
-FINAL_VIEW_PY_NAME = 'final_view.py'
 FINAL_VIEW_OUT_STEM = 'final_view'
-CONTEXT_FILE = 'context.py'
-SELECTIONS_CFG_FILE = 'selections.cfg'
 
 # Dataset setting names
+AUTH_TOKEN_EXPIRE_SETTING = 'auth.token.expire.minutes'
 PARAMETERS_CACHE_SIZE_SETTING = 'parameters.cache.size'
-PARAMETERS_CACHE_TTL_SETTING = 'parameters.cache.ttl'
+PARAMETERS_CACHE_TTL_SETTING = 'parameters.cache.ttl.minutes'
 RESULTS_CACHE_SIZE_SETTING = 'results.cache.size'
-RESULTS_CACHE_TTL_SETTING = 'results.cache.ttl'
+RESULTS_CACHE_TTL_SETTING = 'results.cache.ttl.minutes'
+TEST_SET_DEFAULT_USED_SETTING = 'selection_test_sets.default_name_used'
+DB_CONN_DEFAULT_USED_SETTING = 'db_connections.default_name_used'
+DEFAULT_MATERIALIZE_SETTING = 'defaults.federates.materialized'
 
 # Selection cfg sections
+USER_ATTRIBUTES_SECTION = 'user_attributes'
 PARAMETERS_SECTION = 'parameters'
 
 # Init Command Choices
-FILE_TYPE_CHOICES = ['sql', 'py']
-DATABASE_CHOICES = ['sample_database', 'seattle_weather']
+SQL_FILE_TYPE = 'sql'
+PYTHON_FILE_TYPE = 'py'
+FILE_TYPE_CHOICES = [SQL_FILE_TYPE, PYTHON_FILE_TYPE]
+
+YML_FORMAT = 'yml'
+PYTHON_FORMAT = 'py'
+CONF_FORMAT_CHOICES = [YML_FORMAT, PYTHON_FORMAT]
+
+PYTHON_FORMAT2 = 'py (recommended)'
+CONF_FORMAT_CHOICES2 = [(PYTHON_FORMAT2, PYTHON_FORMAT), YML_FORMAT]
+
+EXPENSES_DB_NAME = 'expenses'
+WEATHER_DB_NAME = 'seattle-weather'
+DATABASE_CHOICES = [EXPENSES_DB_NAME, WEATHER_DB_NAME]
+
+# Function names
+GET_USER_FUNC = "get_user_if_valid"
+DEP_FUNC = "dependencies"
+MAIN_FUNC = "main"

squirrels/_environcfg.py

@@ -0,0 +1,77 @@
+from typing import Any, Optional, Callable
+from dataclasses import dataclass
+import os, yaml
+
+from . import _constants as c, _utils as u
+from ._timer import timer, time
+
+_GLOBAL_SQUIRRELS_CFG_FILE = u.join_paths(os.path.expanduser('~'), '.squirrels', c.ENVIRON_CONFIG_FILE)
+
+
+@dataclass
+class _EnvironConfig:
+    _users: dict[str, dict[str, Any]]
+    _env_vars: dict[str, str]
+    _credentials: dict[str, dict[str, str]]
+    _secrets: dict[str, str]
+    
+    def __post_init__(self):
+        for username, user in self._users.items():
+            user[c.USER_NAME_KEY] = username
+            if c.USER_PWD_KEY not in user:
+                raise u.ConfigurationError(f"All users must have password in environcfg.yml")
+        
+        for key, cred in self._credentials.items():
+            if c.USERNAME_KEY not in cred or c.PASSWORD_KEY not in cred:
+                raise u.ConfigurationError(f'Either "{c.USERNAME_KEY}" or "{c.PASSWORD_KEY}" was not specified for credential "{key}"')
+    
+    def get_users(self) -> dict[str, dict[str, Any]]:
+        return self._users.copy()
+    
+    def get_all_env_vars(self) -> dict[str, str]:
+        return self._env_vars.copy()
+    
+    def get_credential(self, key: Optional[str]) -> tuple[str, str]:
+        if not key:
+            return "", ""
+
+        try:
+            credential = self._credentials[key]
+        except KeyError as e:
+            raise u.ConfigurationError(f'No credentials configured for "{key}"') from e
+    
+        return credential[c.USERNAME_KEY], credential[c.PASSWORD_KEY]
+    
+    def get_secret(self, key: str, *, default_factory: Optional[Callable[[],str]] = None) -> str:
+        if key not in self._secrets and default_factory is not None:
+            self._secrets[key] = default_factory()
+        return self._secrets.get(key)
+
+
+class EnvironConfigIO:
+    obj: _EnvironConfig
+    
+    @classmethod
+    def LoadFromFile(cls):
+        start = time.time()
+        def load_yaml(filename: str) -> dict[str, dict]:
+            try:
+                with open(filename, 'r') as f:
+                    return yaml.safe_load(f)
+            except FileNotFoundError:
+                return {}
+        
+        config1 = load_yaml(_GLOBAL_SQUIRRELS_CFG_FILE)
+        config2 = load_yaml(c.ENVIRON_CONFIG_FILE)
+
+        for key in config2:
+            config1.setdefault(key, {})
+            config1[key].update(config2[key])
+        
+        users = config1.get("users", {})
+        env_vars = config1.get("env_vars", {})
+        credentials = config1.get("credentials", {})
+        secrets = config1.get("secrets", {})
+
+        cls.obj = _EnvironConfig(users, env_vars, credentials, secrets)
+        timer.add_activity_time(f"loading {c.ENVIRON_CONFIG_FILE} file", start)