squirrels 0.1.1.post1__py3-none-any.whl → 0.2.0.dev0__py3-none-any.whl

squirrels/_initializer.py

@@ -1,10 +1,9 @@
+from typing import Optional
 import inquirer, os, shutil
 
-from squirrels import _constants as c, _utils
-from squirrels._version import major_version
+from . import _constants as c, _utils as u
 
-base_proj_dir = _utils.join_paths(os.path.dirname(__file__), 'package_data', 'base_project')
-dataset_dir = _utils.join_paths('datasets', 'sample_dataset')
+base_proj_dir = u.join_paths(os.path.dirname(__file__), c.PACKAGE_DATA_FOLDER, c.BASE_PROJECT_FOLDER)
 
 
 class Initializer:
@@ -17,94 +16,154 @@ class Initializer:
             return True
         return False
     
-    def _copy_file(self, filepath: str):
+    def _copy_file(self, filepath: str, *, src_folder: str = ""):
         if not self._path_exists(filepath):
             dest_dir = os.path.dirname(filepath)
-            if dest_dir != '':
+            if dest_dir != "":
                 os.makedirs(dest_dir, exist_ok=True)
-            src_path = _utils.join_paths(base_proj_dir, filepath)
+            src_path = u.join_paths(base_proj_dir, src_folder, filepath)
             shutil.copy(src_path, filepath)
 
-    def _copy_dataset_file(self, filepath: str):
-        self._copy_file(_utils.join_paths(dataset_dir, filepath))
+    def _copy_dbview_file(self, filepath: str):
+        self._copy_file(u.join_paths(c.MODELS_FOLDER, c.DBVIEWS_FOLDER, filepath))
 
-    def _copy_database_file(self, filepath: str):
-        self._copy_file(_utils.join_paths('database', filepath))
+    def _copy_federate_file(self, filepath: str):
+        self._copy_file(u.join_paths(c.MODELS_FOLDER, c.FEDERATES_FOLDER, filepath))
 
-    def _create_requirements_txt(self):
-        filename = 'requirements.txt'
-        if not self._path_exists(filename):
-            next_major_version = int(major_version) + 1
-            content = f'squirrels<{next_major_version}'
-            with open(filename, 'w') as f:
-                f.write(content)
+    def _copy_database_file(self, filepath: str):
+        self._copy_file(u.join_paths(c.DATABASE_FOLDER, filepath))
+    
+    def _copy_pyconfigs_file(self, filepath: str):
+        self._copy_file(u.join_paths(c.PYCONFIG_FOLDER, filepath))
 
     def init_project(self, args):
-        options = ['core', 'db_view', 'connections', 'context', 'selections_cfg', 'final_view', 'sample_db']
+        options = ["core", "connections", "parameters", "dbview", "federate", "auth", "sample_db"]
+        CORE, CONNECTIONS, PARAMETERS, DBVIEW, FEDERATE, AUTH, SAMPLE_DB = options
+        TMP_FOLDER = "tmp"
+        IGNORES_FOLDER = "ignores"
+
         answers = { x: getattr(args, x) for x in options }
         if not any(answers.values()):
             core_questions = [
-                inquirer.Confirm('core', 
-                                message="Include all core project files?",
-                                default=True)
+                inquirer.Confirm(CORE, 
+                                 message="Include all core project files?",
+                                 default=True)
             ]
             answers = inquirer.prompt(core_questions)
             
-            if answers.get('core', False):
+            if answers.get(CORE, False):
                 conditional_questions = [
-                    inquirer.List('db_view', 
-                                  message="What's the file format for the database view?",
+                    inquirer.List(CONNECTIONS,
+                                  message=f"How would you like to configure the database connections?" ,
+                                  choices=c.CONF_FORMAT_CHOICES),
+                    inquirer.List(PARAMETERS,
+                                  message=f"How would you like to configure the parameters?" ,
+                                  choices=c.CONF_FORMAT_CHOICES2),
+                    inquirer.List(DBVIEW, 
+                                  message="What's the file format for the database view model?",
+                                  choices=c.FILE_TYPE_CHOICES),
+                    inquirer.List(FEDERATE, 
+                                  message="What's the file format for the federated model?",
                                   choices=c.FILE_TYPE_CHOICES),
                 ]
                 answers.update(inquirer.prompt(conditional_questions))
 
             remaining_questions = [
-                inquirer.Confirm('connections',
-                                message=f"Do you want to add a '{c.CONNECTIONS_FILE}' file?" ,
-                                default=False),
-                inquirer.Confirm('context',
-                                message=f"Do you want to add a '{c.CONTEXT_FILE}' file?" ,
-                                default=False),
-                inquirer.Confirm('selections_cfg',
-                                message=f"Do you want to add a '{c.SELECTIONS_CFG_FILE}' file?" ,
-                                default=False),
-                inquirer.List('final_view', 
-                            message="What's the file format for the final view (if any)?",
-                            choices=['none'] + c.FILE_TYPE_CHOICES),
-                inquirer.List('sample_db', 
-                            message="What sample sqlite database do you wish to use (if any)?",
-                            choices=['none'] + c.DATABASE_CHOICES)
+                inquirer.Confirm(AUTH,
+                                 message=f"Do you want to add the '{c.AUTH_FILE}' file?" ,
+                                 default=False),
+                inquirer.List(SAMPLE_DB, 
+                              message="What sample sqlite database do you wish to use (if any)?",
+                              choices=["none"] + c.DATABASE_CHOICES)
             ]
             answers.update(inquirer.prompt(remaining_questions))
+        
+        if answers.get(CONNECTIONS) is None:
+            answers[CONNECTIONS] = c.YML_FORMAT
+        if answers.get(PARAMETERS) is None:
+            answers[PARAMETERS] = c.PYTHON_FORMAT
+        if answers.get(DBVIEW) is None:
+            answers[DBVIEW] = c.SQL_FILE_TYPE
+        if answers.get(FEDERATE) is None:
+            answers[FEDERATE] = c.SQL_FILE_TYPE
+
+        if answers.get(CORE, False):
+            connections_format = answers.get(CONNECTIONS)
+            connections_use_yaml = (connections_format == c.YML_FORMAT)
+            connections_use_py = (connections_format == c.PYTHON_FORMAT)
+
+            parameters_format = answers.get(PARAMETERS)
+            parameters_use_yaml = (parameters_format == c.YML_FORMAT)
+            parameters_use_py = (parameters_format == c.PYTHON_FORMAT)
 
-        if answers.get('core', False):
-            self._copy_file('.gitignore')
-            self._copy_file(c.MANIFEST_FILE)
-            self._create_requirements_txt()
-            self._copy_dataset_file(c.PARAMETERS_FILE)
-            if answers.get('db_view') == 'py':
-                self._copy_dataset_file(c.DATABASE_VIEW_PY_FILE)
+            db_view_format = answers.get(DBVIEW)
+            if db_view_format == c.SQL_FILE_TYPE:
+                db_view_file = c.DATABASE_VIEW_SQL_FILE
+            elif db_view_format == c.PYTHON_FILE_TYPE:
+                db_view_file = c.DATABASE_VIEW_PY_FILE
             else:
-                self._copy_dataset_file(c.DATABASE_VIEW_SQL_FILE)
-        
-        if answers.get('connections', False):
-            self._copy_file(c.CONNECTIONS_FILE)
-        
-        if answers.get('context', False):
-            self._copy_dataset_file(c.CONTEXT_FILE)
+                raise NotImplementedError(f"Database view format '{db_view_format}' not supported")
         
-        if answers.get('selections_cfg', False):
-            self._copy_dataset_file(c.SELECTIONS_CFG_FILE)
+            federate_format = answers.get(FEDERATE)
+            if federate_format == c.SQL_FILE_TYPE:
+                federate_file = c.FEDERATE_SQL_NAME
+            elif federate_format == c.PYTHON_FILE_TYPE:
+                federate_file = c.FEDERATE_PY_NAME
+            else:
+                raise NotImplementedError(f"Dataset format '{federate_format}' not supported")
+    
+            def create_manifest_file():
+                def get_content(file_name: Optional[str]) -> str:
+                    if file_name is None:
+                        return ""
+                    
+                    yaml_path = u.join_paths(base_proj_dir, file_name)
+                    return u.read_file(yaml_path)
+                
+                file_name_dict = {
+                    "parameters": c.PARAMETERS_YML_FILE if parameters_use_yaml else None, 
+                    "connections": c.CONNECTIONS_YML_FILE if connections_use_yaml else None
+                }
+                substitutions = {key: get_content(val) for key, val in file_name_dict.items()}
+                substitutions["db_view_file"] = db_view_file
+                substitutions["final_view_file"] = federate_file
+                
+                manifest_template = get_content(c.MANIFEST_JINJA_FILE)
+                manifest_content = u.render_string(manifest_template, substitutions)
+                output_path = u.join_paths(base_proj_dir, TMP_FOLDER, c.MANIFEST_FILE)
+                with open(u.join_paths(output_path), "w") as f:
+                    f.write(manifest_content)
+
+            create_manifest_file()
+            
+            self._copy_file(".gitignore", src_folder=IGNORES_FOLDER)
+            self._copy_file(c.MANIFEST_FILE, src_folder=TMP_FOLDER)
+            
+            if connections_use_py:
+                self._copy_pyconfigs_file(c.CONNECTIONS_FILE)
+            elif not connections_use_yaml:
+                raise NotImplementedError(f"Format '{connections_format}' not supported for configuring database connections")
+            
+            if parameters_use_py:
+                self._copy_pyconfigs_file(c.PARAMETERS_FILE)
+            elif not parameters_use_yaml:
+                raise NotImplementedError(f"Format '{parameters_format}' not supported for configuring widget parameters")
+            
+            self._copy_pyconfigs_file(c.CONTEXT_FILE)
+            self._copy_file(c.ENVIRON_CONFIG_FILE)
+
+            self._copy_dbview_file(db_view_file)
+            self._copy_federate_file(federate_file)
         
-        final_view_format = answers.get('final_view')
-        if final_view_format == 'py':
-            self._copy_dataset_file(c.FINAL_VIEW_PY_NAME)
-        elif final_view_format == 'sql':
-            self._copy_dataset_file(c.FINAL_VIEW_SQL_NAME)
-
-        sample_db = answers.get('sample_db')
-        if sample_db == 'sample_database':
-            self._copy_database_file('sample_database.db')
-        elif sample_db == 'seattle_weather':
-            self._copy_database_file('seattle_weather.db')
+        if answers.get(AUTH, False):
+            self._copy_pyconfigs_file(c.AUTH_FILE)
+
+        sample_db = answers.get(SAMPLE_DB)
+        if sample_db is not None and sample_db != "none":
+            if sample_db == c.EXPENSES_DB_NAME:
+                self._copy_database_file(c.EXPENSES_DB_NAME+".db")
+            elif sample_db == c.WEATHER_DB_NAME:
+                self._copy_database_file(c.WEATHER_DB_NAME+".db")
+            else:
+                raise NotImplementedError(f"No database found called '{sample_db}'")
     

squirrels/_manifest.py

@@ -1,187 +1,214 @@
-from typing import List, Dict, Any, Optional, Union
-from pathlib import Path
-from sqlalchemy import Engine, create_engine
+from typing import Optional
+from dataclasses import dataclass
+from enum import Enum
 import yaml
 
-from squirrels import _constants as c, _utils
-from squirrels._credentials_manager import Credential, squirrels_config_io
-from squirrels._utils import ConfigurationError, InvalidInputError
+from . import _constants as c, _utils as u
+from ._environcfg import EnvironConfigIO
+from ._timer import timer, time
 
 
-class Manifest:
-    def __init__(self, parms: Dict) -> None:
-        self._parms = parms
-    
+@dataclass
+class ManifestComponentConfig:
     @classmethod
-    def from_yaml_str(cls, parms_str: str):
-        parms = yaml.safe_load(parms_str)
-        return cls(parms)
+    def _validate_required(cls, data: dict, required_keys: list[str], section: str):
+        for key in required_keys:
+            if key not in data:
+                raise u.ConfigurationError(f'In {c.MANIFEST_FILE}, required field missing in {section}: {key}')
 
-    @classmethod
-    def from_file(cls, manifest_path: str):
-        with open(manifest_path, 'r') as f:
-            parms_str = f.read()
+
+@dataclass
+class ProjectVarsConfig(ManifestComponentConfig):
+    data: dict
+
+    def __post_init__(self):
+        required_keys = [c.PROJECT_NAME_KEY, c.MAJOR_VERSION_KEY, c.MINOR_VERSION_KEY]
+        self._validate_required(self.data, required_keys, c.PROJ_VARS_KEY)
         
-        return Manifest.from_yaml_str(parms_str)
+        integer_keys = [c.MAJOR_VERSION_KEY, c.MINOR_VERSION_KEY]
+        for key in integer_keys:
+            if key in self.data and not isinstance(self.data[key], int):
+                raise u.ConfigurationError(f'Project variable "{key}" must be an integer')
     
-    def get_proj_vars(self) -> Dict[str, Any]:
-        return self._parms.get(c.PROJ_VARS_KEY, dict())
+    def get_name(self) -> str:
+        return str(self.data[c.PROJECT_NAME_KEY])
     
-    def get_modules(self) -> List[str]:
-        return self._parms.get(c.MODULES_KEY, list())
+    def get_label(self) -> str:
+        return str(self.data.get(c.PROJECT_LABEL_KEY, self.get_name()))
     
-    def _get_required_field(self, key: str) -> Any:
-        try:
-            return self._parms[key]
-        except KeyError as e:
-            raise ConfigurationError(f'Field "{key}" not found in squirrels.yaml') from e
+    def get_major_version(self) -> int:
+        return self.data[c.MAJOR_VERSION_KEY]
     
-    def get_base_path(self) -> str:
-        project_vars = self.get_proj_vars()
-        try:
-            product = project_vars[c.PRODUCT_KEY]
-            major_version = project_vars[c.MAJOR_VERSION_KEY]
-        except KeyError as e:
-            raise ConfigurationError("Could not construct API endpoint as 'product' and 'major_version'" + 
-                "were not specified in project variables") from e
-        base_path = f"/{product}/v{major_version}"
-        return base_path
-    
-    def get_db_connections(self, test_creds: Dict[str, Credential] = None) -> Dict[str, Engine]:
-        configs: Dict[str, Dict[str, str]] = self._parms.get(c.DB_CONNECTIONS_KEY, {})
-        output = {}
-        for key, config in configs.items():
-            cred_key = config.get(c.DB_CREDENTIALS_KEY)
-            if cred_key is None:
-                cred = Credential("", "")
-            elif test_creds is not None:
-                cred = test_creds[cred_key]
-            else:
-                cred = squirrels_config_io.get_credential(cred_key)
-            url = config[c.URL_KEY].replace("${username}", cred.username).replace("${password}", cred.password)
-            output[key] = create_engine(url)
-        return output
-
-    def _get_dataset_parms(self, dataset: str) -> Dict[str, Any]:
+    def get_minor_version(self) -> int:
+        return self.data[c.MINOR_VERSION_KEY]
+
+
+@dataclass
+class PackageConfig(ManifestComponentConfig):
+    git_url: str
+    directory: str
+    revision: str
+
+    @classmethod
+    def from_dict(cls, kwargs: dict):
+        cls._validate_required(kwargs, [c.PACKAGE_GIT_KEY, c.PACKAGE_REVISION_KEY], c.PACKAGES_KEY)
+        git_url = str(kwargs[c.PACKAGE_GIT_KEY])
+        directory_raw = kwargs.get(c.PACKAGE_DIRECTORY_KEY)
+        directory = git_url.split('/')[-1].removesuffix('.git') if directory_raw is None else str(directory_raw)
+        revision = str(kwargs[c.PACKAGE_REVISION_KEY])
+        return cls(git_url, directory, revision)
+
+
+@dataclass
+class DbConnConfig(ManifestComponentConfig):
+    connection_name: str
+    url: str
+
+    @classmethod
+    def from_dict(cls, kwargs: dict):
+        cls._validate_required(kwargs, [c.DB_CONN_NAME_KEY, c.DB_CONN_URL_KEY], c.DB_CONNECTIONS_KEY)
+        connection_name = str(kwargs[c.DB_CONN_NAME_KEY])
+        credential_key = kwargs.get(c.CREDENTIALS_KEY)
+        username, password = EnvironConfigIO.obj.get_credential(credential_key)
+        url = str(kwargs[c.DB_CONN_URL_KEY]).format(username=username, password=password)
+        return cls(connection_name, url)
+
+
+@dataclass
+class ParametersConfig(ManifestComponentConfig):
+    name: str
+    type: str
+    factory: str
+    arguments: dict
+
+    @classmethod
+    def from_dict(cls, kwargs: dict):
+        all_keys = [c.PARAMETER_NAME_KEY, c.PARAMETER_TYPE_KEY, c.PARAMETER_FACTORY_KEY, c.PARAMETER_ARGS_KEY]
+        cls._validate_required(kwargs, all_keys, c.PARAMETERS_KEY)
+        args = {key: kwargs[key] for key in all_keys}
+        return cls(**args)
+
+
+@dataclass
+class TestSetsConfig(ManifestComponentConfig):
+    name: str
+    user_attributes: dict
+    parameters: dict
+
+    @classmethod
+    def from_dict(cls, kwargs: dict):
+        cls._validate_required(kwargs, [c.TEST_SET_NAME_KEY], c.TEST_SETS_KEY)
+        name = str(kwargs[c.TEST_SET_NAME_KEY])
+        user_attributes = kwargs.get(c.TEST_SET_USER_ATTR_KEY, {})
+        parameters = kwargs.get(c.TEST_SET_PARAMETERS_KEY, {})
+        return cls(name, user_attributes, parameters)
+
+
+@dataclass
+class DbviewConfig(ManifestComponentConfig):
+    name: str
+    connection_name: Optional[str]
+
+    @classmethod
+    def from_dict(cls, kwargs: dict):
+        cls._validate_required(kwargs, [c.DBVIEW_NAME_KEY], c.DBVIEWS_KEY)
+        name = str(kwargs[c.DBVIEW_NAME_KEY])
+        connection_name = str(kwargs.get(c.DBVIEW_CONN_KEY))
+        return cls(name, connection_name)
+
+
+@dataclass
+class FederateConfig(ManifestComponentConfig):
+    name: str
+    materialized: Optional[str]
+
+    @classmethod
+    def from_dict(cls, kwargs: dict):
+        cls._validate_required(kwargs, [c.FEDERATE_NAME_KEY], c.FEDERATES_KEY)
+        name = str(kwargs[c.FEDERATE_NAME_KEY])
+        materialized = str(kwargs.get(c.MATERIALIZED_KEY))
+        return cls(name, materialized)
+
+
+class DatasetScope(Enum):
+    PUBLIC = 0
+    PROTECTED = 1
+    PRIVATE = 2
+
+@dataclass
+class DatasetsConfig(ManifestComponentConfig):
+    name: str
+    label: str
+    model: str
+    scope: DatasetScope
+    parameters: Optional[list[str]]
+    args: dict
+
+    @classmethod
+    def from_dict(cls, kwargs: dict):
+        cls._validate_required(kwargs, [c.DATASET_NAME_KEY], c.DATASETS_KEY)
+        name = str(kwargs[c.DATASET_NAME_KEY])
+        label = str(kwargs.get(c.DATASET_LABEL_KEY, name))
+        model = str(kwargs.get(c.DATASET_MODEL_KEY, name))
+        scope_raw = kwargs.get(c.DATASET_SCOPE_KEY)
         try:
-            return self._get_required_field(c.DATASETS_KEY)[dataset]
+            scope = DatasetScope[str(scope_raw).upper()] if scope_raw is not None else DatasetScope.PUBLIC
         except KeyError as e:
-            raise InvalidInputError(f'No such dataset named "{dataset}" exists') from e
+            scope_list = [scope.name.lower() for scope in DatasetScope]
+            raise u.ConfigurationError(f'Scope not found for dataset "{name}". Scope must be one of {scope_list}') from e
         
-    def _get_required_field_from_dataset_parms(self, dataset: str, key: str):
+        parameters = kwargs.get(c.DATASET_PARAMETERS_KEY)
+        args = kwargs.get(c.DATASET_ARGS_KEY, {})
+        return cls(name, label, model, scope, parameters, args)
+
+
+@dataclass
+class _ManifestConfig:
+    project_variables: ProjectVarsConfig
+    packages: list[PackageConfig]
+    db_connections: list[DbConnConfig]
+    parameters: list[ParametersConfig]
+    selection_test_sets: dict[str, TestSetsConfig]
+    dbviews: dict[str, DbviewConfig]
+    federates: dict[str, FederateConfig]
+    datasets: dict[str, DatasetsConfig]
+    settings: dict
+
+    @classmethod
+    def from_dict(cls, kwargs: dict):
+        settings: dict = kwargs.get(c.SETTINGS_KEY, {})
+
         try:
-            return self._get_dataset_parms(dataset)[key]
+            proj_vars = ProjectVarsConfig(kwargs[c.PROJ_VARS_KEY])
         except KeyError as e:
-            raise ConfigurationError(f'The "{key}" field is not defined for dataset "{dataset}"') from e
-    
-    def _get_all_database_view_parms(self, dataset: str) -> Dict[str, Dict[str, str]]:
-        return self._get_required_field_from_dataset_parms(dataset, c.DATABASE_VIEWS_KEY)
-    
-    def get_all_dataset_names(self) -> str:
-        datasets: Dict[str, Any] = self._get_required_field(c.DATASETS_KEY)
-        return list(datasets.keys())
-    
-    def get_dataset_folder(self, dataset: str) -> Path:
-        return _utils.join_paths(c.DATASETS_FOLDER, dataset)
-        
-    def get_dataset_args(self, dataset: str) -> Dict[str, Any]:
-        dataset_args = self._get_dataset_parms(dataset).get("args", {})
-        full_args = {**self.get_proj_vars(), **dataset_args}
-        return full_args
-    
-    def get_all_database_view_names(self, dataset: str) -> List[str]:
-        all_database_views = self._get_all_database_view_parms(dataset)
-        return list(all_database_views.keys())
-    
-    def get_database_view_file(self, dataset: str, database_view: str) -> Path:
-        database_view_parms = self._get_all_database_view_parms(dataset)[database_view]
-        if isinstance(database_view_parms, str):
-            db_view_file = database_view_parms
-        else:
-            try:
-                db_view_file = database_view_parms[c.FILE_KEY]
-            except KeyError as e:
-                raise ConfigurationError(f'The "{c.FILE_KEY}" field is not defined for "{database_view}" in dataset "{dataset}"') from e
-        dataset_folder = self.get_dataset_folder(dataset)
-        return _utils.join_paths(dataset_folder, db_view_file)
-    
-    def get_view_args(self, dataset: str, database_view: str = None) -> Dict[str, Any]:
-        dataset_args = self.get_dataset_args(dataset)
-        if database_view is None:
-            view_parms: Dict[str, Any] = self._get_required_field_from_dataset_parms(dataset, c.FINAL_VIEW_KEY)
-        else:
-            view_parms: Dict[str, Any] = self._get_all_database_view_parms(dataset)[database_view]
-        view_args: Dict[str, Any] = {} if isinstance(view_parms, str) else view_parms.get("args", {})
-        full_args = {**dataset_args, **view_args}
-        return full_args
-
-    def get_database_view_db_connection(self, dataset: str, database_view: str) -> Optional[str]:
-        database_view_parms = self._get_all_database_view_parms(dataset)[database_view]
-        if isinstance(database_view_parms, str):
-            db_connection = c.DEFAULT_DB_CONN 
-        else: 
-            db_connection = database_view_parms.get(c.DB_CONNECTION_KEY, c.DEFAULT_DB_CONN)
-        return db_connection
-    
-    def get_dataset_label(self, dataset: str) -> str:
-        return self._get_required_field_from_dataset_parms(dataset, c.DATASET_LABEL_KEY)
-    
-    def get_dataset_final_view_file(self, dataset: str) -> Union[str, Path]:
-        final_view_parms: Dict[str, Any] = self._get_required_field_from_dataset_parms(dataset, c.FINAL_VIEW_KEY)
-        if isinstance(final_view_parms, str):
-            final_view_file = final_view_parms
-        else:
-            try:
-                final_view_file = final_view_parms[c.FILE_KEY]
-            except KeyError as e:
-                raise ConfigurationError(f'The "{c.FILE_KEY}" field is not defined for the final view') from e
+            raise u.ConfigurationError(f'In {c.MANIFEST_FILE}, section for {c.PROJ_VARS_KEY} is required') from e
         
-        database_views = self.get_all_database_view_names(dataset)
-        if final_view_file in database_views:
-            return final_view_file
-        else:
-            dataset_path = self.get_dataset_folder(dataset)
-            return _utils.join_paths(dataset_path, final_view_file)
-
-    def get_setting(self, key: str, default: Any) -> Any:
-        settings: Dict[str, Any] = self._parms.get(c.SETTINGS_KEY, dict())
-        return settings.get(key, default)
-    
-    def get_catalog(self, parameters_path: str, results_path: str) -> Any:
-        """
-        Gets the component of the catalog API response that's generated by this manifest
+        packages = [PackageConfig.from_dict(x) for x in kwargs.get(c.PACKAGES_KEY, [])]
+        db_conns = [DbConnConfig.from_dict(x) for x in kwargs.get(c.DB_CONNECTIONS_KEY, [])]
+        params = [ParametersConfig.from_dict(x) for x in kwargs.get(c.PARAMETERS_KEY, [])]
 
-        Parameters:
-            parameters_path: The path to the parameters API endpoint
-            results_path: The path to the results API endpoint
-        
-        Returns:
-            A JSON response for the catalog API
-        """
-        datasets_info = []
-        for dataset in self.get_all_dataset_names():
-            dataset_normalized = _utils.normalize_name_for_api(dataset)
-            datasets_info.append({
-                'name': dataset,
-                'label': self.get_dataset_label(dataset),
-                'parameters_path': parameters_path.format(dataset=dataset_normalized),
-                'result_path': results_path.format(dataset=dataset_normalized),
-                'first_minor_version': 0
-            })
+        test_sets = {x[c.TEST_SET_NAME_KEY]: TestSetsConfig.from_dict(x) for x in kwargs.get(c.TEST_SETS_KEY, [])}
+        default_test_set: str = settings.get(c.TEST_SET_DEFAULT_USED_SETTING, c.DEFAULT_TEST_SET_NAME)
+        test_sets.setdefault(default_test_set, TestSetsConfig.from_dict({c.TEST_SET_NAME_KEY: default_test_set}))
+
+        dbviews = {x[c.DBVIEW_NAME_KEY]: DbviewConfig.from_dict(x) for x in kwargs.get(c.DBVIEWS_KEY, [])}
+        federates = {x[c.FEDERATE_NAME_KEY]: FederateConfig.from_dict(x) for x in kwargs.get(c.FEDERATES_KEY, [])}
+        datasets = {x[c.DATASET_NAME_KEY]: DatasetsConfig.from_dict(x) for x in kwargs.get(c.DATASETS_KEY, [])}
+
+        return cls(proj_vars, packages, db_conns, params, test_sets, dbviews, federates, datasets, settings)
+
+
+class ManifestIO:
+    obj: _ManifestConfig
+
+    @classmethod
+    def LoadFromFile(cls) -> None:
+        EnvironConfigIO.LoadFromFile()
         
-        project_vars = self.get_proj_vars()
-        return {
-            'response_version': 0,
-            'products': [{
-                'name': project_vars[c.PRODUCT_KEY],
-                'versions': [{
-                    'major_version': project_vars[c.MAJOR_VERSION_KEY],
-                    'latest_minor_version': project_vars[c.MINOR_VERSION_KEY],
-                    'datasets': datasets_info
-                }]
-            }]
-        }
-
-
-def _from_file():
-    return Manifest.from_file(c.MANIFEST_FILE)
+        start = time.time()
+        raw_content = u.read_file(c.MANIFEST_FILE)
+        env_config = EnvironConfigIO.obj.get_all_env_vars()
+        content = u.render_string(raw_content, env_config)
+        proj_config = yaml.safe_load(content)
+        cls.obj = _ManifestConfig.from_dict(proj_config)
+        timer.add_activity_time(f"loading {c.MANIFEST_FILE} file", start)