squirrels 0.4.0__py3-none-any.whl → 0.5.0rc0__py3-none-any.whl

squirrels/_initializer.py

@@ -1,14 +1,17 @@
 from typing import Optional
 from datetime import datetime
-import inquirer, os, shutil
+import inquirer, os, shutil, secrets
 
 from . import _constants as c, _utils as u
 
 base_proj_dir = u.Path(os.path.dirname(__file__), c.PACKAGE_DATA_FOLDER, c.BASE_PROJECT_FOLDER)
 
+TMP_FOLDER = "tmp"
+
 
 class Initializer:
-    def __init__(self, *, overwrite: bool = False):
+    def __init__(self, *, project_name: Optional[str] = None, overwrite: bool = False):
+        self.project_name = project_name
         self.overwrite = overwrite
 
     def _path_exists(self, filepath: u.Path) -> bool:
@@ -26,27 +29,37 @@ class Initializer:
     def _copy_file(self, filepath: u.Path, *, src_folder: str = ""):
         src_path = u.Path(base_proj_dir, src_folder, filepath)
         
-        dest_dir = os.path.dirname(filepath)
+        filepath2 = u.Path(self.project_name, filepath) if self.project_name else filepath
+        dest_dir = os.path.dirname(filepath2)
         if dest_dir != "":
             os.makedirs(dest_dir, exist_ok=True)
         
         perform_copy = True
-        if self._path_exists(filepath):
-            old_filepath = filepath
-            if self._files_have_same_content(src_path, filepath):
+        if self._path_exists(filepath2):
+            old_filepath = filepath2
+            if self._files_have_same_content(src_path, filepath2):
                 perform_copy = False
                 extra_msg = "Skipping... file contents is same as source"
             elif self.overwrite:
                 extra_msg = "Overwriting file..."
             else:
-                filepath = self._add_timestamp_to_filename(old_filepath)
-                extra_msg = f'Creating file as "{filepath}" instead...'
+                filepath2 = self._add_timestamp_to_filename(old_filepath)
+                extra_msg = f'Creating file as "{filepath2}" instead...'
             print(f'File "{old_filepath}" already exists.', extra_msg)
         else:
-            print(f'Creating file "{filepath}"...')
+            print(f'Creating file "{filepath2}"...')
         
         if perform_copy:
-            shutil.copy(src_path, filepath)
+            shutil.copy(src_path, filepath2)
+
+    def _copy_macros_file(self, filepath: str):
+        self._copy_file(u.Path(c.MACROS_FOLDER, filepath))
+
+    def _copy_models_file(self, filepath: str):
+        self._copy_file(u.Path(c.MODELS_FOLDER, filepath))
+
+    def _copy_build_file(self, filepath: str):
+        self._copy_file(u.Path(c.MODELS_FOLDER, c.BUILDS_FOLDER, filepath))
 
     def _copy_dbview_file(self, filepath: str):
         self._copy_file(u.Path(c.MODELS_FOLDER, c.DBVIEWS_FOLDER, filepath))
@@ -66,34 +79,54 @@ class Initializer:
     def _copy_dashboard_file(self, filepath: str):
         self._copy_file(u.Path(c.DASHBOARDS_FOLDER, filepath))
 
-    def _create_manifest_file(self, has_connections: bool, has_parameters: bool, has_dashboards: bool):
-        TMP_FOLDER = "tmp"
-
+    def _create_manifest_file(self, has_connections: bool, has_parameters: bool):
         def get_content(file_name: Optional[str]) -> str:
             if file_name is None:
                 return ""
             
             yaml_path = u.Path(base_proj_dir, file_name)
-            return u.read_file(yaml_path)
+            return yaml_path.read_text()
         
         file_name_dict = {
             "parameters": c.PARAMETERS_YML_FILE if has_parameters else None, 
             "connections": c.CONNECTIONS_YML_FILE if has_connections else None,
-            "dashboards": c.DASHBOARDS_YML_FILE if has_dashboards else None
         }
         substitutions = {key: get_content(val) for key, val in file_name_dict.items()}
         
         manifest_template = get_content(c.MANIFEST_JINJA_FILE)
         manifest_content = u.render_string(manifest_template, **substitutions)
         output_path = u.Path(base_proj_dir, TMP_FOLDER, c.MANIFEST_FILE)
-        with open(u.Path(output_path), "w") as f:
-            f.write(manifest_content)
+        output_path.write_text(manifest_content)
         
         self._copy_file(u.Path(c.MANIFEST_FILE), src_folder=TMP_FOLDER)
+    
+    def _copy_dotenv_files(self, admin_password: str | None = None):
+        substitutions = {
+            "random_secret_key": secrets.token_hex(32),
+            "random_admin_password": admin_password if admin_password else secrets.token_urlsafe(8),
+        }
+
+        dotenv_path = u.Path(base_proj_dir, c.DOTENV_FILE)
+        contents = u.render_string(dotenv_path.read_text(), **substitutions)
+
+        output_path = u.Path(base_proj_dir, TMP_FOLDER, c.DOTENV_FILE)
+        output_path.write_text(contents)
+
+        self._copy_file(u.Path(c.DOTENV_FILE), src_folder=TMP_FOLDER)
+        self._copy_file(u.Path(c.DOTENV_FILE + ".example"))
 
     def init_project(self, args):
-        options = ["core", "connections", "parameters", "dbview", "federate", "dashboard", "auth"]
-        _, CONNECTIONS, PARAMETERS, DBVIEW, FEDERATE, DASHBOARD, AUTH = options
+        options = ["core", "connections", "parameters", "build", "federate", "dashboard"]
+        _, CONNECTIONS, PARAMETERS, BUILD, FEDERATE, DASHBOARD = options
+
+        # Add project name prompt if not provided
+        if self.project_name is None:
+            questions = [
+                inquirer.Text('project_name', message="What is your project name? (leave blank to create in current directory)")
+            ]
+            answers = inquirer.prompt(questions)
+            assert isinstance(answers, dict)
+            self.project_name = answers['project_name']
 
         answers = { x: getattr(args, x) for x in options }
         if not any(answers.values()):
@@ -105,7 +138,7 @@ class Initializer:
                     PARAMETERS, message=f"How would you like to configure the parameters?", choices=c.CONF_FORMAT_CHOICES2
                 ),
                 inquirer.List(
-                    DBVIEW, message="What's the file format for the database view model?", choices=c.FILE_TYPE_CHOICES
+                    BUILD, message="What's the file format for the build model?", choices=c.FILE_TYPE_CHOICES
                 ),
                 inquirer.List(
                     FEDERATE, message="What's the file format for the federated model?", choices=c.FILE_TYPE_CHOICES
@@ -113,8 +146,8 @@ class Initializer:
                 inquirer.Confirm(
                     DASHBOARD, message=f"Do you want to include a dashboard example?", default=False
                 ),
-                inquirer.Confirm(
-                    AUTH, message=f"Do you want to add the '{c.AUTH_FILE}' file to enable custom API authentication?", default=False
+                inquirer.Password(
+                    "admin_password", message="What's the admin password? (leave blank to generate a random one)"
                 ),
             ]
             answers = inquirer.prompt(questions)
@@ -133,6 +166,8 @@ class Initializer:
             answer = answers.get(key)
             return answer if answer is not None else default
         
+        admin_password = get_answer("admin_password", None)
+        
         connections_format = get_answer(CONNECTIONS, c.YML_FORMAT)
         connections_use_yaml = (connections_format == c.YML_FORMAT)
         connections_use_py = (connections_format == c.PYTHON_FORMAT)
@@ -141,14 +176,19 @@ class Initializer:
         parameters_use_yaml = (parameters_format == c.YML_FORMAT)
         parameters_use_py = (parameters_format == c.PYTHON_FORMAT)
 
-        dbview_format = get_answer(DBVIEW, c.SQL_FILE_TYPE)
-        if dbview_format == c.SQL_FILE_TYPE:
-            db_view_file = c.DBVIEW_FILE_STEM + ".sql"
-        elif dbview_format == c.PYTHON_FILE_TYPE:
-            db_view_file = c.DBVIEW_FILE_STEM + ".py"
+        build_config_file = c.BUILD_FILE_STEM + ".yml"
+        build_format = get_answer(BUILD, c.PYTHON_FILE_TYPE)
+        if build_format == c.SQL_FILE_TYPE:
+            build_file = c.BUILD_FILE_STEM + ".sql"
+        elif build_format == c.PYTHON_FILE_TYPE:
+            build_file = c.BUILD_FILE_STEM + ".py"
         else:
-            raise NotImplementedError(f"Dbview model format '{dbview_format}' not supported")
+            raise NotImplementedError(f"Build model format '{build_format}' not supported")
+
+        db_view_config_file = c.DBVIEW_FILE_STEM + ".yml"
+        db_view_file = c.DBVIEW_FILE_STEM + ".sql"
     
+        federate_config_file = c.FEDERATE_FILE_STEM + ".yml"
         federate_format = get_answer(FEDERATE, c.SQL_FILE_TYPE)
         if federate_format == c.SQL_FILE_TYPE:
             federate_file = c.FEDERATE_FILE_STEM + ".sql"
@@ -159,10 +199,10 @@ class Initializer:
 
         dashboards_enabled = get_answer(DASHBOARD, False)
 
-        self._create_manifest_file(connections_use_yaml, parameters_use_yaml, dashboards_enabled)
+        self._copy_dotenv_files(admin_password)
+        self._create_manifest_file(connections_use_yaml, parameters_use_yaml)
         
-        self._copy_file(u.Path(".gitignore"))
-        self._copy_file(u.Path(c.ENV_CONFIG_FILE))
+        self._copy_file(u.Path(c.GITIGNORE_FILE))
         
         if connections_use_py:
             self._copy_pyconfig_file(c.CONNECTIONS_FILE)
@@ -178,44 +218,77 @@ class Initializer:
         else:
             raise NotImplementedError(f"Format '{parameters_format}' not supported for configuring widget parameters")
         
+        self._copy_pyconfig_file(c.USER_FILE)
+
         self._copy_pyconfig_file(c.CONTEXT_FILE)
-        self._copy_seed_file(c.CATEGORY_SEED_FILE)
-        self._copy_seed_file(c.SUBCATEGORY_SEED_FILE)
+        self._copy_seed_file(c.SEED_CATEGORY_FILE_STEM + ".csv")
+        self._copy_seed_file(c.SEED_CATEGORY_FILE_STEM + ".yml")
+        self._copy_seed_file(c.SEED_SUBCATEGORY_FILE_STEM + ".csv")
+        self._copy_seed_file(c.SEED_SUBCATEGORY_FILE_STEM + ".yml")
+
+        self._copy_macros_file(c.MACROS_FILE)
 
+        self._copy_models_file(c.SOURCES_FILE)
+        self._copy_build_file(build_file)
+        self._copy_build_file(build_config_file)
         self._copy_dbview_file(db_view_file)
+        self._copy_dbview_file(db_view_config_file)
         self._copy_federate_file(federate_file)
+        self._copy_federate_file(federate_config_file)
         
         if dashboards_enabled:
             self._copy_dashboard_file(c.DASHBOARD_FILE_STEM + ".py")
+            self._copy_dashboard_file(c.DASHBOARD_FILE_STEM + ".yml")
         
-        if get_answer(AUTH, False):
-            self._copy_pyconfig_file(c.AUTH_FILE)
-
         self._copy_database_file(c.EXPENSES_DB)
         
         print(f"\nSuccessfully created new Squirrels project in current directory!\n")
     
     def get_file(self, args):
-        if args.file_name == c.ENV_CONFIG_FILE:
-            self._copy_file(u.Path(c.ENV_CONFIG_FILE))
-            print("PLEASE ENSURE THE FILE IS INCLUDED IN .gitignore")
+        if args.file_name == c.DOTENV_FILE:
+            self._copy_dotenv_files()
+            print(f"A random admin password was generated for your project. You can change it in the new {c.DOTENV_FILE} file.")
+            print()
+            print(f"IMPORTANT: Please ensure the {c.DOTENV_FILE} file is added to your {c.GITIGNORE_FILE} file.")
+            print(f"You may also run `sqrl get-file {c.GITIGNORE_FILE}` to add a sample {c.GITIGNORE_FILE} file to your project.")
+            print()
+        elif args.file_name == c.GITIGNORE_FILE:
+            self._copy_file(u.Path(c.GITIGNORE_FILE))
         elif args.file_name == c.MANIFEST_FILE:
-            self._create_manifest_file(not args.no_connections, args.parameters, args.dashboards)
-        elif args.file_name in (c.AUTH_FILE, c.CONNECTIONS_FILE, c.PARAMETERS_FILE, c.CONTEXT_FILE):
+            self._create_manifest_file(not args.no_connections, args.parameters)
+        elif args.file_name in (c.USER_FILE, c.CONNECTIONS_FILE, c.PARAMETERS_FILE, c.CONTEXT_FILE):
             self._copy_pyconfig_file(args.file_name)
-        elif args.file_name in (c.DBVIEW_FILE_STEM, c.FEDERATE_FILE_STEM):
-            if args.format == c.SQL_FILE_TYPE:
+        elif args.file_name == c.MACROS_FILE:
+            self._copy_macros_file(args.file_name)
+        elif args.file_name == c.SOURCES_FILE:
+            self._copy_models_file(args.file_name)
+        elif args.file_name in (c.BUILD_FILE_STEM, c.DBVIEW_FILE_STEM, c.FEDERATE_FILE_STEM):
+            if args.file_name == c.DBVIEW_FILE_STEM or args.format == c.SQL_FILE_TYPE:
                 extension = ".sql"
             elif args.format == c.PYTHON_FILE_TYPE:
                 extension = ".py"
             else:
                 raise NotImplementedError(f"Format '{args.format}' not supported for {args.file_name}")
-            copy_method = self._copy_dbview_file if args.file_name == c.DBVIEW_FILE_STEM else self._copy_federate_file
+            
+            if args.file_name == c.BUILD_FILE_STEM:
+                copy_method = self._copy_build_file
+            elif args.file_name == c.DBVIEW_FILE_STEM:
+                copy_method = self._copy_dbview_file
+            elif args.file_name == c.FEDERATE_FILE_STEM:
+                copy_method = self._copy_federate_file
+            else:
+                raise NotImplementedError(f"File '{args.file_name}' not supported")
+            
             copy_method(args.file_name + extension)
+            copy_method(args.file_name + ".yml")
         elif args.file_name == c.DASHBOARD_FILE_STEM:
             self._copy_dashboard_file(args.file_name + ".py")
+            self._copy_dashboard_file(args.file_name + ".yml")
         elif args.file_name in (c.EXPENSES_DB, c.WEATHER_DB):
             self._copy_database_file(args.file_name)
+        elif args.file_name in (c.SEED_CATEGORY_FILE_STEM, c.SEED_SUBCATEGORY_FILE_STEM):
+            self._copy_seed_file(args.file_name + ".csv")
+            self._copy_seed_file(args.file_name + ".yml")
         else:
             raise NotImplementedError(f"File '{args.file_name}' not supported")
         

squirrels/_manifest.py

@@ -1,16 +1,19 @@
+from functools import cached_property
 from typing import Any
+from urllib.parse import urlparse
+from sqlalchemy import Engine, create_engine
 from typing_extensions import Self
 from enum import Enum
 from pydantic import BaseModel, Field, field_validator, model_validator, ValidationInfo, ValidationError
 import yaml, time
 
-from . import _constants as c, _utils as _u
-from ._environcfg import EnvironConfig
+from . import _constants as c, _utils as u
 
 
 class ProjectVarsConfig(BaseModel, extra="allow"):
     name: str
     label: str = ""
+    description: str = ""
     major_version: int
 
     @model_validator(mode="after")
@@ -36,13 +39,78 @@ class _ConfigWithNameBaseModel(BaseModel):
     name: str
 
 
-class DbConnConfig(_ConfigWithNameBaseModel):
-    credential: str | None = None
-    url: str
+class ConnectionType(Enum):
+    SQLALCHEMY = "sqlalchemy"
+    CONNECTORX = "connectorx"
+    ADBC = "adbc"
+
+
+class ConnectionProperties(BaseModel):
+    """
+    A class for holding the properties of a connection
+
+    Arguments:
+        type: The type of connection, one of "sqlalchemy", "connectorx", or "adbc"
+        uri: The URI for the connection
+    """
+    label: str | None = None
+    type: ConnectionType = Field(default=ConnectionType.SQLALCHEMY)
+    uri: str
+    sa_create_engine_args: dict[str, Any] = Field(default_factory=dict)
+
+    @cached_property
+    def engine(self) -> Engine:
+        """
+        Creates and caches a SQLAlchemy engine if the connection type is sqlalchemy.
+        Returns None for other connection types.
+        """
+        if self.type == ConnectionType.SQLALCHEMY:
+            return create_engine(self.uri, **self.sa_create_engine_args)
+        else:
+            raise ValueError(f'Connection type "{self.type}" does not support engine property')
 
-    def finalize_url(self, base_path: str, env_cfg: EnvironConfig) -> Self:
-        username, password = env_cfg.get_credential(self.credential)
-        self.url = self.url.format(username=username, password=password, project_path=base_path)
+    @cached_property
+    def dialect(self) -> str:
+        if self.type == ConnectionType.SQLALCHEMY:
+            dialect = self.engine.dialect.name
+        else:
+            url = urlparse(self.uri)
+            dialect = url.scheme
+        
+        processed_dialect = next((d for d in ['sqlite', 'postgres', 'mysql'] if dialect.lower().startswith(d)), None)
+        dialect = processed_dialect if processed_dialect is not None else dialect
+        return dialect
+    
+    @cached_property
+    def attach_uri_for_duckdb(self) -> str | None:
+        if self.type == ConnectionType.SQLALCHEMY:
+            url = self.engine.url
+            host = url.host
+            port = url.port
+            username = url.username
+            password = url.password
+            database = url.database
+            sqlite_database = database if database is not None else ""
+        else:
+            url = urlparse(self.uri)
+            host = url.hostname
+            port = url.port
+            username = url.username
+            password = url.password
+            database = url.path.lstrip('/')
+            sqlite_database = self.uri.replace(f"{self.dialect}://", "")
+        
+        if self.dialect == 'sqlite':
+            return sqlite_database
+        elif self.dialect in ('postgres', 'mysql'):
+            return f"dbname={database} user={username} password={password} host={host} port={port}"
+        else:
+            return None
+
+
+class DbConnConfig(ConnectionProperties, _ConfigWithNameBaseModel):
+    def finalize_uri(self, base_path: str) -> Self:
+        self.uri = self.uri.format(project_path=base_path)
         return self
 
 
@@ -52,15 +120,7 @@ class ParametersConfig(BaseModel):
     arguments: dict[str, Any]
 
 
-class DbviewConfig(_ConfigWithNameBaseModel):
-    connection_name: str | None = None
-
-
-class FederateConfig(_ConfigWithNameBaseModel):
-    materialized: str | None = None
-
-
-class DatasetScope(Enum):
+class PermissionScope(Enum):
     PUBLIC = 0
     PROTECTED = 1
     PRIVATE = 2
@@ -69,8 +129,8 @@ class DatasetScope(Enum):
 class AnalyticsOutputConfig(_ConfigWithNameBaseModel):
     label: str = ""
     description: str = ""
-    scope: DatasetScope = DatasetScope.PUBLIC
-    parameters: list[str] = Field(default_factory=list)
+    scope: PermissionScope = PermissionScope.PUBLIC
+    parameters: list[str] | None = Field(default=None, description="The list of parameter names used by the dataset/dashboard")
 
     @model_validator(mode="after")
     def finalize_label(self) -> Self:
@@ -80,15 +140,19 @@ class AnalyticsOutputConfig(_ConfigWithNameBaseModel):
 
     @field_validator("scope", mode="before")
     @classmethod
-    def validate_scope(cls, value: str, info: ValidationInfo) -> DatasetScope:
+    def validate_scope(cls, value: str, info: ValidationInfo) -> PermissionScope:
         try:
-            return DatasetScope[str(value).upper()]
+            return PermissionScope[str(value).upper()]
         except KeyError as e:
             name = info.data.get("name")
-            scope_list = [scope.name.lower() for scope in DatasetScope]
+            scope_list = [scope.name.lower() for scope in PermissionScope]
             raise ValueError(f'Scope "{value}" is invalid for dataset/dashboard "{name}". Scope must be one of {scope_list}') from e
 
 
+class DatasetTraitConfig(_ConfigWithNameBaseModel):
+    default: Any 
+
+
 class DatasetConfig(AnalyticsOutputConfig):
     model: str = ""
     traits: dict = Field(default_factory=dict)
@@ -104,47 +168,26 @@ class DatasetConfig(AnalyticsOutputConfig):
         return self
 
 
-class DashboardConfig(AnalyticsOutputConfig):
-    def __hash__(self) -> int:
-        return hash("dashboard_"+self.name)
-
-
 class TestSetsConfig(_ConfigWithNameBaseModel):
     datasets: list[str] | None = None
-    is_authenticated: bool = False
-    user_attributes: dict[str, Any] = Field(default_factory=dict)
+    user_attributes: dict[str, Any] | None = None
     parameters: dict[str, Any] = Field(default_factory=dict)
 
-    @model_validator(mode="after")
-    def finalize_is_authenticated(self) -> Self:
-        if len(self.user_attributes) > 0:
-            self.is_authenticated = True
-        return self
-
-
-class Settings(BaseModel):
-    data: dict[str, Any]
-    
-    def get_default_connection_name(self) -> str:
-        return self.data.get(c.DB_CONN_DEFAULT_USED_SETTING, c.DEFAULT_DB_CONN)
-    
-    def do_use_duckdb(self) -> bool:
-        return self.data.get(c.IN_MEMORY_DB_SETTING, c.SQLITE) == c.DUCKDB
+    @property
+    def is_authenticated(self) -> bool:
+        return self.user_attributes is not None
 
 
 class ManifestConfig(BaseModel):
-    env_cfg: EnvironConfig
     project_variables: ProjectVarsConfig
     packages: list[PackageConfig] = Field(default_factory=list)
     connections: dict[str, DbConnConfig] = Field(default_factory=dict)
     parameters: list[ParametersConfig] = Field(default_factory=list)
     selection_test_sets: dict[str, TestSetsConfig] = Field(default_factory=dict)
-    dbviews: dict[str, DbviewConfig] = Field(default_factory=dict)
-    federates: dict[str, FederateConfig] = Field(default_factory=dict)
+    dataset_traits: dict[str, DatasetTraitConfig] = Field(default_factory=dict)
     datasets: dict[str, DatasetConfig] = Field(default_factory=dict)
-    dashboards: dict[str, DashboardConfig] = Field(default_factory=dict)
-    settings: dict[str, Any] = Field(default_factory=dict)
     base_path: str = "."
+    env_vars: dict[str, str] = Field(default_factory=dict)
 
     @field_validator("packages")
     @classmethod
@@ -156,7 +199,7 @@ class ManifestConfig(BaseModel):
             set_of_directories.add(package.directory)
         return packages
     
-    @field_validator("connections", "selection_test_sets", "dbviews", "federates", "datasets", "dashboards", mode="before")
+    @field_validator("connections", "selection_test_sets", "dataset_traits", "datasets", mode="before")
     @classmethod
     def names_are_unique(cls, values: list[dict] | dict[str, dict], info: ValidationInfo) -> dict[str, dict]:
         if isinstance(values, list):
@@ -173,19 +216,33 @@ class ManifestConfig(BaseModel):
     @model_validator(mode="after")
     def finalize_connections(self) -> Self:
         for conn in self.connections.values():
-            conn.finalize_url(self.base_path, self.env_cfg)
+            conn.finalize_uri(self.base_path)
+        return self
+    
+    @model_validator(mode="after")
+    def validate_dataset_traits(self) -> Self:
+        for dataset_name, dataset in self.datasets.items():
+            # Validate that all trait keys in dataset.traits exist in dataset_traits
+            for trait_key in dataset.traits.keys():
+                if trait_key not in self.dataset_traits:
+                    raise ValueError(
+                        f'Dataset "{dataset_name}" references undefined trait "{trait_key}". '
+                        f'Traits must be defined with a default value in the dataset_traits section.'
+                    )
+            
+            # Set default values for any traits that are missing
+            for trait_name, trait_config in self.dataset_traits.items():
+                if trait_name not in dataset.traits:
+                    dataset.traits[trait_name] = trait_config.default
+                    
         return self
     
-    @property
-    def settings_obj(self) -> Settings:
-        return Settings(data=self.settings)
-
     def get_default_test_set(self, dataset_name: str) -> TestSetsConfig:
         """
         Raises KeyError if dataset name doesn't exist
         """
         default_name_1 = self.datasets[dataset_name].default_test_set
-        default_name_2 = self.settings.get(c.TEST_SET_DEFAULT_USED_SETTING, c.DEFAULT_TEST_SET_NAME)
+        default_name_2 = self.env_vars.get(c.SQRL_TEST_SETS_DEFAULT_NAME_USED, "default")
         default_name = default_name_1 if default_name_1 else default_name_2
         default_test_set = self.selection_test_sets.get(default_name, TestSetsConfig(name=default_name))
         return default_test_set
@@ -196,22 +253,27 @@ class ManifestConfig(BaseModel):
             if test_set_config.datasets is None or dataset in test_set_config.datasets:
                 applicable_test_sets.append(test_set_name)
         return applicable_test_sets
+    
+    def get_default_traits(self) -> dict[str, Any]:
+        default_traits = {}
+        for trait_name, trait_config in self.dataset_traits.items():
+            default_traits[trait_name] = trait_config.default
+        return default_traits
 
 
 class ManifestIO:
 
     @classmethod
-    def load_from_file(cls, logger: _u.Logger, base_path: str, env_cfg: EnvironConfig) -> ManifestConfig:
+    def load_from_file(cls, logger: u.Logger, base_path: str, env_vars: dict[str, str]) -> ManifestConfig:
         start = time.time()
 
-        raw_content = _u.read_file(_u.Path(base_path, c.MANIFEST_FILE))
-        env_vars = env_cfg.get_all_env_vars()
-        content = _u.render_string(raw_content, base_path=base_path, env_vars=env_vars)
+        raw_content = u.read_file(u.Path(base_path, c.MANIFEST_FILE))
+        content = u.render_string(raw_content, base_path=base_path, env_vars=env_vars)
         manifest_content = yaml.safe_load(content)
         try:
-            manifest_cfg = ManifestConfig(base_path=base_path, env_cfg=env_cfg, **manifest_content)
+            manifest_cfg = ManifestConfig(base_path=base_path, **manifest_content)
         except ValidationError as e:
-            raise _u.ConfigurationError(f"Failed to process {c.MANIFEST_FILE} file. " + str(e)) from e
+            raise u.ConfigurationError(f"Failed to process {c.MANIFEST_FILE} file. " + str(e)) from e
         
         logger.log_activity_time(f"loading {c.MANIFEST_FILE} file", start)
         return manifest_cfg