squirrels 0.2.2__py3-none-any.whl → 0.3.1__py3-none-any.whl

squirrels/_constants.py

@@ -39,6 +39,7 @@ PARAMETER_ARGS_KEY = 'arguments'
 TEST_SETS_KEY = 'selection_test_sets'
 TEST_SET_NAME_KEY = 'name'
 DEFAULT_TEST_SET_NAME = 'default'
+TEST_SET_DATASETS_KEY = 'datasets'
 TEST_SET_USER_ATTR_KEY = 'user_attributes'
 TEST_SET_PARAMETERS_KEY = 'parameters'
 
@@ -48,6 +49,7 @@ DATASET_LABEL_KEY = 'label'
 DATASET_MODEL_KEY = 'model'
 DATASET_PARAMETERS_KEY = 'parameters'
 DATASET_TRAITS_KEY = 'traits'
+DATASET_DEFAULT_TEST_SET_KEY = 'default_test_set'
 
 DATASET_SCOPE_KEY = 'scope'
 PUBLIC_SCOPE = 'public'
@@ -77,13 +79,14 @@ ASSETS_FOLDER = 'assets'
 TEMPLATES_FOLDER = 'templates'
 
 ENVIRON_CONFIG_FILE = 'environcfg.yml'
+ENV_CONFIG_FILE = 'env.yml'
 MANIFEST_JINJA_FILE = 'squirrels.yml.j2'
 CONNECTIONS_YML_FILE = 'connections.yml'
 PARAMETERS_YML_FILE = 'parameters.yml'
 MANIFEST_FILE = 'squirrels.yml'
 LU_DATA_FILE = 'lu_data.xlsx'
 
-DATABASE_FOLDER = 'database'
+DATABASE_FOLDER = 'assets'
 PACKAGES_FOLDER = 'sqrl_packages'
 
 MODELS_FOLDER = 'models'
@@ -94,7 +97,7 @@ FEDERATES_FOLDER = 'federates'
 FEDERATE_SQL_NAME = 'dataset_example.sql'
 FEDERATE_PY_NAME = 'dataset_example.py'
 
-PYCONFIG_FOLDER = 'pyconfigs'
+PYCONFIGS_FOLDER = 'pyconfigs'
 AUTH_FILE = 'auth.py'
 CONNECTIONS_FILE = 'connections.py'
 CONTEXT_FILE = 'context.py'
@@ -103,9 +106,9 @@ PARAMETERS_FILE = 'parameters.py'
 TARGET_FOLDER = 'target'
 COMPILE_FOLDER = 'compile'
 
-OUTPUTS_FOLDER = 'outputs'
-PARAMETERS_OUTPUT = 'parameters.json'
-FINAL_VIEW_OUT_STEM = 'final_view'
+SEEDS_FOLDER = 'seeds'
+CATEGORY_SEED_FILE = 'seed_categories.csv'
+SUBCATEGORY_SEED_FILE = 'seed_subcategories.csv'
 
 # Dataset setting names
 AUTH_TOKEN_EXPIRE_SETTING = 'auth.token.expire_minutes'
@@ -116,6 +119,8 @@ RESULTS_CACHE_TTL_SETTING = 'results.cache.ttl_minutes'
 TEST_SET_DEFAULT_USED_SETTING = 'selection_test_sets.default_name_used'
 DB_CONN_DEFAULT_USED_SETTING = 'connections.default_name_used'
 DEFAULT_MATERIALIZE_SETTING = 'defaults.federates.materialized'
+SEEDS_INFER_SCHEMA_SETTING = 'seeds.infer_schema'
+SEEDS_NA_VALUES_SETTING = 'seeds.na_values'
 IN_MEMORY_DB_SETTING = 'in_memory_database'
 SQLITE = 'sqlite'
 DUCKDB = 'duckdb'
@@ -138,9 +143,13 @@ CONF_FORMAT_CHOICES2 = [(PYTHON_FORMAT2, PYTHON_FORMAT), YML_FORMAT]
 
 EXPENSES_DB_NAME = 'expenses'
 WEATHER_DB_NAME = 'weather'
-DATABASE_CHOICES = [EXPENSES_DB_NAME, WEATHER_DB_NAME]
+NO_DB = 'none'
+DATABASE_CHOICES = [EXPENSES_DB_NAME, WEATHER_DB_NAME, NO_DB]
 
 # Function names
 GET_USER_FUNC = "get_user_if_valid"
 DEP_FUNC = "dependencies"
 MAIN_FUNC = "main"
+
+# Regex
+date_regex = r'^\d{4}\-\d{2}\-\d{2}$'

squirrels/_environcfg.py

@@ -5,7 +5,8 @@ import os, yaml
 from . import _constants as c, _utils as u
 from ._timer import timer, time
 
-_GLOBAL_SQUIRRELS_CFG_FILE = u.join_paths(os.path.expanduser('~'), '.squirrels', c.ENVIRON_CONFIG_FILE)
+_GLOBAL_SQUIRRELS_CFG_FILE1 = u.join_paths(os.path.expanduser('~'), '.squirrels', c.ENVIRON_CONFIG_FILE)
+_GLOBAL_SQUIRRELS_CFG_FILE2 = u.join_paths(os.path.expanduser('~'), '.squirrels', c.ENV_CONFIG_FILE)
 
 
 @dataclass
@@ -61,17 +62,20 @@ class EnvironConfigIO:
             except FileNotFoundError:
                 return {}
         
-        config1 = load_yaml(_GLOBAL_SQUIRRELS_CFG_FILE)
-        config2 = load_yaml(c.ENVIRON_CONFIG_FILE)
+        master_env_config1 = load_yaml(_GLOBAL_SQUIRRELS_CFG_FILE1)
+        master_env_config2 = load_yaml(_GLOBAL_SQUIRRELS_CFG_FILE2)
+        proj_env_config1 = load_yaml(c.ENVIRON_CONFIG_FILE)
+        proj_env_config2 = load_yaml(c.ENV_CONFIG_FILE)
 
-        for key in config2:
-            config1.setdefault(key, {})
-            config1[key].update(config2[key])
+        for project_config in [master_env_config2, proj_env_config1, proj_env_config2]:
+            for key in project_config:
+                master_env_config1.setdefault(key, {})
+                master_env_config1[key].update(project_config[key])
         
-        users = config1.get(c.USERS_KEY, {})
-        env_vars = config1.get(c.ENV_VARS_KEY, {})
-        credentials = config1.get(c.CREDENTIALS_KEY, {})
-        secrets = config1.get(c.SECRETS_KEY, {})
+        users = master_env_config1.get(c.USERS_KEY, {})
+        env_vars = master_env_config1.get(c.ENV_VARS_KEY, {})
+        credentials = master_env_config1.get(c.CREDENTIALS_KEY, {})
+        secrets = master_env_config1.get(c.SECRETS_KEY, {})
 
         cls.obj = _EnvironConfig(users, env_vars, credentials, secrets)
-        timer.add_activity_time(f"loading {c.ENVIRON_CONFIG_FILE} file", start)
+        timer.add_activity_time(f"loading {c.ENV_CONFIG_FILE} file", start)

squirrels/_initializer.py

@@ -33,8 +33,11 @@ class Initializer:
     def _copy_database_file(self, filepath: str):
         self._copy_file(u.join_paths(c.DATABASE_FOLDER, filepath))
     
-    def _copy_pyconfigs_file(self, filepath: str):
-        self._copy_file(u.join_paths(c.PYCONFIG_FOLDER, filepath))
+    def _copy_pyconfig_file(self, filepath: str):
+        self._copy_file(u.join_paths(c.PYCONFIGS_FOLDER, filepath))
+    
+    def _copy_seed_file(self, filepath: str):
+        self._copy_file(u.join_paths(c.SEEDS_FOLDER, filepath))
 
     def init_project(self, args):
         options = ["core", "connections", "parameters", "dbview", "federate", "auth", "sample_db"]
@@ -69,11 +72,11 @@ class Initializer:
 
             remaining_questions = [
                 inquirer.Confirm(AUTH,
-                                 message=f"Do you want to add the '{c.AUTH_FILE}' file?" ,
+                                 message=f"Do you want to add the '{c.AUTH_FILE}' file to enable custom API authentication?" ,
                                  default=False),
                 inquirer.List(SAMPLE_DB, 
                               message="What sample sqlite database do you wish to use (if any)?",
-                              choices=["none"] + c.DATABASE_CHOICES)
+                              choices= c.DATABASE_CHOICES)
             ]
             answers.update(inquirer.prompt(remaining_questions))
         
@@ -124,11 +127,9 @@ class Initializer:
                     "connections": c.CONNECTIONS_YML_FILE if connections_use_yaml else None
                 }
                 substitutions = {key: get_content(val) for key, val in file_name_dict.items()}
-                substitutions["db_view_file"] = db_view_file
-                substitutions["final_view_file"] = federate_file
                 
                 manifest_template = get_content(c.MANIFEST_JINJA_FILE)
-                manifest_content = u.render_string(manifest_template, substitutions)
+                manifest_content = u.render_string(manifest_template, **substitutions)
                 output_path = u.join_paths(base_proj_dir, TMP_FOLDER, c.MANIFEST_FILE)
                 with open(u.join_paths(output_path), "w") as f:
                     f.write(manifest_content)
@@ -139,26 +140,35 @@ class Initializer:
             self._copy_file(c.MANIFEST_FILE, src_folder=TMP_FOLDER)
             
             if connections_use_py:
-                self._copy_pyconfigs_file(c.CONNECTIONS_FILE)
-            elif not connections_use_yaml:
+                self._copy_pyconfig_file(c.CONNECTIONS_FILE)
+            elif connections_use_yaml:
+                pass # already included in squirrels.yml
+            else:
                 raise NotImplementedError(f"Format '{connections_format}' not supported for configuring database connections")
             
             if parameters_use_py:
-                self._copy_pyconfigs_file(c.PARAMETERS_FILE)
-            elif not parameters_use_yaml:
+                self._copy_pyconfig_file(c.PARAMETERS_FILE)
+            elif parameters_use_yaml:
+                pass # already included in squirrels.yml
+            else:
                 raise NotImplementedError(f"Format '{parameters_format}' not supported for configuring widget parameters")
             
-            self._copy_pyconfigs_file(c.CONTEXT_FILE)
-            self._copy_file(c.ENVIRON_CONFIG_FILE)
+            self._copy_pyconfig_file(c.CONTEXT_FILE)
+            self._copy_file(c.ENV_CONFIG_FILE)
+            self._copy_seed_file(c.CATEGORY_SEED_FILE)
+            self._copy_seed_file(c.SUBCATEGORY_SEED_FILE)
 
             self._copy_dbview_file(db_view_file)
             self._copy_federate_file(federate_file)
         
         if answers.get(AUTH, False):
-            self._copy_pyconfigs_file(c.AUTH_FILE)
+            self._copy_pyconfig_file(c.AUTH_FILE)
 
         sample_db = answers.get(SAMPLE_DB)
-        if sample_db is not None and sample_db != "none":
+        if sample_db is None: # None if not prompt mode and '--sample-db' option was not specified
+            sample_db = c.EXPENSES_DB_NAME
+        
+        if sample_db != c.NO_DB:
             if sample_db == c.EXPENSES_DB_NAME:
                 self._copy_database_file(c.EXPENSES_DB_NAME+".db")
             elif sample_db == c.WEATHER_DB_NAME:

squirrels/_manifest.py

@@ -1,5 +1,5 @@
 from typing import Optional
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import Enum
 import yaml
 
@@ -96,16 +96,20 @@ class ParametersConfig(ManifestComponentConfig):
 @dataclass
 class TestSetsConfig(ManifestComponentConfig):
     name: str
-    user_attributes: dict
-    parameters: dict
+    datasets: Optional[list[str]] = None
+    is_authenticated: bool = False
+    user_attributes: dict = field(default_factory=dict)
+    parameters: dict = field(default_factory=dict)
 
     @classmethod
     def from_dict(cls, kwargs: dict):
         cls._validate_required(kwargs, [c.TEST_SET_NAME_KEY], c.TEST_SETS_KEY)
         name = str(kwargs[c.TEST_SET_NAME_KEY])
+        datasets = kwargs.get(c.TEST_SET_DATASETS_KEY)
+        is_authenticated = (c.TEST_SET_USER_ATTR_KEY in kwargs)
         user_attributes = kwargs.get(c.TEST_SET_USER_ATTR_KEY, {})
         parameters = kwargs.get(c.TEST_SET_PARAMETERS_KEY, {})
-        return cls(name, user_attributes, parameters)
+        return cls(name, datasets, is_authenticated, user_attributes, parameters)
 
 
 @dataclass
@@ -145,8 +149,9 @@ class DatasetsConfig(ManifestComponentConfig):
     label: str
     model: str
     scope: DatasetScope
-    parameters: Optional[list[str]]
+    parameters: list[str]
     traits: dict
+    default_test_set: Optional[str]
 
     @classmethod
     def from_dict(cls, kwargs: dict):
@@ -161,9 +166,10 @@ class DatasetsConfig(ManifestComponentConfig):
             scope_list = [scope.name.lower() for scope in DatasetScope]
             raise u.ConfigurationError(f'Scope not found for dataset "{name}". Scope must be one of {scope_list}') from e
         
-        parameters = kwargs.get(c.DATASET_PARAMETERS_KEY)
+        parameters = kwargs.get(c.DATASET_PARAMETERS_KEY, [])
         traits = kwargs.get(c.DATASET_TRAITS_KEY, {})
-        return cls(name, label, model, scope, parameters, traits)
+        default_test_set = kwargs.get(c.DATASET_DEFAULT_TEST_SET_KEY)
+        return cls(name, label, model, scope, parameters, traits, default_test_set)
 
 
 @dataclass
@@ -208,14 +214,18 @@ class _ManifestConfig:
         params = [ParametersConfig.from_dict(x) for x in kwargs.get(c.PARAMETERS_KEY, [])]
 
         test_sets = cls._create_configs_as_dict(TestSetsConfig, kwargs, c.TEST_SETS_KEY, c.TEST_SET_NAME_KEY)
-        default_test_set: str = settings.get(c.TEST_SET_DEFAULT_USED_SETTING, c.DEFAULT_TEST_SET_NAME)
-        test_sets.setdefault(default_test_set, TestSetsConfig.from_dict({c.TEST_SET_NAME_KEY: default_test_set}))
-
         dbviews = cls._create_configs_as_dict(DbviewConfig, kwargs, c.DBVIEWS_KEY, c.DBVIEW_NAME_KEY)
         federates = cls._create_configs_as_dict(FederateConfig, kwargs, c.FEDERATES_KEY, c.FEDERATE_NAME_KEY)
         datasets = cls._create_configs_as_dict(DatasetsConfig, kwargs, c.DATASETS_KEY, c.DATASET_NAME_KEY)
 
         return cls(proj_vars, packages, db_conns, params, test_sets, dbviews, federates, datasets, settings)
+    
+    def get_default_test_set(self, dataset_name: str) -> tuple[str, dict]:
+        default_1 = self.datasets[dataset_name].default_test_set
+        default_2 = self.settings.get(c.TEST_SET_DEFAULT_USED_SETTING, c.DEFAULT_TEST_SET_NAME)
+        default_name = default_1 if default_1 is not None else default_2
+        default_test_set = self.selection_test_sets.get(default_name, TestSetsConfig.from_dict({c.TEST_SET_NAME_KEY: default_name}))
+        return default_name, default_test_set
 
 
 class ManifestIO:
@@ -227,8 +237,8 @@ class ManifestIO:
         
         start = time.time()
         raw_content = u.read_file(c.MANIFEST_FILE)
-        env_config = EnvironConfigIO.obj.get_all_env_vars()
-        content = u.render_string(raw_content, env_config)
+        env_vars = EnvironConfigIO.obj.get_all_env_vars()
+        content = u.render_string(raw_content, env_vars=env_vars, **env_vars) # TODO: deprecate **env_vars
         proj_config = yaml.safe_load(content)
         cls.obj = _ManifestConfig.from_dict(proj_config)
         timer.add_activity_time(f"loading {c.MANIFEST_FILE} file", start)