dragon-ml-toolbox 19.10.0__py3-none-any.whl → 19.12.0__py3-none-any.whl

ml_tools/_core/_ML_utilities.py

@@ -46,17 +46,24 @@ class ArtifactFinder:
         └── FeatureSchema.json     (Required if `load_schema` is True)
     ```
     """
-    def __init__(self, directory: Union[str, Path], load_scaler: bool, load_schema: bool) -> None:
+    def __init__(self, 
+                 directory: Union[str, Path], 
+                 load_scaler: bool, 
+                 load_schema: bool,
+                 strict: bool=False,
+                 verbose: bool=True) -> None:
         """
         Args:
             directory (str | Path): The path to the directory that contains training artifacts.
             load_scaler (bool): If True, requires and searches for a scaler file `scaler_*.pth`.
             load_schema (bool): If True, requires and searches for a FeatureSchema file `FeatureSchema.json`.
+            strict (bool): If True, raises an error if any artifact is missing. If False, returns None for missing artifacts silently.
+            verbose (bool): Displays the missing artifacts in the directory or a success message.
         """
         # validate directory
         dir_path = make_fullpath(directory, enforce="directory")
         
-        parsing_dict = _find_model_artifacts(target_directory=dir_path, load_scaler=load_scaler, verbose=False)
+        parsing_dict = _find_model_artifacts(target_directory=dir_path, load_scaler=load_scaler, verbose=False, strict=strict)
         
         self._weights_path = parsing_dict[PytorchArtifactPathKeys.WEIGHTS_PATH]
         self._feature_names_path = parsing_dict[PytorchArtifactPathKeys.FEATURES_PATH]
@@ -64,71 +71,121 @@ class ArtifactFinder:
         self._model_architecture_path = parsing_dict[PytorchArtifactPathKeys.ARCHITECTURE_PATH]
         self._scaler_path = None
         self._schema = None
+        self._strict = strict
         
         if load_scaler:
             self._scaler_path = parsing_dict[PytorchArtifactPathKeys.SCALER_PATH]
             
         if load_schema:
-            self._schema = FeatureSchema.from_json(directory=dir_path)
+            try:
+                self._schema = FeatureSchema.from_json(directory=dir_path)
+            except Exception:
+                if strict:
+                    # FeatureSchema logs its own error details
+                    # _LOGGER.error(f"Failed to load FeatureSchema from '{dir_path.name}': {e}")
+                    raise FileNotFoundError()
+                else:
+                    # _LOGGER.warning(f"Could not load FeatureSchema from '{dir_path.name}': {e}")
+                    self._schema = None
+
+        # Process feature names
+        if self._feature_names_path is not None:
+            self._feature_names = self._process_text(self._feature_names_path)
+        else:
+            self._feature_names = None
+        # Process target names
+        if self._target_names_path is not None:
+            self._target_names = self._process_text(self._target_names_path)
+        else:
+            self._target_names = None
+            
+        if verbose:
+            # log missing artifacts
+            missing_artifacts = []
+            if self._feature_names is None:
+                missing_artifacts.append("Feature Names")
+            if self._target_names is None:
+                missing_artifacts.append("Target Names")
+            if self._weights_path is None:
+                missing_artifacts.append("Weights File")
+            if self._model_architecture_path is None:
+                missing_artifacts.append("Model Architecture File")
+            if load_scaler and self._scaler_path is None:
+                missing_artifacts.append("Scaler File")
+            if load_schema and self._schema is None:
+                missing_artifacts.append("FeatureSchema File")
+            
+            if missing_artifacts:
+                _LOGGER.warning(f"Missing artifacts in '{dir_path.name}': {', '.join(missing_artifacts)}.")
+            else:
+                _LOGGER.info(f"All artifacts successfully loaded from '{dir_path.name}'.")
 
-        # Process text files
-        self._feature_names = self._process_text(self._feature_names_path)
-        self._target_names = self._process_text(self._target_names_path)
-        
     def _process_text(self, text_file_path: Path):
         list_strings = load_list_strings(text_file=text_file_path, verbose=False)
         return list_strings
     
     @property
-    def feature_names(self) -> list[str]:
+    def feature_names(self) -> Union[list[str], None]:
         """Returns the feature names as a list of strings."""
+        if self._strict and not self._feature_names:
+            _LOGGER.error("No feature names loaded for Strict mode.")
+            raise ValueError()
         return self._feature_names
     
     @property
-    def target_names(self) -> list[str]:
+    def target_names(self) -> Union[list[str], None]:
         """Returns the target names as a list of strings."""
+        if self._strict and not self._target_names:
+            _LOGGER.error("No target names loaded for Strict mode.")
+            raise ValueError()
         return self._target_names
     
     @property
-    def weights_path(self) -> Path:
+    def weights_path(self) -> Union[Path, None]:
         """Returns the path to the state dictionary pth file."""
+        if self._strict and self._weights_path is None:
+            _LOGGER.error("No weights file loaded for Strict mode.")
+            raise ValueError()
         return self._weights_path
     
     @property
-    def model_architecture_path(self) -> Path:
+    def model_architecture_path(self) -> Union[Path, None]:
         """Returns the path to the model architecture json file."""
+        if self._strict and self._model_architecture_path is None:
+            _LOGGER.error("No model architecture file loaded for Strict mode.")
+            raise ValueError()
         return self._model_architecture_path
     
     @property
-    def scaler_path(self) -> Path:
+    def scaler_path(self) -> Union[Path, None]:
         """Returns the path to the scaler file."""
-        if self._scaler_path is None:
-            _LOGGER.error("No scaler file loaded. Set 'load_scaler=True'.")
+        if self._strict and self._scaler_path is None:
+            _LOGGER.error("No scaler file loaded for Strict mode.")
             raise ValueError()
         else:
             return self._scaler_path
         
     @property
-    def feature_schema(self) -> FeatureSchema:
+    def feature_schema(self) -> Union[FeatureSchema, None]:
         """Returns the FeatureSchema object."""
-        if self._schema is None:
-            _LOGGER.error("No FeatureSchema loaded. Set 'load_schema=True'.")
+        if self._strict and self._schema is None:
+            _LOGGER.error("No FeatureSchema loaded for Strict mode.")
             raise ValueError()
         else:
             return self._schema
         
     def __repr__(self) -> str:
-        dir_name = self._weights_path.parent.name
-        n_features = len(self._feature_names)
-        n_targets = len(self._target_names)
+        dir_name = self._weights_path.parent.name if self._weights_path else "Unknown"
+        n_features = len(self._feature_names) if self._feature_names else "None"
+        n_targets = len(self._target_names) if self._target_names else "None"
         scaler_status = self._scaler_path.name if self._scaler_path else "None"
         schema_status = "Loaded" if self._schema else "None"
         
         return (
             f"{self.__class__.__name__}\n"
             f"    directory='{dir_name}'\n"
-            f"    weights='{self._weights_path.name}'\n"
-            f"    architecture='{self._model_architecture_path.name}'\n"
+            f"    weights='{self._weights_path.name if self._weights_path else 'None'}'\n"
+            f"    architecture='{self._model_architecture_path.name if self._model_architecture_path else 'None'}'\n"
             f"    scaler='{scaler_status}'\n"
             f"    schema='{schema_status}'\n"
             f"    features={n_features}\n" 
@@ -136,7 +193,7 @@ class ArtifactFinder:
         )
 
 
-def _find_model_artifacts(target_directory: Union[str,Path], load_scaler: bool, verbose: bool=False) -> dict[str, Path]:
+def _find_model_artifacts(target_directory: Union[str,Path], load_scaler: bool, verbose: bool=True, strict:bool=True) -> dict[str, Union[Path, None]]:
     """
     Scans a directory to find paths to model weights, target names, feature names, and model architecture. Optionally an scaler path if `load_scaler` is True.
     
@@ -155,41 +212,70 @@ def _find_model_artifacts(target_directory: Union[str,Path], load_scaler: bool,
         target_directory (str | Path): The path to the directory that contains training artifacts.
         load_scaler (bool): If True, the function requires and searches for a scaler file `scaler_*.pth`.
         verbose (bool): If True, enables detailed logging during the search process.
+        strict (bool): If True, raises errors on missing files. If False, returns None for missing files.
     """
     # validate directory
     dir_path = make_fullpath(target_directory, enforce="directory")
     dir_name = dir_path.name
     
     # find files
-    model_pth_dict = list_files_by_extension(directory=dir_path, extension="pth", verbose=verbose)
+    model_pth_dict = list_files_by_extension(directory=dir_path, extension="pth", verbose=False, raise_on_empty=False)
     
-    # restriction
-    if load_scaler:
-        if len(model_pth_dict) != 2:
-            _LOGGER.error(f"Directory '{dir_name}' should contain exactly 2 '.pth' files: scaler and weights.")
-            raise IOError()
-    else:
-        if len(model_pth_dict) != 1:
-            _LOGGER.error(f"Directory '{dir_name}' should contain exactly 1 '.pth' file for weights.")
+    if not model_pth_dict:
+        pth_msg=f"No '.pth' files found in directory: {dir_name}."
+        if strict:
+            _LOGGER.error(pth_msg)
             raise IOError()
+        else:
+            if verbose:
+                _LOGGER.warning(pth_msg)
+            model_pth_dict = None
+    
+    # restriction
+    if model_pth_dict is not None:
+        valid_count = False
+        msg = ""
+        
+        if load_scaler:
+            if len(model_pth_dict) == 2:
+                valid_count = True
+            else:
+                msg = f"Directory '{dir_name}' should contain exactly 2 '.pth' files: scaler and weights. Found {len(model_pth_dict)}."
+        else:
+            if len(model_pth_dict) == 1:
+                valid_count = True
+            else:
+                msg = f"Directory '{dir_name}' should contain exactly 1 '.pth' file for weights. Found {len(model_pth_dict)}."
+        
+        # Respect strict mode for count mismatch
+        if not valid_count:
+            if strict:
+                _LOGGER.error(msg)
+                raise IOError()
+            else:
+                if verbose:
+                    _LOGGER.warning(msg)
+                # Invalidate dictionary
+                model_pth_dict = None
     
     ##### Scaler and Weights #####
     scaler_path = None
     weights_path = None
     
     # load weights and scaler if present
-    for pth_filename, pth_path in model_pth_dict.items():
-        if load_scaler and pth_filename.lower().startswith(DatasetKeys.SCALER_PREFIX):
-            scaler_path = pth_path
-        else:
-            weights_path = pth_path
+    if model_pth_dict is not None:
+        for pth_filename, pth_path in model_pth_dict.items():
+            if load_scaler and pth_filename.lower().startswith(DatasetKeys.SCALER_PREFIX):
+                scaler_path = pth_path
+            else:
+                weights_path = pth_path
     
     # validation
-    if not weights_path:
+    if not weights_path and strict:
         _LOGGER.error(f"Error parsing the model weights path from '{dir_name}'")
         raise IOError()
     
-    if load_scaler and not scaler_path:
+    if strict and load_scaler and not scaler_path:
         _LOGGER.error(f"Error parsing the scaler path from '{dir_name}'")
         raise IOError()
     
@@ -198,32 +284,44 @@ def _find_model_artifacts(target_directory: Union[str,Path], load_scaler: bool,
     feature_names_path = None
     
     # load feature and target names
-    model_txt_dict = list_files_by_extension(directory=dir_path, extension="txt", verbose=verbose)
+    model_txt_dict = list_files_by_extension(directory=dir_path, extension="txt", verbose=False, raise_on_empty=False)
     
+    # if the directory has no txt files, the loop is skipped
     for txt_filename, txt_path in model_txt_dict.items():
         if txt_filename == DatasetKeys.FEATURE_NAMES:
             feature_names_path = txt_path
         elif txt_filename == DatasetKeys.TARGET_NAMES:
             target_names_path = txt_path
     
-    # validation
-    if not target_names_path or not feature_names_path:
-        _LOGGER.error(f"Error parsing features path or targets path from '{dir_name}'")
+    # validation per case
+    if strict and not target_names_path:
+        _LOGGER.error(f"Error parsing the target names path from '{dir_name}'")
         raise IOError()
+    elif verbose and not target_names_path:
+        _LOGGER.warning(f"Target names file not found in '{dir_name}'.")
     
+    if strict and not feature_names_path:
+        _LOGGER.error(f"Error parsing the feature names path from '{dir_name}'")
+        raise IOError()
+    elif verbose and not feature_names_path:
+        _LOGGER.warning(f"Feature names file not found in '{dir_name}'.")
+
     ##### load model architecture path #####
     architecture_path = None
     
-    model_json_dict = list_files_by_extension(directory=dir_path, extension="json", verbose=verbose)
+    model_json_dict = list_files_by_extension(directory=dir_path, extension="json", verbose=False, raise_on_empty=False)
     
+    # if the directory has no json files, the loop is skipped
     for json_filename, json_path in model_json_dict.items():
         if json_filename == PytorchModelArchitectureKeys.SAVENAME:
             architecture_path = json_path
     
     # validation
-    if not architecture_path:
+    if strict and not architecture_path:
         _LOGGER.error(f"Error parsing the model architecture path from '{dir_name}'")
         raise IOError()
+    elif verbose and not architecture_path:
+        _LOGGER.warning(f"Model architecture file not found in '{dir_name}'.")
     
     ##### Paths dictionary #####
     parsing_dict = {
@@ -233,7 +331,7 @@ def _find_model_artifacts(target_directory: Union[str,Path], load_scaler: bool,
         PytorchArtifactPathKeys.TARGETS_PATH: target_names_path,
     }
     
-    if scaler_path is not None:
+    if load_scaler:
         parsing_dict[PytorchArtifactPathKeys.SCALER_PATH] = scaler_path
     
     return parsing_dict
@@ -246,6 +344,9 @@ def find_model_artifacts_multi(target_directory: Union[str,Path], load_scaler: b
     This function operates on a specific directory structure. It expects the
     `target_directory` to contain one or more subdirectories, where each
     subdirectory represents a single trained model result.
+    
+    This function works using a strict mode, meaning that it will raise errors if
+    any required artifacts are missing in a model's subdirectory.
 
     The expected directory structure for each model is as follows:
     ```
@@ -278,14 +379,16 @@ def find_model_artifacts_multi(target_directory: Union[str,Path], load_scaler: b
     all_artifacts: list[dict[str, Path]] = list()
     
     # find model directories
-    result_dirs_dict = list_subdirectories(root_dir=root_path, verbose=verbose)
+    result_dirs_dict = list_subdirectories(root_dir=root_path, verbose=verbose, raise_on_empty=True)
     for _dir_name, dir_path in result_dirs_dict.items():
         
         parsing_dict = _find_model_artifacts(target_directory=dir_path,
                                             load_scaler=load_scaler,
-                                            verbose=verbose)
+                                            verbose=verbose,
+                                            strict=True)
         
-        all_artifacts.append(parsing_dict)
+        # parsing_dict is guaranteed to have all required paths due to strict=True
+        all_artifacts.append(parsing_dict)  # type: ignore
     
     return all_artifacts
 
@@ -721,7 +824,7 @@ def select_features_by_shap(
     root_path = make_fullpath(root_directory, enforce="directory")
 
     # --- Step 2: Directory and File Discovery ---
-    subdirectories = list_subdirectories(root_dir=root_path, verbose=False)
+    subdirectories = list_subdirectories(root_dir=root_path, verbose=False, raise_on_empty=True)
     
     shap_filename = SHAPKeys.SAVENAME + ".csv"
 

ml_tools/_core/_PSO_optimization.py

@@ -169,7 +169,7 @@ def multiple_objective_functions_from_dir(directory: Union[str,Path], add_noise:
     """
     objective_functions = list()
     objective_function_names = list()
-    for file_name, file_path in list_files_by_extension(directory=directory, extension='joblib').items():
+    for file_name, file_path in list_files_by_extension(directory=directory, extension='joblib', raise_on_empty=True).items():
         current_objective = ObjectiveFunction(trained_model_path=file_path,
                                               add_noise=add_noise,
                                               task=task,

ml_tools/_core/_ensemble_inference.py

@@ -42,7 +42,7 @@ class DragonEnsembleInferenceHandler:
         self.verbose = verbose
         self._feature_names: Optional[List[str]] = None
         
-        model_files = list_files_by_extension(directory=models_dir, extension="joblib")
+        model_files = list_files_by_extension(directory=models_dir, extension="joblib", raise_on_empty=True)
         
         for fname, fpath in model_files.items():
             try:

ml_tools/_core/_keys.py

@@ -1,6 +1,6 @@
 class MagicWords:
     """General purpose keys"""
-    LATEST = "latest"
+    BEST = "best"
     CURRENT = "current"
     RENAME = "rename"
     UNKNOWN = "unknown"
@@ -200,6 +200,37 @@ class MLTaskKeys:
     ALL_BINARY_TASKS = [BINARY_CLASSIFICATION, MULTILABEL_BINARY_CLASSIFICATION, BINARY_IMAGE_CLASSIFICATION, BINARY_SEGMENTATION]
 
 
+class _PublicTaskKeys:
+    """
+    Task keys used in the Dragon ML pipeline:
+    
+    1. REGRESSION
+    2. MULTITARGET_REGRESSION
+    3. BINARY_CLASSIFICATION
+    4. MULTICLASS_CLASSIFICATION
+    5. MULTILABEL_BINARY_CLASSIFICATION
+    6. BINARY_IMAGE_CLASSIFICATION
+    7. MULTICLASS_IMAGE_CLASSIFICATION  
+    8. BINARY_SEGMENTATION
+    9. MULTICLASS_SEGMENTATION
+    10. OBJECT_DETECTION
+    11. SEQUENCE_SEQUENCE
+    12. SEQUENCE_VALUE
+    """
+    REGRESSION = MLTaskKeys.REGRESSION
+    MULTITARGET_REGRESSION = MLTaskKeys.MULTITARGET_REGRESSION
+    BINARY_CLASSIFICATION = MLTaskKeys.BINARY_CLASSIFICATION
+    MULTICLASS_CLASSIFICATION = MLTaskKeys.MULTICLASS_CLASSIFICATION
+    MULTILABEL_BINARY_CLASSIFICATION = MLTaskKeys.MULTILABEL_BINARY_CLASSIFICATION
+    BINARY_IMAGE_CLASSIFICATION = MLTaskKeys.BINARY_IMAGE_CLASSIFICATION
+    MULTICLASS_IMAGE_CLASSIFICATION = MLTaskKeys.MULTICLASS_IMAGE_CLASSIFICATION
+    BINARY_SEGMENTATION = MLTaskKeys.BINARY_SEGMENTATION
+    MULTICLASS_SEGMENTATION = MLTaskKeys.MULTICLASS_SEGMENTATION
+    OBJECT_DETECTION = MLTaskKeys.OBJECT_DETECTION
+    SEQUENCE_SEQUENCE = MLTaskKeys.SEQUENCE_SEQUENCE
+    SEQUENCE_VALUE = MLTaskKeys.SEQUENCE_VALUE
+
+
 class DragonTrainerKeys:
     VALIDATION_METRICS_DIR = "Validation_Metrics"
     TEST_METRICS_DIR = "Test_Metrics"

ml_tools/_core/_optimization_tools.py

@@ -269,7 +269,7 @@ def plot_optimal_feature_distributions(results_dir: Union[str, Path],
     output_path = make_fullpath(results_path / "DistributionPlots", make=True)
     
     # Check that the directory contains csv files
-    list_csv_paths(results_path, verbose=False)
+    list_csv_paths(results_path, verbose=False, raise_on_empty=True)
 
     # --- Data Loading and Preparation ---
     _LOGGER.debug(f"📁 Starting analysis from results in: '{results_dir}'")