mlrl-testbed-sklearn 0.12.0__py3-none-any.whl

mlrl/testbed_sklearn/experiments/__init__.py

@@ -0,0 +1,8 @@
+"""
+Author Michael Rapp (michael.rapp.ml@gmail.com)
+
+Provides classes that allow running experiments using the scikit-learn framework.
+"""
+from mlrl.testbed_sklearn.experiments.experiment import SkLearnExperiment
+from mlrl.testbed_sklearn.experiments.problem_domain import SkLearnClassificationProblem, SkLearnProblem, \
+    SkLearnRegressionProblem

mlrl/testbed_sklearn/experiments/dataset.py

@@ -0,0 +1,141 @@
+"""
+Author: Michael Rapp (michael.rapp.ml@gmail.com)
+
+Provides classes for representing tabular datasets.
+"""
+from dataclasses import dataclass, replace
+from enum import Enum, auto
+from functools import reduce
+from typing import List, Optional
+
+from scipy.sparse import lil_array
+
+from mlrl.testbed.experiments.dataset import Dataset
+
+from mlrl.util.arrays import is_sparse
+
+
+class AttributeType(Enum):
+    """
+    All supported types of attributes.
+    """
+    NUMERICAL = auto()
+    ORDINAL = auto()
+    NOMINAL = auto()
+
+
+@dataclass
+class Attribute:
+    """
+    An attribute, e.g., a feature, a ground truth label, or a regression score, that is contained by a dataset.
+
+    Attributes:
+        name:           The name of the attribute
+        attribute_type: The type of the attribute
+        nominal_values: A list that contains the possible values in case of a nominal feature
+    """
+    name: str
+    attribute_type: AttributeType
+    nominal_values: Optional[List[str]] = None
+
+
+@dataclass
+class TabularDataset(Dataset):
+    """
+    A tabular dataset consisting of two matrices `x` and `y`, storing the features of examples and their respective
+    ground truth, respectively.
+
+    Attributes:
+        x:          A `lil_array`, shape `(num_examples, num_features)`, that stores the features of examples
+        y:          A `lil_array`, shape `(num_examples, num_features)`, that stores the ground truth of examples
+        features:   A list that contains all features in the dataset
+        outputs:    A list that contains all outputs in the dataset
+    """
+    x: lil_array
+    y: lil_array
+    features: List[Attribute]
+    outputs: List[Attribute]
+
+    @property
+    def num_examples(self) -> int:
+        """
+        The number of examples in the dataset.
+        """
+        return self.x.shape[0]
+
+    @property
+    def num_features(self) -> int:
+        """
+        The number of features in the dataset.
+        """
+        return self.x.shape[1]
+
+    @property
+    def num_outputs(self) -> int:
+        """
+        The number of outputs in the dataset.
+        """
+        return self.y.shape[1]
+
+    @property
+    def has_sparse_features(self) -> bool:
+        """
+        True, if feature values in the dataset are sparse, False otherwise.
+        """
+        return is_sparse(self.x)
+
+    @property
+    def has_sparse_outputs(self) -> bool:
+        """
+        True, if the ground truth in the dataset is sparse, False otherwise.
+        """
+        return is_sparse(self.y)
+
+    def enforce_dense_features(self) -> 'TabularDataset':
+        """
+        Creates and returns a copy of this dataset, where the feature values have been converted into a dense format.
+
+        :return: The dataset that has been created
+        """
+        if self.has_sparse_features:
+            return replace(self, x=self.x.toarray())
+        return self
+
+    def enforce_dense_outputs(self) -> 'TabularDataset':
+        """
+        Creates and returns a copy of this dataset, where the ground truth has been converted into a dense format.
+
+        :return: The dataset that has been created
+        """
+        if self.has_sparse_outputs:
+            return replace(self, y=self.y.toarray())
+        return self
+
+    def get_num_features(self, *feature_types: AttributeType) -> int:
+        """
+        Returns the number of features with one out of a given set of types.  If no types are given, all features are
+        counted.
+
+        :param feature_types:   The types of the features to be counted
+        :return:                The number of features of the given types
+        """
+        feature_types = set(feature_types)
+
+        if feature_types:
+            return reduce(lambda aggr, feature: aggr + (1 if feature.attribute_type in feature_types else 0),
+                          self.features, 0)
+
+        return len(self.features)
+
+    def get_feature_indices(self, *feature_types: AttributeType) -> List[int]:
+        """
+        Returns a list that contains the indices of all features with one out of a given set of types (in ascending
+        order). If no types are given, all indices are returned.
+
+        :param feature_types:   The types of the features whose indices should be returned
+        :return:                A list that contains the indices of all features of the given types
+        """
+        feature_types = set(feature_types)
+        return [
+            i for i, feature in enumerate(self.features) if not feature_types or feature.attribute_type in feature_types
+        ]

mlrl/testbed_sklearn/experiments/experiment.py

@@ -0,0 +1,164 @@
+"""
+Author: Michael Rapp (michael.rapp.ml@gmail.com)
+
+Provides classes for performing experiments using the scikit-learn framework.
+"""
+import logging as log
+
+from dataclasses import replace
+from functools import reduce
+from typing import Any, Dict, Generator, Optional
+
+from sklearn.base import BaseEstimator, clone
+
+from mlrl.testbed_sklearn.experiments.dataset import TabularDataset
+from mlrl.testbed_sklearn.experiments.output.characteristics.data.writer_data import DataCharacteristicsWriter
+from mlrl.testbed_sklearn.experiments.output.characteristics.data.writer_prediction import \
+    PredictionCharacteristicsWriter
+from mlrl.testbed_sklearn.experiments.output.dataset.writer_ground_truth import GroundTruthWriter
+from mlrl.testbed_sklearn.experiments.output.dataset.writer_prediction import PredictionWriter
+from mlrl.testbed_sklearn.experiments.output.evaluation.writer import EvaluationWriter
+from mlrl.testbed_sklearn.experiments.output.label_vectors import LabelVectorWriter
+from mlrl.testbed_sklearn.experiments.problem_domain import SkLearnProblem
+
+from mlrl.testbed.experiments.dataset import Dataset
+from mlrl.testbed.experiments.experiment import Experiment
+from mlrl.testbed.experiments.input.dataset.splitters.splitter import DatasetSplitter
+from mlrl.testbed.experiments.output.model.writer import ModelWriter
+from mlrl.testbed.experiments.output.parameters.writer import ParameterWriter
+from mlrl.testbed.experiments.problem_domain import ProblemDomain
+from mlrl.testbed.experiments.state import ExperimentState, ParameterDict, PredictionState, TrainingState
+from mlrl.testbed.experiments.timer import Timer
+
+
+class SkLearnExperiment(Experiment):
+    """
+    An experiment that trains and evaluates a machine learning model using the scikit-learn framework.
+    """
+
+    class Builder(Experiment.Builder):
+        """
+        Allows to configure and create instances of the class `SkLearnExperiment`.
+        """
+
+        def __init__(self, problem_domain: ProblemDomain, dataset_splitter: DatasetSplitter):
+            """
+            :param problem_domain:      The problem domain, the experiment should be concerned with
+            :param dataset_splitter:    The method to be used for splitting the dataset into training and test datasets
+            """
+            super().__init__(problem_domain=problem_domain, dataset_splitter=dataset_splitter)
+            self.data_characteristics_writer = DataCharacteristicsWriter()
+            self.prediction_characteristics_writer = PredictionCharacteristicsWriter()
+            self.ground_truth_writer = GroundTruthWriter()
+            self.prediction_writer = PredictionWriter()
+            self.label_vector_writer = LabelVectorWriter()
+            self.evaluation_writer = EvaluationWriter()
+            self.model_writer = ModelWriter()
+            self.parameter_writer = ParameterWriter()
+            self.add_pre_training_output_writers(
+                self.data_characteristics_writer,
+                self.parameter_writer,
+            )
+            self.add_post_training_output_writers(
+                self.model_writer,
+                self.label_vector_writer,
+            )
+            self.add_prediction_output_writers(
+                self.prediction_characteristics_writer,
+                self.ground_truth_writer,
+                self.prediction_writer,
+                self.evaluation_writer,
+            )
+
+        def _create_experiment(self, problem_domain: ProblemDomain, dataset_splitter: DatasetSplitter) -> Experiment:
+            return SkLearnExperiment(problem_domain=problem_domain, dataset_splitter=dataset_splitter)
+
+    def __create_learner(self, parameters: ParameterDict) -> BaseEstimator:
+        learner = clone(self.problem_domain.base_learner)
+
+        if parameters:
+            learner.set_params(**parameters)
+            log.info('Successfully applied parameter setting: %s', parameters)
+
+        return learner
+
+    @staticmethod
+    def __check_for_parameter_changes(expected_parameters: Dict[str, Any], actual_parameters: Dict[str, Any]):
+        changes = []
+
+        for key, expected_value in expected_parameters.items():
+            expected_value = str(expected_value)
+            actual_value = str(actual_parameters[key])
+
+            if actual_value != expected_value:
+                changes.append((key, expected_value, actual_value))
+
+        if changes:
+            log.warning(
+                'The loaded model\'s values for the following parameters differ from the expected configuration: %s',
+                reduce(
+                    lambda aggr, change: aggr +
+                    (', '
+                     if aggr else '') + '"' + change[0] + '" is "' + change[2] + '" instead of "' + change[1] + '"',
+                    changes, ''))
+
+    # pylint: disable=useless-parent-delegation
+    def __init__(self, problem_domain: SkLearnProblem, dataset_splitter: DatasetSplitter):
+        """
+        :param problem_domain:      The problem domain, the experiment is concerned with
+        :param dataset_splitter:    The method to be used for splitting the dataset into training and test datasets
+        """
+        super().__init__(problem_domain=problem_domain, dataset_splitter=dataset_splitter)
+
+    def _fit(self, estimator: BaseEstimator, dataset: TabularDataset,
+             fit_kwargs: Optional[Dict[str, Any]]) -> Timer.Duration:
+        """
+        May be overridden by subclasses in order to fit a scikit-learn estimator to a dataset.
+
+        :param estimator:   A scikit-learn estimator
+        :param fit_kwargs:  Optional keyword arguments to be passed to the estimator
+
+        """
+        fit_kwargs = fit_kwargs if fit_kwargs else {}
+
+        try:
+            start_time = Timer.start()
+            estimator.fit(dataset.x, dataset.y, **fit_kwargs)
+            return Timer.stop(start_time)
+        except ValueError as error:
+            if dataset.has_sparse_features:
+                return self._fit(estimator, dataset.enforce_dense_features(), fit_kwargs)
+            if dataset.has_sparse_outputs:
+                return self._fit(estimator, dataset.enforce_dense_outputs(), fit_kwargs)
+            raise error
+
+    def _train(self, learner: Optional[Any], parameters: ParameterDict, dataset: Dataset) -> TrainingState:
+        new_learner = self.__create_learner(parameters=parameters)
+
+        # Use existing model, if possible, otherwise train a new model...
+        if isinstance(learner, type(new_learner)):
+            self.__check_for_parameter_changes(expected_parameters=parameters, actual_parameters=learner.get_params())
+            return TrainingState(learner=learner)
+
+        log.info('Fitting model to %s training examples...', dataset.num_examples)
+        training_duration = self._fit(new_learner, dataset, fit_kwargs=self.problem_domain.fit_kwargs)
+        log.info('Successfully fit model in %s', training_duration)
+        return TrainingState(learner=new_learner, training_duration=training_duration)
+
+    def _predict(self, state: ExperimentState) -> Generator[PredictionState, None, None]:
+        dataset = state.dataset_as(self, TabularDataset)
+        learner = state.learner_as(self, BaseEstimator)
+
+        if dataset and learner:
+            try:
+                problem_domain = self.problem_domain
+                predict_kwargs = problem_domain.predict_kwargs
+                predict_kwargs = predict_kwargs if predict_kwargs else {}
+                predictor = problem_domain.predictor_factory.create()
+                dataset_type = state.dataset_type
+                yield from predictor.obtain_predictions(learner, dataset, dataset_type, **predict_kwargs)
+            except ValueError as error:
+                if dataset.has_sparse_features:
+                    yield self._predict(replace(state, dataset=dataset.enforce_dense_features()))
+
+                raise error

mlrl/testbed_sklearn/experiments/input/dataset/preprocessors/extension.py

@@ -0,0 +1,47 @@
+"""
+Author: Michael Rapp (michael.rapp.ml@gmail.com)
+
+Provides classes that allow configuring the functionality to preprocess tabular datasets.
+"""
+from argparse import Namespace
+from typing import List, Set
+
+from mlrl.testbed_sklearn.experiments.input.dataset.preprocessors.one_hot_encoder import OneHotEncoder
+
+from mlrl.testbed.experiments.input.dataset.preprocessors.preprocessor import Preprocessor
+from mlrl.testbed.extensions.extension import Extension
+
+from mlrl.util.cli import Argument, BoolArgument
+
+
+class PreprocessorExtension(Extension):
+    """
+    An extension that configures the functionality to preprocess tabular datasets.
+    """
+
+    ONE_HOT_ENCODING = BoolArgument(
+        '--one-hot-encoding',
+        default=False,
+        description='Whether one-hot-encoding should be used to encode nominal features or not.',
+    )
+
+    def _get_arguments(self) -> Set[Argument]:
+        """
+        See :func:`mlrl.testbed.extensions.extension.Extension._get_arguments`
+        """
+        return {self.ONE_HOT_ENCODING}
+
+    @staticmethod
+    def get_preprocessors(args: Namespace) -> List[Preprocessor]:
+        """
+        Returns the preprocessors to be used for preprocessing datasets according to the configuration.
+
+        :param args:    The command line arguments specified by the user
+        :return:        The preprocessors to be used
+        """
+        preprocessors = []
+
+        if PreprocessorExtension.ONE_HOT_ENCODING.get_value(args):
+            preprocessors.append(OneHotEncoder())
+
+        return preprocessors

mlrl/testbed_sklearn/experiments/input/dataset/preprocessors/one_hot_encoder.py

@@ -0,0 +1,63 @@
+"""
+Author: Michael Rapp (michael.rapp.ml@gmail.com)
+
+Provides classes for preprocessing datasets.
+"""
+import logging as log
+
+from dataclasses import replace
+
+from sklearn.compose import ColumnTransformer
+from sklearn.preprocessing import OneHotEncoder as SkLearnOneHotEncoder
+
+from mlrl.testbed_sklearn.experiments.dataset import AttributeType
+
+from mlrl.testbed.experiments.dataset import Dataset
+from mlrl.testbed.experiments.input.dataset.preprocessors.preprocessor import Preprocessor
+
+
+class OneHotEncoder(Preprocessor):
+    """
+    Allows one-hot-encoding all nominal features contained in a tabular dataset, if any.
+    """
+
+    class Encoder(Preprocessor.Encoder):
+        """
+        Allows one-hot-encoding all nominal features contained in a tabular dataset.
+        """
+
+        def __init__(self):
+            self.encoder = None
+
+        def encode(self, dataset: Dataset) -> Dataset:
+            """
+            See :func:`mlrl.testbed.experiments.input.dataset.preprocessors.Preprocessor.Encoder.encode`
+            """
+            nominal_indices = dataset.get_feature_indices(AttributeType.NOMINAL)
+            num_nominal_features = len(nominal_indices)
+            log.info('Dataset contains %s nominal and %s numerical features.', num_nominal_features,
+                     (len(dataset.features) - num_nominal_features))
+
+            if num_nominal_features > 0:
+                dataset = dataset.enforce_dense_features()
+
+                encoder = self.encoder
+
+                if not encoder:
+                    log.info('Applying one-hot encoding...')
+                    one_hot_encoder = SkLearnOneHotEncoder(handle_unknown='ignore', sparse_output=False)
+                    transformers = [('one_hot_encoder', one_hot_encoder, nominal_indices)]
+                    encoder = ColumnTransformer(transformers, remainder='passthrough')
+                    encoder.fit(dataset.x, dataset.y)
+                    self.encoder = encoder
+
+                return replace(dataset, x=encoder.transform(dataset.x), features=[])
+
+            log.debug('No need to apply one-hot encoding, as the dataset does not contain any nominal features.')
+            return dataset
+
+    def create_encoder(self) -> Preprocessor.Encoder:
+        """
+        See :func:`mlrl.testbed.experiments.input.dataset.preprocessors.Preprocessor.create_encoder`
+        """
+        return OneHotEncoder.Encoder()

mlrl/testbed_sklearn/experiments/input/dataset/splitters/extension.py

@@ -0,0 +1,115 @@
+"""
+Author: Michael Rapp (michael.rapp.ml@gmail.com)
+
+Provides classes that allow configuring the functionality to split datasets into training and test datasets.
+"""
+from argparse import Namespace
+from typing import Set
+
+from mlrl.testbed_sklearn.experiments.input.dataset.preprocessors.extension import PreprocessorExtension
+from mlrl.testbed_sklearn.experiments.input.dataset.splitters.splitter_bipartition import BipartitionSplitter
+from mlrl.testbed_sklearn.experiments.input.dataset.splitters.splitter_cross_validation import CrossValidationSplitter
+
+from mlrl.testbed.experiments.input.dataset.splitters.extension import DatasetFileExtension
+from mlrl.testbed.experiments.input.dataset.splitters.splitter import DatasetSplitter
+from mlrl.testbed.experiments.input.dataset.splitters.splitter_no import NoSplitter
+from mlrl.testbed.extensions.extension import Extension
+
+from mlrl.util.cli import NONE, Argument, IntArgument, SetArgument
+from mlrl.util.validation import assert_greater, assert_greater_or_equal, assert_less, assert_less_or_equal
+
+VALUE_TRAIN_TEST = 'train-test'
+
+OPTION_TEST_SIZE = 'test_size'
+
+VALUE_CROSS_VALIDATION = 'cross-validation'
+
+OPTION_NUM_FOLDS = 'num_folds'
+
+OPTION_FIRST_FOLD = 'last_fold'
+
+OPTION_LAST_FOLD = 'first_fold'
+
+
+class DatasetSplitterExtension(Extension):
+    """
+    An extension that configures the functionality to split tabular datasets into training and test datasets.
+    """
+
+    RANDOM_STATE = IntArgument(
+        '--random-state',
+        description='The seed to be used by random number generators. Must be at least 1.',
+        default=1,
+    )
+
+    DATASET_SPLITTER = SetArgument(
+        '--data-split',
+        description='The strategy to be used for splitting the available data into training and test sets.',
+        default=VALUE_TRAIN_TEST,
+        values={
+            NONE: {},
+            VALUE_TRAIN_TEST: {OPTION_TEST_SIZE},
+            VALUE_CROSS_VALIDATION: {OPTION_NUM_FOLDS, OPTION_FIRST_FOLD, OPTION_LAST_FOLD}
+        },
+    )
+
+    def __init__(self, *dependencies: Extension):
+        """
+        :param dependencies: Other extensions, this extension depends on
+        """
+        super().__init__(PreprocessorExtension(), DatasetFileExtension(), *dependencies)
+
+    def _get_arguments(self) -> Set[Argument]:
+        """
+        See :func:`mlrl.testbed.extensions.extension.Extension._get_arguments`
+        """
+        return {self.RANDOM_STATE, self.DATASET_SPLITTER}
+
+    @staticmethod
+    def get_random_state(args: Namespace) -> int:
+        """
+        Returns the seed to be used by random number generators.
+
+        :param args:    The command line arguments specified by the user
+        :return:        The seed to be used
+        """
+        random_state = DatasetSplitterExtension.RANDOM_STATE.get_value(args)
+        assert_greater_or_equal(DatasetSplitterExtension.RANDOM_STATE.name, random_state, 1)
+        return random_state
+
+    @staticmethod
+    def get_dataset_splitter(args: Namespace) -> DatasetSplitter:
+        """
+        Returns the `DatasetSplitter` to be used for splitting datasets into training and test datasets according to the
+        configuration.
+
+        :param args:    The command line arguments specified by the user
+        :return:        The `DatasetSplitter` to be used
+        """
+        dataset_reader = DatasetFileExtension.get_dataset_reader(args)
+        dataset_reader.add_preprocessors(*PreprocessorExtension.get_preprocessors(args))
+        dataset_splitter, options = DatasetSplitterExtension.DATASET_SPLITTER.get_value(args)
+
+        if dataset_splitter == VALUE_CROSS_VALIDATION:
+            num_folds = options.get_int(OPTION_NUM_FOLDS, 10)
+            assert_greater_or_equal(OPTION_NUM_FOLDS, num_folds, 2)
+            first_fold = options.get_int(OPTION_FIRST_FOLD, 1)
+            assert_greater_or_equal(OPTION_FIRST_FOLD, first_fold, 1)
+            assert_less_or_equal(OPTION_FIRST_FOLD, first_fold, num_folds)
+            last_fold = options.get_int(OPTION_LAST_FOLD, num_folds)
+            assert_greater_or_equal(OPTION_LAST_FOLD, last_fold, first_fold)
+            assert_less_or_equal(OPTION_LAST_FOLD, last_fold, num_folds)
+            return CrossValidationSplitter(dataset_reader,
+                                           num_folds=num_folds,
+                                           first_fold=first_fold - 1,
+                                           last_fold=last_fold,
+                                           random_state=DatasetSplitterExtension.get_random_state(args))
+        if dataset_splitter == VALUE_TRAIN_TEST:
+            test_size = options.get_float(OPTION_TEST_SIZE, 0.33)
+            assert_greater(OPTION_TEST_SIZE, test_size, 0)
+            assert_less(OPTION_TEST_SIZE, test_size, 1)
+            return BipartitionSplitter(dataset_reader,
+                                       test_size=test_size,
+                                       random_state=DatasetSplitterExtension.get_random_state(args))
+
+        return NoSplitter(dataset_reader)