chaine 4.0.0b2__cp314-cp314-musllinux_1_2_x86_64.whl

chaine/optimization/spaces.py

@@ -0,0 +1,394 @@
+"""
+chaine.optimization.spaces
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This module implements hyperparameter search spaces for the different training methods.
+"""
+
+import random
+from abc import ABC, abstractmethod
+
+from chaine.optimization.utils import NumberSeries
+
+
+class SearchSpace(ABC):
+    @property
+    @abstractmethod
+    def algorithm(self) -> str:
+        ...
+
+    @abstractmethod
+    def random_hyperparameters(self) -> dict[str, int | float | bool | str]:
+        ...
+
+
+class LBFGSSearchSpace(SearchSpace):
+    def __init__(
+        self,
+        min_freq: NumberSeries = NumberSeries(start=0, stop=5, step=1),
+        num_memories: NumberSeries = NumberSeries(start=1, stop=10, step=1),
+        c1: NumberSeries = NumberSeries(start=0.0, stop=2.0, step=0.01),
+        c2: NumberSeries = NumberSeries(start=0.0, stop=2.0, step=0.01),
+        epsilon: NumberSeries = NumberSeries(start=0.00001, stop=0.001, step=0.00001),
+        period: NumberSeries = NumberSeries(start=1, stop=20, step=1),
+        delta: NumberSeries = NumberSeries(start=0.00001, stop=0.001, step=0.00001),
+        max_linesearch: NumberSeries = NumberSeries(start=0, stop=50, step=1),
+        linesearch: set[str] = {"MoreThuente", "Backtracking", "StrongBacktracking"},
+        all_possible_states: set[bool] = {True, False},
+        all_possible_transitions: set[bool] = {True, False},
+    ):
+        """Hyperparameter search space for Limited-Memory BFGS.
+
+        Parameters
+        ----------
+        min_freq : NumberSeries, optional
+            Threshold value for minimum frequency of a feature occurring in training data,
+            by default NumberSeries(start=0, stop=5, step=1).
+        num_memories : NumberSeries, optional
+            Number of limited memories for approximating the inverse hessian matrix,
+            by default NumberSeries(start=1, stop=10, step=1)
+        c1 : NumberSeries, optional
+            Coefficient for L1 regularization,
+            by default NumberSeries(start=0.0, stop=2.0, step=0.01).
+        c2 : NumberSeries, optional
+            Coefficient for L2 regularization,
+            by default NumberSeries(start=0.0, stop=2.0, step=0.01).
+        epsilon : NumberSeries, optional
+            Parameter that determines the condition of convergence,
+            by default NumberSeries(start=0.00001, stop=0.001, step=0.00001).
+        period : NumberSeries, optional
+            Threshold value for iterations to test the stopping criterion,
+            by default NumberSeries(start=1, stop=20, step=1).
+        delta : NumberSeries, optional
+            Top iteration when log likelihood is not greater than this,
+            by default NumberSeries(start=0.00001, stop=0.001, step=0.00001).
+        max_linesearch : NumberSeries, optional
+            Maximum number of trials for the line search algorithm,
+            by default NumberSeries(start=0, stop=50, step=1).
+        linesearch : set[str], optional
+            Line search algorithm used in updates,
+            by default {"MoreThuente", "Backtracking", "StrongBacktracking"}.
+        all_possible_states : set[bool], optional
+            Generate state features that do not even occur in the training data,
+            by default {True, False}.
+        all_possible_transitions : set[bool], optional
+            Generate transition features that do not even occur in the training data,
+            by default {True, False}.
+        """
+        self.min_freq = min_freq
+        self.all_possible_states = all_possible_states
+        self.all_possible_transitions = all_possible_transitions
+        self.num_memories = num_memories
+        self.c1 = c1
+        self.c2 = c2
+        self.epsilon = epsilon
+        self.period = period
+        self.delta = delta
+        self.linesearch = linesearch
+        self.max_linesearch = max_linesearch
+
+    @property
+    def algorithm(self) -> str:
+        return "lbfgs"
+
+    def random_hyperparameters(self) -> dict[str, int | float | bool | str]:
+        """Select random hyperparameters from the search space.
+
+        Returns
+        -------
+        dict[str, int | float | bool | str]
+            Randomly selected hyperparameters.
+        """
+        return {
+            "algorithm": self.algorithm,
+            "min_freq": random.choice(list(self.min_freq)),
+            "all_possible_states": random.choice(list(self.all_possible_states)),
+            "all_possible_transitions": random.choice(list(self.all_possible_transitions)),
+            "num_memories": random.choice(list(self.num_memories)),
+            "c1": random.choice(list(self.c1)),
+            "c2": random.choice(list(self.c2)),
+            "epsilon": random.choice(list(self.epsilon)),
+            "period": random.choice(list(self.period)),
+            "delta": random.choice(list(self.delta)),
+            "linesearch": random.choice(list(self.linesearch)),
+            "max_linesearch": random.choice(list(self.max_linesearch)),
+        }
+
+
+class L2SGDSearchSpace(SearchSpace):
+    def __init__(
+        self,
+        min_freq: NumberSeries = NumberSeries(start=0, stop=5, step=1),
+        all_possible_states: set[bool] = {True, False},
+        all_possible_transitions: set[bool] = {True, False},
+        c2: NumberSeries = NumberSeries(start=0.0, stop=2.0, step=0.01),
+        period: NumberSeries = NumberSeries(start=1, stop=20, step=1),
+        delta: NumberSeries = NumberSeries(start=0.00001, stop=0.001, step=0.00001),
+        calibration_eta: NumberSeries = NumberSeries(start=0.00001, stop=0.001, step=0.00001),
+        calibration_rate: NumberSeries = NumberSeries(start=0.5, stop=5.0, step=0.1),
+        calibration_samples: NumberSeries = NumberSeries(start=100, stop=3000, step=10),
+        calibration_candidates: NumberSeries = NumberSeries(start=1, stop=30, step=1),
+        calibration_max_trials: NumberSeries = NumberSeries(start=1, stop=30, step=1),
+    ):
+        """Hyperparameter search space for SGD with L2 parameters.
+
+        Parameters
+        ----------
+        min_freq : NumberSeries, optional
+            Threshold value for minimum frequency of a feature occurring in training data,
+            by default NumberSeries(start=0, stop=5, step=1).
+        all_possible_states : set[bool], optional
+            Generate state features that do not even occur in the training data,
+            by default {True, False}.
+        all_possible_transitions : set[bool], optional
+            Generate transition features that do not even occur in the training data,
+            by default {True, False}.
+        c2 : NumberSeries, optional
+            Coefficient for L2 regularization,
+            by default NumberSeries(start=0.0, stop=2.0, step=0.01).
+        period : NumberSeries, optional
+            Threshold value for iterations to test the stopping criterion,
+            by default NumberSeries(start=1, stop=20, step=1).
+        delta : NumberSeries, optional
+            Top iteration when log likelihood is not greater than this,
+            by default NumberSeries(start=0.00001, stop=0.001, step=0.00001).
+        calibration_eta : NumberSeries, optional
+            Initial value of learning rate (eta) used for calibration,
+            by default NumberSeries(start=0.00001, stop=0.001, step=0.00001).
+        calibration_rate : NumberSeries, optional
+            Rate of increase/decrease of learning rate for calibration,
+            by default NumberSeries(start=0.5, stop=5.0, step=0.1).
+        calibration_samples : NumberSeries, optional
+            Number of instances used for calibration,
+            by default NumberSeries(start=100, stop=3000, step=10).
+        calibration_candidates : NumberSeries, optional
+            Number of candidates of learning rate,
+            by default NumberSeries(start=1, stop=30, step=1).
+        calibration_max_trials : NumberSeries, optional
+            Maximum number of trials of learning rates for calibration,
+            by default NumberSeries(start=1, stop=30, step=1).
+        """
+        self.min_freq = min_freq
+        self.all_possible_states = all_possible_states
+        self.all_possible_transitions = all_possible_transitions
+        self.c2 = c2
+        self.period = period
+        self.delta = delta
+        self.calibration_eta = calibration_eta
+        self.calibration_rate = calibration_rate
+        self.calibration_samples = calibration_samples
+        self.calibration_candidates = calibration_candidates
+        self.calibration_max_trials = calibration_max_trials
+
+    @property
+    def algorithm(self) -> str:
+        return "l2sgd"
+
+    def random_hyperparameters(self) -> dict[str, int | float | bool | str]:
+        """Select random hyperparameters from the search space.
+
+        Returns
+        -------
+        dict[str, int | float | bool | str]
+            Randomly selected hyperparameters.
+        """
+        return {
+            "algorithm": self.algorithm,
+            "min_freq": random.choice(list(self.min_freq)),
+            "all_possible_states": random.choice(list(self.all_possible_states)),
+            "all_possible_transitions": random.choice(list(self.all_possible_transitions)),
+            "c2": random.choice(list(self.c2)),
+            "period": random.choice(list(self.period)),
+            "delta": random.choice(list(self.delta)),
+            "calibration_eta": random.choice(list(self.calibration_eta)),
+            "calibration_rate": random.choice(list(self.calibration_rate)),
+            "calibration_samples": random.choice(list(self.calibration_samples)),
+            "calibration_candidates": random.choice(list(self.calibration_candidates)),
+            "calibration_max_trials": random.choice(list(self.calibration_max_trials)),
+        }
+
+
+class APSearchSpace(SearchSpace):
+    def __init__(
+        self,
+        min_freq: NumberSeries = NumberSeries(start=0, stop=5, step=1),
+        all_possible_states: set[bool] = {True, False},
+        all_possible_transitions: set[bool] = {True, False},
+        epsilon: NumberSeries = NumberSeries(start=0.00001, stop=0.001, step=0.00001),
+    ):
+        """Hyperparameter search space for Averaged Perceptron.
+
+        Parameters
+        ----------
+        min_freq : NumberSeries, optional
+            Threshold value for minimum frequency of a feature occurring in training data,
+            by default NumberSeries(start=0, stop=5, step=1).
+        all_possible_states : set[bool], optional
+            Generate state features that do not even occur in the training data,
+            by default {True, False}.
+        all_possible_transitions : set[bool], optional
+            Generate transition features that do not even occur in the training data,
+            by default {True, False}.
+        epsilon : NumberSeries, optional
+            Parameter that determines the condition of convergence,
+            by default NumberSeries(start=0.00001, stop=0.001, step=0.00001).
+        """
+        self.min_freq = min_freq
+        self.all_possible_states = all_possible_states
+        self.all_possible_transitions = all_possible_transitions
+        self.epsilon = epsilon
+
+    @property
+    def algorithm(self) -> str:
+        return "ap"
+
+    def random_hyperparameters(self) -> dict[str, int | float | bool | str]:
+        """Select random hyperparameters from the search space.
+
+        Returns
+        -------
+        dict[str, int | float | bool | str]
+            Randomly selected hyperparameters.
+        """
+        return {
+            "algorithm": self.algorithm,
+            "min_freq": random.choice(list(self.min_freq)),
+            "all_possible_states": random.choice(list(self.all_possible_states)),
+            "all_possible_transitions": random.choice(list(self.all_possible_transitions)),
+            "epsilon": random.choice(list(self.epsilon)),
+        }
+
+
+class PASearchSpace(SearchSpace):
+    def __init__(
+        self,
+        min_freq: NumberSeries = NumberSeries(start=0, stop=5, step=1),
+        all_possible_states: set[bool] = {True, False},
+        all_possible_transitions: set[bool] = {True, False},
+        epsilon: NumberSeries = NumberSeries(start=0.00001, stop=0.001, step=0.00001),
+        pa_type: NumberSeries = {0, 1, 2},
+        c: NumberSeries = NumberSeries(start=0.0, stop=2.0, step=0.01),
+        error_sensitive: set[bool] = {True, False},
+        averaging: set[bool] = {True, False},
+    ):
+        """Hyperparameter search space for Passive Aggressive.
+
+        Parameters
+        ----------
+        min_freq : NumberSeries, optional
+            Threshold value for minimum frequency of a feature occurring in training data,
+            by default NumberSeries(start=0, stop=5, step=1).
+        all_possible_states : set[bool], optional
+            Generate state features that do not even occur in the training data,
+            by default {True, False}.
+        all_possible_transitions : set[bool], optional
+            Generate transition features that do not even occur in the training data,
+            by default {True, False}.
+        epsilon : NumberSeries, optional
+            Parameter that determines the condition of convergence,
+            by default NumberSeries(start=0.00001, stop=0.001, step=0.00001).
+        pa_type : NumberSeries, optional
+            Strategy for updating feature weights, by default {0, 1, 2}.
+        c : NumberSeries, optional
+            Aggressiveness parameter, by default NumberSeries(start=0.0, stop=2.0, step=0.01).
+        error_sensitive : set[bool], optional
+            Include square root of predicted incorrect labels into optimization routine,
+            by default {True, False}.
+        averaging : set[bool], optional
+            Compute average of feature weights at all updates, by default {True, False}.
+        """
+        self.min_freq = min_freq
+        self.all_possible_states = all_possible_states
+        self.all_possible_transitions = all_possible_transitions
+        self.epsilon = epsilon
+        self.pa_type = pa_type
+        self.c = c
+        self.error_sensitive = error_sensitive
+        self.averaging = averaging
+
+    @property
+    def algorithm(self) -> str:
+        return "pa"
+
+    def random_hyperparameters(self) -> dict[str, int | float | bool | str]:
+        """Select random hyperparameters from the search space.
+
+        Returns
+        -------
+        dict[str, int | float | bool | str]
+            Randomly selected hyperparameters.
+        """
+        return {
+            "algorithm": self.algorithm,
+            "min_freq": random.choice(list(self.min_freq)),
+            "all_possible_states": random.choice(list(self.all_possible_states)),
+            "all_possible_transitions": random.choice(list(self.all_possible_transitions)),
+            "epsilon": random.choice(list(self.epsilon)),
+            "pa_type": random.choice(list(self.pa_type)),
+            "c": random.choice(list(self.c)),
+            "error_sensitive": random.choice(list(self.error_sensitive)),
+            "averaging": random.choice(list(self.averaging)),
+        }
+
+
+class AROWSearchSpace(SearchSpace):
+    def __init__(
+        self,
+        min_freq: NumberSeries = NumberSeries(start=0, stop=5, step=1),
+        all_possible_states: set[bool] = {True, False},
+        all_possible_transitions: set[bool] = {True, False},
+        epsilon: NumberSeries = NumberSeries(start=0.00001, stop=0.001, step=0.00001),
+        variance: NumberSeries = NumberSeries(start=0.00001, stop=0.001, step=0.00001),
+        gamma: NumberSeries = NumberSeries(start=0.00001, stop=0.001, step=0.00001),
+    ):
+        """Hyperparameter search space for AROW.
+
+        Parameters
+        ----------
+        min_freq : NumberSeries, optional
+            Threshold value for minimum frequency of a feature occurring in training data,
+            by default NumberSeries(start=0, stop=5, step=1).
+        all_possible_states : set[bool], optional
+            Generate state features that do not even occur in the training data,
+            by default {True, False}.
+        all_possible_transitions : set[bool], optional
+            Generate transition features that do not even occur in the training data,
+            by default {True, False}.
+        epsilon : NumberSeries, optional
+            Parameter that determines the condition of convergence,
+            by default NumberSeries(start=0.00001, stop=0.001, step=0.00001).
+        variance : NumberSeries, optional
+            Initial variance of every feature weight,
+            by default NumberSeries(start=0.00001, stop=0.001, step=0.00001).
+        gamma : NumberSeries, optional
+            Trade-off between loss function and changes of feature weights,
+            by default NumberSeries(start=0.00001, stop=0.001, step=0.00001).
+        """
+        self.min_freq = min_freq
+        self.all_possible_states = all_possible_states
+        self.all_possible_transitions = all_possible_transitions
+        self.epsilon = epsilon
+        self.variance = variance
+        self.gamma = gamma
+
+    @property
+    def algorithm(self) -> str:
+        return "arow"
+
+    def random_hyperparameters(self) -> dict[str, int | float | bool | str]:
+        """Select random hyperparameters from the search space.
+
+        Returns
+        -------
+        dict[str, int | float | bool | str]
+            Randomly selected hyperparameters.
+        """
+        return {
+            "algorithm": self.algorithm,
+            "min_freq": random.choice(list(self.min_freq)),
+            "all_possible_states": random.choice(list(self.all_possible_states)),
+            "all_possible_transitions": random.choice(list(self.all_possible_transitions)),
+            "epsilon": random.choice(list(self.epsilon)),
+            "variance": random.choice(list(self.variance)),
+            "gamma": random.choice(list(self.gamma)),
+        }

chaine/optimization/trial.py

@@ -0,0 +1,103 @@
+"""
+chaine.optimization.trial
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This module implements a class for a hyperparameter optimization trial.
+"""
+
+import statistics
+import tempfile
+import time
+import uuid
+from pathlib import Path
+
+from chaine.optimization.metrics import evaluate_predictions
+from chaine.optimization.spaces import SearchSpace
+from chaine.typing import Iterable, Iterator, Labels, Sequence
+
+
+class OptimizationTrial:
+    def __init__(
+        self,
+        splits: Iterator[tuple[tuple[Iterable[Sequence], Iterable[Labels]]]],
+        space: SearchSpace,
+        *,
+        is_baseline: bool
+    ):
+        """Hyperparameter optimization trial.
+
+        Parameters
+        ----------
+        splits : Iterator[tuple[tuple[Iterable[Sequence], Iterable[Labels]]]]
+            K-fold split data set.
+        space : SearchSpace
+            Search space for hyperparameter optimization.
+        is_baseline : bool
+            True if trial is a baseline (i.e. default hyperparameters to be used).
+        """
+        self.splits = splits
+        self.space = space
+        self.is_baseline = is_baseline
+        self.model_filepath = Path(tempfile.gettempdir(), str(uuid.uuid4()))
+        self.precision = []
+        self.recall = []
+        self.f1 = []
+        self.time = []
+
+    def __enter__(self) -> dict[str, dict]:
+        """Train and evaluate a model.
+
+        Returns
+        -------
+        dict[str, dict]
+            Selected hyperparameters and evaluation scores.
+        """
+        from chaine.crf import Model, Trainer
+
+        if self.is_baseline:
+            # default hyperparameters as baseline
+            params = {"algorithm": self.space.algorithm}
+        else:
+            # select random hyperparameters
+            params = self.space.random_hyperparameters()
+
+        for (train_dataset, train_labels), (test_dataset, test_labels) in self.splits:
+            # fire!
+            start = time.time()
+            trainer = Trainer(max_iterations=100, **params)
+            trainer.train(train_dataset, train_labels, model_filepath=self.model_filepath)
+            end = time.time()
+
+            # evaluate model
+            model = Model(self.model_filepath)
+            predicted_labels = model.predict(test_dataset)
+            scores = evaluate_predictions(test_labels, predicted_labels)
+
+            # save scores
+            self.precision.append(scores["precision"])
+            self.recall.append(scores["recall"])
+            self.f1.append(scores["f1"])
+            self.time.append(end - start)
+
+        # return both hyperparameters and evaluation metrics
+        return {
+            "hyperparameters": params,
+            "stats": {
+                "mean_precision": statistics.mean(self.precision) if self.precision else None,
+                "stdev_precision": statistics.stdev(self.precision) if self.precision else None,
+                "mean_recall": statistics.mean(self.recall) if self.recall else None,
+                "stdev_recall": statistics.stdev(self.recall) if self.recall else None,
+                "mean_f1": statistics.mean(self.f1) if self.f1 else None,
+                "stdev_f1": statistics.stdev(self.f1) if self.f1 else None,
+                "mean_time": statistics.mean(self.time) if self.time else None,
+                "stdev_time": statistics.stdev(self.time) if self.time else None,
+            },
+        }
+
+    def __exit__(self, *args) -> bool:
+        # clean up
+        if self.model_filepath.exists():
+            self.model_filepath.unlink()
+
+        # ignore exceptions
+        return True

chaine/optimization/utils.py

@@ -0,0 +1,119 @@
+"""
+chaine.optimization.utils
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This module implements utility functions for hyperparameter optimization.
+"""
+
+import random
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
+
+from chaine.typing import Labels, Sequence
+
+
+@dataclass
+class NumberSeries(Iterable):
+    start: int
+    stop: int
+    step: int | float
+
+    def __repr__(self) -> str:
+        return f"<NumberSeries (start={self.start}, stop={self.stop}, step={self.step})>"
+
+    def __iter__(self) -> Iterator[int | float]:
+        n = int(round((self.stop - self.start) / float(self.step)))
+        if n > 1:
+            yield from [self.start + self.step * i for i in range(n + 1)]
+        elif n == 1:
+            yield self.start
+
+
+def cross_validation(
+    dataset: Iterable[Sequence], labels: Iterable[Labels], k: int, seed: int | None = None
+) -> Iterator[tuple[tuple[Iterable[Sequence], Iterable[Labels]]]]:
+    """K-fold cross validation.
+
+    Parameters
+    ----------
+    dataset : Iterable[Sequence]
+        Data set to split into k folds.
+    labels : Iterable[Labels]
+        Labels to split into k folds.
+    k : int
+        Number of folds.
+    shuffle : bool, optional
+        True if data set should be shuffled first, by default True.
+
+    Yields
+    -------
+    Iterator[tuple[tuple[Iterable[Sequence], Iterable[Labels]]]]
+        Train and test set.
+    """
+    # get indices of the examples
+    indices = list(range(len(dataset)))
+
+    # shuffle examples
+    random.seed(seed)
+    random.shuffle(indices)
+
+    # split into k folds
+    folds = [indices[i::k] for i in range(k)]
+
+    # yield every fold split
+    for i in range(k):
+        # get train and test split
+        test = folds[i]
+        train = [s for x in [fold for fold in folds if fold != test] for s in x]
+
+        # yield train and test split
+        yield (
+            [d for i, d in enumerate(dataset) if i in train],
+            [l for i, l in enumerate(labels) if i in train],
+        ), (
+            [d for i, d in enumerate(dataset) if i in test],
+            [l for i, l in enumerate(labels) if i in test],
+        )
+
+
+def downsample(
+    dataset: Iterable[Sequence], labels: Iterable[Labels], n: int, seed: int | None = None
+) -> tuple[Iterable[Sequence], Iterable[Labels]]:
+    """Downsample the given data set to the specified size.
+
+    Parameters
+    ----------
+    dataset : Iterable[Sequence]
+        Data set to downsample.
+    labels : Iterable[Labels]
+        Labels for the data set.
+    n : int
+        Number of samples to keep.
+    seed : int | None, optional
+        Random seed, by default None.
+
+    Returns
+    -------
+    tuple[Iterable[Sequence], Iterable[Labels]]
+        Downsampled data set and labels.
+
+    Raises
+    ------
+    ValueError
+        If number of instances in the data set is smaller than specified size.
+    """
+    if len(dataset) < n:
+        raise ValueError("Data set is too small")
+
+    # get indices of the data set
+    indices = list(range(len(dataset)))
+
+    # sample indices
+    random.seed(seed)
+    sample = set(random.sample(indices, n))
+
+    # keep only instances of the sample
+    dataset = [s for i, s in enumerate(dataset) if i in sample]
+    labels = [l for i, l in enumerate(labels) if i in sample]
+
+    return dataset, labels