useful-optimizer 0.1.1__py3-none-any.whl

opt/__init__.py

@@ -0,0 +1,6 @@
+"""Useful optimizers, a set of optimization algorithms."""
+
+from __future__ import annotations
+
+
+__version__ = "0.1.1"

opt/abstract_optimizer.py

@@ -0,0 +1,74 @@
+"""This module defines an abstract base class for optimizers."""
+
+from __future__ import annotations
+
+from abc import ABC
+from abc import abstractmethod
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from numpy import ndarray
+
+
+class AbstractOptimizer(ABC):
+    """An abstract base class for optimizers.
+
+    Args:
+        func (Callable): The objective function to be optimized.
+        lower_bound (float): The lower bound of the search space.
+        upper_bound (float): The upper bound of the search space.
+        dim (int): The dimensionality of the search space.
+        max_iter (int, optional): The maximum number of iterations for the optimization process. Defaults to 1000.
+
+    Attributes:
+        func (Callable): The objective function to be optimized.
+        lower_bound (float): The lower bound of the search space.
+        upper_bound (float): The upper bound of the search space.
+        dim (int): The dimensionality of the search space.
+        max_iter (int): The maximum number of iterations for the optimization process.
+        seed (int): The seed for the random number generator.
+        population_size (int): The number of individuals in the population.
+
+
+    Methods:
+        search() -> Tuple[TBestSolution, TBestFitness]:
+            Perform the optimization search.
+
+    Returns:
+        Tuple containing the best solution found and its corresponding fitness value.
+    """
+
+    def __init__(
+        self,
+        func: Callable[[ndarray], float],
+        lower_bound: float,
+        upper_bound: float,
+        dim: int,
+        max_iter: int = 1000,
+        seed: int | None = None,
+        population_size: int = 100,
+    ) -> None:
+        """Initialize the optimizer."""
+        self.func = func
+        self.lower_bound = lower_bound
+        self.upper_bound = upper_bound
+        self.dim = dim
+        self.max_iter = max_iter
+        if seed is None:
+            self.seed = np.random.default_rng(42).integers(0, 2**32)
+        else:
+            self.seed = seed
+        self.population_size = population_size
+
+    @abstractmethod
+    def search(self) -> tuple[np.ndarray, float]:
+        """Perform the optimization search.
+
+        Returns:
+            Tuple containing the best solution found and its corresponding fitness value.
+        """

opt/adadelta.py

@@ -0,0 +1,148 @@
+"""AdaDelta Optimizer.
+
+This module implements the AdaDelta optimizer, which is an extension of AdaGrad that
+seeks to reduce its sensitivity to the learning rate hyperparameter.
+
+AdaDelta is a gradient-based optimization algorithm that adapts the learning rate
+for each of the parameters in the model. It is designed to converge faster than
+AdaGrad by using a moving average of the squared gradient values to scale the learning rate.
+
+The AdaDelta optimizer is defined by the following update rule:
+
+    Eg = rho * Eg + (1 - rho) * g^2
+    dx = -sqrt(Edx + eps) / sqrt(Eg + eps) * g
+    Edx = rho * Edx + (1 - rho) * dx^2
+    x = x + dx
+
+where:
+    - x: current solution
+    - g: gradient of the objective function
+    - rho: decay rate
+    - eps: small constant to avoid dividing by zero
+    - Eg: moving average of squared gradient values
+    - Edx: moving average of squared updates
+
+The algorithm iteratively updates the solution x by computing the gradient of the
+objective function at x, scaling it by the moving average of the squared gradients,
+and dividing it by the square root of the moving average of the squared updates.
+
+The algorithm continues for a fixed number of iterations or until a specified
+stopping criterion is met, returning the best solution found.
+
+This module provides a simple example of how to use the AdaDelta optimizer to minimize
+the Shifted Ackley's function in two dimensions.
+
+Example:
+    $ python adadelta.py
+
+Attributes:
+    rho (float): Decay rate for the moving average of squared gradients
+    eps (float): Small constant to avoid dividing by zero
+    Eg (np.ndarray): Moving average of squared gradient values
+    Edx (np.ndarray): Moving average of squared updates
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from scipy.optimize import approx_fprime
+
+from opt.abstract_optimizer import AbstractOptimizer
+from opt.benchmark.functions import shifted_ackley
+
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from numpy import ndarray
+
+
+class AdaDelta(AbstractOptimizer):
+    """AdaDelta optimizer implementation.
+
+    Args:
+        func (Callable[[ndarray], float]): The objective function to be minimized.
+        lower_bound (float): The lower bound of the search space.
+        upper_bound (float): The upper bound of the search space.
+        dim (int): The dimensionality of the search space.
+        max_iter (int, optional): The maximum number of iterations. Defaults to 1000.
+        seed (Optional[int], optional): The seed value for random number generation. Defaults to None.
+        rho (float, optional): The decay rate for the moving average of squared gradients. Defaults to 0.97.
+        eps (float, optional): A small constant to avoid division by zero. Defaults to 1e-8.
+    """
+
+    def __init__(
+        self,
+        func: Callable[[ndarray], float],
+        lower_bound: float,
+        upper_bound: float,
+        dim: int,
+        max_iter: int = 1000,
+        rho: float = 0.97,
+        eps: float = 1e-8,
+        seed: int | None = None,
+    ) -> None:
+        """Initialize the AdaDelta optimizer."""
+        super().__init__(
+            func=func,
+            lower_bound=lower_bound,
+            upper_bound=upper_bound,
+            dim=dim,
+            max_iter=max_iter,
+            seed=seed,
+        )
+        self.rho = rho
+        self.eps = eps
+        self.Eg = np.zeros(self.dim)
+        self.Edx = np.zeros(self.dim)
+
+    def _update(self, x: np.ndarray) -> np.ndarray:
+        """Perform a single update step of the AdaDelta algorithm.
+
+        Args:
+            x (np.ndarray): The current solution.
+
+        Returns:
+            np.ndarray: The updated solution.
+        """
+        g = approx_fprime(x, self.func, self.eps)
+        self.Eg = self.rho * self.Eg + (1 - self.rho) * g**2
+        dx = -np.sqrt(self.Edx + self.eps) / np.sqrt(self.Eg + self.eps) * g
+        self.Edx = self.rho * self.Edx + (1 - self.rho) * dx**2
+        return x + dx
+
+    def search(self) -> tuple[np.ndarray, float]:
+        """Perform the AdaDelta search.
+
+        Returns:
+            Tuple[np.ndarray, float]: A tuple containing the best solution found and its fitness value.
+        """
+        # Initialize solution
+        x = np.random.default_rng(self.seed).uniform(
+            self.lower_bound, self.upper_bound, self.dim
+        )
+        best_solution = x
+        best_fitness = np.inf
+
+        # Main loop
+        for _ in range(self.max_iter):
+            x = self._update(x)
+            x = np.clip(x, self.lower_bound, self.upper_bound)
+            fitness = self.func(x)
+            if fitness < best_fitness:
+                best_fitness = fitness
+                best_solution = x
+
+        return best_solution, best_fitness
+
+
+if __name__ == "__main__":
+    optimizer = AdaDelta(
+        func=shifted_ackley, lower_bound=-2.768, upper_bound=+2.768, dim=2
+    )
+    best_solution, best_fitness = optimizer.search()
+    print(f"Best solution found: {best_solution}")
+    print(f"Best fitness value: {best_fitness}")

opt/adagrad.py

@@ -0,0 +1,111 @@
+"""ADAGrad Optimizer.
+
+This module implements the Adaptive Gradient Algorithm (ADAGrad) optimizer. ADAGrad is
+a gradient-based optimization algorithm that adapts the learning rate to the parameters,
+performing smaller updates for parameters associated with frequently occurring features,
+and larger updates for parameters associated with infrequent features. It is particularly
+useful for dealing with sparse data.
+
+ADAGrad's main strength is that it eliminates the need to manually tune the learning rate.
+Most implementations also include a 'smoothing term' to avoid division by zero when the
+gradient is zero.
+
+The ADAGrad optimizer is commonly used in machine learning and deep learning applications.
+
+Example:
+    optimizer = ADAGrad(func=objective_function, learning_rate=0.01, initial_guess=[0, 0])
+    best_solution, best_fitness = optimizer.optimize()
+
+Attributes:
+    func (Callable): The objective function to optimize.
+    learning_rate (float): The learning rate for the optimization.
+    initial_guess (List[float]): The starting point for the optimization.
+
+Methods:
+    optimize(): Perform the ADAGrad optimization.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from scipy.optimize import approx_fprime
+
+from opt.abstract_optimizer import AbstractOptimizer
+from opt.benchmark.functions import shifted_ackley
+
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from numpy import ndarray
+
+
+class ADAGrad(AbstractOptimizer):
+    """ADAGrad optimizer implementation.
+
+    Args:
+        func (Callable[[ndarray], float]): The objective function to be optimized.
+        lower_bound (float): The lower bound of the search space.
+        upper_bound (float): The upper bound of the search space.
+        dim (int): The dimensionality of the search space.
+        max_iter (int, optional): The maximum number of iterations. Defaults to 1000.
+        seed (Optional[int], optional): The seed value for random number generation. Defaults to None.
+        lr (float, optional): The learning rate. Defaults to 0.01.
+        eps (float, optional): A small value added to the denominator for numerical stability. Defaults to 1e-8.
+    """
+
+    def __init__(
+        self,
+        func: Callable[[ndarray], float],
+        lower_bound: float,
+        upper_bound: float,
+        dim: int,
+        max_iter: int = 1000,
+        lr: float = 0.01,
+        eps: float = 1e-8,
+        seed: int | None = None,
+    ) -> None:
+        """Initialize the ADAGrad optimizer."""
+        super().__init__(
+            func=func,
+            lower_bound=lower_bound,
+            upper_bound=upper_bound,
+            dim=dim,
+            max_iter=max_iter,
+            seed=seed,
+        )
+        self.lr = lr
+        self.eps = eps
+
+    def search(self) -> tuple[np.ndarray, float]:
+        """Perform the ADAGrad search algorithm.
+
+        Returns:
+            Tuple[np.ndarray, float]: A tuple containing the best solution found and its fitness value.
+        """
+        x = np.random.default_rng(self.seed).uniform(
+            self.lower_bound, self.upper_bound, self.dim
+        )
+        grad_accumulator = np.zeros(self.dim)
+
+        for _ in range(self.max_iter):
+            grad = approx_fprime(x, self.func, np.sqrt(np.finfo(float).eps))
+            grad_accumulator += grad**2
+            adjusted_grad = grad / (np.sqrt(grad_accumulator) + self.eps)
+            x = x - self.lr * adjusted_grad
+
+        best_solution = x
+        best_fitness = self.func(best_solution)
+        return best_solution, best_fitness
+
+
+if __name__ == "__main__":
+    optimizer = ADAGrad(
+        func=shifted_ackley, lower_bound=-2.768, upper_bound=2.768, dim=2
+    )
+    best_solution, best_fitness = optimizer.search()
+    print(f"Best solution found: {best_solution}")
+    print(f"Best fitness value: {best_fitness}")

opt/adaptive_moment_estimation.py

@@ -0,0 +1,140 @@
+"""Adaptive Moment Estimation (Adam) Optimizer.
+
+This module implements the Adam optimization algorithm. Adam is a gradient-based
+optimization algorithm that computes adaptive learning rates for each parameter. It
+combines the advantages of two other extensions of stochastic gradient descent:
+
+    - AdaGrad
+    - RMSProp
+
+Adam works well in practice and compares favorably to other adaptive learning-method
+algorithms as it converges fast and the learning speed of the Model is quite fast and
+efficient. It is straightforward to implement, is computationally efficient, has little
+memory requirements, is invariant to diagonal rescaling of the gradients, and is well
+suited for problems that are large in terms of data and/or parameters.
+
+Example:
+    optimizer = Adam(func=objective_function, learning_rate=0.01, initial_guess=[0, 0])
+    best_solution, best_fitness = optimizer.optimize()
+
+Attributes:
+    func (Callable): The objective function to optimize.
+    learning_rate (float): The learning rate for the optimization.
+    initial_guess (List[float]): The starting point for the optimization.
+
+Methods:
+    optimize(): Perform the Adam optimization.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from scipy.optimize import approx_fprime
+
+from opt.abstract_optimizer import AbstractOptimizer
+from opt.benchmark.functions import shifted_ackley
+
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from numpy import ndarray
+
+
+class ADAMOptimization(AbstractOptimizer):
+    """ADAMOptimization is a class that implements the ADAM optimization algorithm.
+
+    Args:
+        func (Callable[[ndarray], float]): The objective function to be optimized.
+        lower_bound (float): The lower bound of the search space.
+        upper_bound (float): The upper bound of the search space.
+        dim (int): The dimensionality of the search space.
+        max_iter (int, optional): The maximum number of iterations. Defaults to 1000.
+        alpha (float, optional): The learning rate. Defaults to 0.001.
+        beta1 (float, optional): The exponential decay rate for the first moment estimates. Defaults to 0.9.
+        beta2 (float, optional): The exponential decay rate for the second moment estimates. Defaults to 0.999.
+        epsilon (float, optional): A small value added to the denominator for numerical stability. Defaults to 1e-13.
+        seed (Optional[int], optional): The random seed for reproducibility. Defaults to None.
+    """
+
+    def __init__(
+        self,
+        func: Callable[[ndarray], float],
+        lower_bound: float,
+        upper_bound: float,
+        dim: int,
+        max_iter: int = 1000,
+        alpha: float = 0.001,
+        beta1: float = 0.9,
+        beta2: float = 0.999,
+        epsilon: float = 1e-13,
+        seed: int | None = None,
+    ) -> None:
+        """Initialize the ADAM optimization algorithm."""
+        super().__init__(
+            func=func,
+            lower_bound=lower_bound,
+            upper_bound=upper_bound,
+            dim=dim,
+            max_iter=max_iter,
+            seed=seed,
+        )
+        self.alpha = alpha
+        self.beta1 = beta1
+        self.beta2 = beta2
+        self.epsilon = epsilon
+
+    def search(self) -> tuple[np.ndarray, float]:
+        """Perform the ADAM optimization search.
+
+        Returns:
+            Tuple[np.ndarray, float]: A tuple containing the best solution found and its fitness value.
+        """
+        m = np.zeros(self.dim)
+        v = np.zeros(self.dim)
+        best_solution = np.random.default_rng(self.seed).uniform(
+            self.lower_bound, self.upper_bound, self.dim
+        )
+        best_fitness = self.func(best_solution)
+
+        for t in range(1, self.max_iter + 1):
+            grad = self._compute_gradient(best_solution)
+            m = self.beta1 * m + (1 - self.beta1) * grad
+            v = self.beta2 * v + (1 - self.beta2) * np.square(grad)
+            m_hat = m / (1 - np.power(self.beta1, t))
+            v_hat = v / (1 - np.power(self.beta2, t))
+
+            best_solution = best_solution - self.alpha * m_hat / (
+                np.sqrt(v_hat) + self.epsilon
+            )
+            best_solution = np.clip(best_solution, self.lower_bound, self.upper_bound)
+
+            fitness = self.func(best_solution)
+            if fitness < best_fitness:
+                best_fitness = fitness
+
+        return best_solution, best_fitness
+
+    def _compute_gradient(self, x: np.ndarray) -> np.ndarray:
+        """Compute the gradient of the objective function at a given point.
+
+        Args:
+            x (np.ndarray): The point at which to compute the gradient.
+
+        Returns:
+            np.ndarray: The gradient vector.
+        """
+        epsilon = np.sqrt(np.finfo(float).eps)
+        return approx_fprime(x, self.func, epsilon)
+
+
+if __name__ == "__main__":
+    optimizer = ADAMOptimization(
+        func=shifted_ackley, lower_bound=-2.768, upper_bound=+2.768, dim=2
+    )
+    best_solution, best_fitness = optimizer.search()
+    print(f"Best solution: {best_solution}")
+    print(f"Best fitness: {best_fitness}")

opt/ant_colony.py

@@ -0,0 +1,194 @@
+"""Ant Colony Optimization (ACO) Algorithm.
+
+This module implements the Ant Colony Optimization (ACO) algorithm. ACO is a
+population-based metaheuristic that can be used to find approximate solutions to
+difficult optimization problems.
+
+In ACO, a set of software agents called artificial ants search for good solutions to a
+given optimization problem. To apply ACO, the optimization problem is transformed into
+the problem of finding the best path on a weighted graph. The artificial ants
+incrementally build solutions by moving on the graph. The solution construction process
+ is stochastic and is biased by a pheromone model, that is, a set of parameters
+associated with graph components (either nodes or edges) whose values are modified
+at runtime by the ants.
+
+ACO is particularly useful for problems that can be reduced to finding paths on
+weighted graphs, like the traveling salesman problem, the vehicle routing problem, and
+the quadratic assignment problem.
+
+Example:
+    optimizer = AntColony(func=objective_function, lower_bound=-10, upper_bound=10,
+    dim=2, n_ants=50, max_iter=1000)
+    best_solution, best_fitness = optimizer.search()
+
+Attributes:
+    func (Callable): The objective function to optimize.
+    lower_bound (float): The lower bound of the search space.
+    upper_bound (float): The upper bound of the search space.
+    dim (int): The dimension of the search space.
+    n_ants (int): The number of ants (candidate solutions).
+    max_iter (int): The maximum number of iterations.
+
+Methods:
+    search(): Perform the ACO optimization.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from opt.abstract_optimizer import AbstractOptimizer
+from opt.benchmark.functions import shifted_ackley
+
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from numpy import ndarray
+
+
+class AntColony(AbstractOptimizer):
+    """Ant Colony Optimization algorithm for optimization problems.
+
+    Args:
+        func (Callable[[ndarray], float]): The objective function to be minimized.
+        lower_bound (float): The lower bound of the search space.
+        upper_bound (float): The upper bound of the search space.
+        dim (int): The dimensionality of the search space.
+        max_iter (int, optional): The maximum number of iterations. Defaults to 1000.
+        population_size (int, optional): The number of ants in the colony. Defaults to 100.
+        seed (Optional[int], optional): The seed value for random number generation. Defaults to None.
+        alpha (float, optional): The importance of pheromone in the solution construction. Defaults to 1.
+        beta (float, optional): The importance of heuristic information in the solution construction. Defaults to 1.
+        rho (float, optional): The pheromone evaporation rate. Defaults to 0.5.
+        q (float, optional): The pheromone deposit factor. Defaults to 1.
+
+    Attributes:
+        alpha (float): The importance of pheromone in the solution construction.
+        beta (float): The importance of heuristic information in the solution construction.
+        rho (float): The pheromone evaporation rate.
+        q (float): The pheromone deposit factor.
+        ants (np.ndarray): The current solutions generated by the ants.
+        pheromone (np.ndarray): The pheromone matrix.
+
+    """
+
+    def __init__(
+        self,
+        func: Callable[[ndarray], float],
+        lower_bound: float,
+        upper_bound: float,
+        dim: int,
+        max_iter: int = 1000,
+        population_size: int = 100,
+        seed: int | None = None,
+        alpha: float = 1,
+        beta: float = 1,
+        rho: float = 0.5,
+        q: float = 1,
+    ) -> None:
+        """Initialize the Ant Colony Optimization algorithm."""
+        super().__init__(
+            func=func,
+            lower_bound=lower_bound,
+            upper_bound=upper_bound,
+            dim=dim,
+            max_iter=max_iter,
+            seed=seed,
+            population_size=population_size,
+        )
+
+        self.alpha = alpha
+        self.beta = beta
+        self.rho = rho
+        self.q = q
+        self.ants = np.random.default_rng(self.seed).uniform(
+            self.lower_bound, self.upper_bound, (self.population_size, self.dim)
+        )
+        self.pheromone = np.ones((self.population_size, self.dim))
+
+    def search(self) -> tuple[np.ndarray, float]:
+        """Run the Ant Colony Optimization algorithm.
+
+        Returns:
+            Tuple[np.ndarray, float]: The best solution found and its corresponding fitness value.
+
+        """
+        best_fitness = np.inf
+        best_solution = None
+
+        for _ in range(self.max_iter):
+            for i in range(self.population_size):
+                solution = self.ants[i]
+                fitness = self.func(solution)
+
+                if fitness < best_fitness:
+                    best_fitness = fitness
+                    best_solution = solution
+
+                self.update_pheromone(i, fitness)
+                self.ants[i] = self.generate_new_solution(i)
+
+            # Local search
+            for i in range(self.population_size):
+                local_best = self.local_search(self.ants[i])
+                local_fitness = self.func(local_best)
+
+                if local_fitness < best_fitness:
+                    best_fitness = local_fitness
+                    best_solution = local_best
+
+        return best_solution, best_fitness
+
+    def update_pheromone(self, i: int, fitness: float) -> None:
+        """Update the pheromone matrix based on the fitness of the ant's solution.
+
+        Args:
+            i (int): The index of the ant.
+            fitness (float): The fitness value of the ant's solution.
+
+        """
+        self.pheromone[i] = (1 - self.rho) * self.pheromone[i] + self.rho * (
+            self.q / fitness
+        )
+
+    def generate_new_solution(self, i: int) -> np.ndarray:
+        """Generate a new solution for the ant.
+
+        Args:
+            i (int): The index of the ant.
+
+        Returns:
+            np.ndarray: The new solution generated for the ant.
+
+        """
+        new_solution = self.ants[i] + self.pheromone[
+            i
+        ] ** self.alpha * np.random.default_rng(self.seed).uniform(-1, 1, self.dim)
+        return np.clip(new_solution, self.lower_bound, self.upper_bound)
+
+    def local_search(self, solution: np.ndarray) -> np.ndarray:
+        """Perform a local search by adding a small perturbation to the solution.
+
+        Args:
+            solution (np.ndarray): The solution to perform local search on.
+
+        Returns:
+            np.ndarray: The new solution after local search.
+
+        """
+        new_solution = solution + np.random.default_rng(self.seed).uniform(
+            -0.01, 0.01, self.dim
+        )
+        return np.clip(new_solution, self.lower_bound, self.upper_bound)
+
+
+if __name__ == "__main__":
+    optimizer = AntColony(
+        func=shifted_ackley, lower_bound=-2.768, upper_bound=+2.768, dim=2
+    )
+    best_solution, best_fitness = optimizer.search()
+    print(f"Best solution found: {best_solution}")
+    print(f"Best fitness value: {best_fitness}")