pyperch 0.2.2__tar.gz → 0.3.0__tar.gz

pyperch-0.3.0/PKG-INFO

@@ -0,0 +1,32 @@
+Metadata-Version: 2.4
+Name: pyperch
+Version: 0.3.0
+Summary: PyTorch-native randomized optimization algorithms.
+License: MIT
+Keywords: pytorch,optimization,machine-learning,randomized-optimization
+Author: John Mansfield
+Requires-Python: >=3.10,<=3.13
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: numpy (>=1.24)
+Requires-Dist: torch (>=2.1,<3.0)
+Project-URL: Homepage, https://github.com/jlm429/pyperch
+Project-URL: Repository, https://github.com/jlm429/pyperch
+Description-Content-Type: text/markdown
+
+# pyperch
+
+![PyPI](https://img.shields.io/pypi/v/pyperch.svg)
+![Python Versions](https://img.shields.io/pypi/pyversions/pyperch.svg)
+![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)
+![Code Style: Black](https://img.shields.io/badge/code%20style-black-000000.svg)
+![Linter: Ruff](https://img.shields.io/badge/lint-ruff-blue.svg)
+[![CircleCI](https://dl.circleci.com/status-badge/img/circleci/WH9eaoZnQRJ8SGFDrvqQAd/5meq6x5R3uDA3KSuHARdVk/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/circleci/WH9eaoZnQRJ8SGFDrvqQAd/5meq6x5R3uDA3KSuHARdVk/tree/master)
+
+
+A lightweight, modular library for neural network weight optimization using randomized search algorithms built directly on top of PyTorch.  Pyperch is a research and teaching-oriented library for training neural networks using randomized optimization methods (RHC, SA, GA), gradient-based methods, and hybrid combinations.
+

pyperch-0.3.0/README.md

@@ -0,0 +1,11 @@
+# pyperch
+
+![PyPI](https://img.shields.io/pypi/v/pyperch.svg)
+![Python Versions](https://img.shields.io/pypi/pyversions/pyperch.svg)
+![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)
+![Code Style: Black](https://img.shields.io/badge/code%20style-black-000000.svg)
+![Linter: Ruff](https://img.shields.io/badge/lint-ruff-blue.svg)
+[![CircleCI](https://dl.circleci.com/status-badge/img/circleci/WH9eaoZnQRJ8SGFDrvqQAd/5meq6x5R3uDA3KSuHARdVk/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/circleci/WH9eaoZnQRJ8SGFDrvqQAd/5meq6x5R3uDA3KSuHARdVk/tree/master)
+
+
+A lightweight, modular library for neural network weight optimization using randomized search algorithms built directly on top of PyTorch.  Pyperch is a research and teaching-oriented library for training neural networks using randomized optimization methods (RHC, SA, GA), gradient-based methods, and hybrid combinations.

pyperch-0.3.0/pyperch/__init__.py

@@ -0,0 +1,5 @@
+"""PyPerch randomized optimizer rebuild package."""
+
+from .optim import RHC, RandomizedOptimizer
+
+__all__ = ["RandomizedOptimizer", "RHC"]

pyperch-0.3.0/pyperch/optim/__init__.py

@@ -0,0 +1,11 @@
+from .base import RandomizedOptimizer
+from .ga import GA
+from .rhc import RHC
+from .sa import SA
+
+__all__ = [
+    "RandomizedOptimizer",
+    "RHC",
+    "SA",
+    "GA",
+]

pyperch-0.3.0/pyperch/optim/base.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+import torch
+
+
+class RandomizedOptimizer(torch.optim.Optimizer):
+    """Base class for randomized optimizers that operate on PyTorch parameters."""
+
+    def __init__(self, params: Iterable[torch.nn.Parameter], defaults: dict):
+        super().__init__(params, defaults)
+        self.reset_counters()
+
+    def reset_counters(self) -> None:
+        """Reset optimization counters without changing model parameters."""
+        self.function_evals = 0
+        self.proposed_steps = 0
+        self.accepted_steps = 0
+        self.rejected_steps = 0
+        self.best_loss: float | None = None
+
+    @torch.no_grad()
+    def _parameters(self) -> list[torch.nn.Parameter]:
+        """Return trainable parameters managed by this optimizer."""
+        return [
+            p for group in self.param_groups for p in group["params"] if p.requires_grad
+        ]
+
+    @torch.no_grad()
+    def _clone_params(self) -> list[torch.Tensor]:
+        """Copy the current trainable parameter values."""
+        return [p.detach().clone() for p in self._parameters()]
+
+    @torch.no_grad()
+    def _restore_params(self, values: list[torch.Tensor]) -> None:
+        """Restore trainable parameters from a copied parameter list."""
+        for p, value in zip(self._parameters(), values):
+            p.copy_(value)
+
+    def _record_eval(self, loss: float | None = None) -> None:
+        """Record one objective evaluation and optionally update the best loss."""
+        self.function_evals += 1
+
+        if loss is not None:
+            self._update_best_loss(loss)
+
+    def _update_best_loss(self, loss: float) -> None:
+        """Update the best loss when improved."""
+        if self.best_loss is None or loss < self.best_loss:
+            self.best_loss = loss

pyperch-0.3.0/pyperch/optim/ga.py

@@ -0,0 +1,284 @@
+"""Genetic Algorithm optimizer.
+
+Randomized Optimization methods for PyPerch.
+
+Based on the original PyPerch optimizers by Jakub Owczarek
+(BSD 3-Clause License).
+
+These were also inspired by ABAGAIL’s randomized optimization algorithms - https://github.com/pushkar/ABAGAIL.
+
+Substantial refactoring and redesign by John Mansfield (2026).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+import torch
+
+from .base import RandomizedOptimizer
+
+
+class GA(RandomizedOptimizer):
+    """Genetic algorithm optimizer for arbitrary PyTorch models.
+
+    This optimizer builds a population around the current parameters,
+    evaluates candidate solutions, applies selection, crossover, mutation,
+    and adopts the best candidate when it improves the loss.
+
+    Lower loss is assumed to be better.
+    """
+
+    def __init__(
+        self,
+        params,
+        population_size: int = 50,
+        mutation_rate: float = 0.1,
+        step_size: float = 0.1,
+        random_state: int | None = None,
+    ):
+        if population_size < 2:
+            raise ValueError("population_size must be at least 2.")
+        if mutation_rate < 0 or mutation_rate > 1:
+            raise ValueError("mutation_rate must be in the interval [0, 1].")
+        if step_size <= 0:
+            raise ValueError("step_size must be positive.")
+
+        defaults = {
+            "population_size": population_size,
+            "mutation_rate": mutation_rate,
+            "step_size": step_size,
+        }
+
+        super().__init__(params, defaults)
+
+        self.population_size = population_size
+        self.mutation_rate = mutation_rate
+        self.step_size = step_size
+
+        self._generator = torch.Generator()
+        if random_state is not None:
+            self._generator.manual_seed(random_state)
+
+        self._initialized = False
+        self._current_loss: float | None = None
+        self._best_params: list[torch.Tensor] | None = None
+
+    def step(self, closure: Callable[[], torch.Tensor]) -> torch.Tensor:
+        if closure is None:
+            raise ValueError("GA requires a closure that returns the loss.")
+
+        if not self._initialized:
+            loss_tensor = self._evaluate_loss(closure)
+            loss = float(loss_tensor.detach().item())
+
+            self._initialized = True
+            self._current_loss = loss
+            self._update_best_loss(loss)
+            self._best_params = self._clone_params()
+
+            return loss_tensor
+
+        current_params = self._clone_params()
+        current_loss = self._current_loss
+
+        population = self._initialize_population(current_params)
+        population_losses = self._evaluate_population(population, closure)
+
+        selected = self._select_population(population, population_losses)
+        children = self._crossover(selected)
+        children = self._mutate(children)
+
+        candidate_population = selected + children
+        candidate_losses = self._evaluate_population(candidate_population, closure)
+
+        best_idx = min(
+            range(len(candidate_losses)),
+            key=lambda idx: candidate_losses[idx],
+        )
+
+        best_candidate = candidate_population[best_idx]
+        best_candidate_loss = candidate_losses[best_idx]
+
+        self.proposed_steps += len(candidate_population)
+
+        if best_candidate_loss <= current_loss:
+            self._restore_params(best_candidate)
+            self.accepted_steps += 1
+            self._current_loss = best_candidate_loss
+
+            if self.best_loss is None or best_candidate_loss < self.best_loss:
+                self.best_loss = best_candidate_loss
+                self._best_params = self._clone_params()
+
+            return torch.tensor(best_candidate_loss)
+
+        self._restore_params(current_params)
+        self.rejected_steps += 1
+
+        return torch.tensor(current_loss)
+
+    def _evaluate_loss(
+        self,
+        closure: Callable[[], torch.Tensor],
+    ) -> torch.Tensor:
+        with torch.enable_grad():
+            loss_tensor = closure()
+
+        self._record_eval(float(loss_tensor.detach().item()))
+        return loss_tensor
+
+    @torch.no_grad()
+    def _initialize_population(
+        self,
+        base_params: list[torch.Tensor],
+    ) -> list[list[torch.Tensor]]:
+        population = [base_params]
+
+        for _ in range(self.population_size - 1):
+            individual = []
+
+            for param in base_params:
+                noise = torch.randn(
+                    param.shape,
+                    generator=self._generator,
+                    device=param.device,
+                    dtype=param.dtype,
+                )
+
+                individual.append(param + self.step_size * noise)
+
+            population.append(individual)
+
+        return population
+
+    def _evaluate_population(
+        self,
+        population: list[list[torch.Tensor]],
+        closure: Callable[[], torch.Tensor],
+    ) -> list[float]:
+        original = self._clone_params()
+        losses = []
+
+        for individual in population:
+            self._restore_params(individual)
+
+            with torch.enable_grad():
+                loss_tensor = closure()
+
+            loss = float(loss_tensor.detach().item())
+            self._record_eval()
+            losses.append(loss)
+
+        self._restore_params(original)
+        return losses
+
+    def _select_population(
+        self,
+        population: list[list[torch.Tensor]],
+        losses: list[float],
+    ) -> list[list[torch.Tensor]]:
+        keep_count = max(2, self.population_size // 2)
+
+        ranked_indices = sorted(
+            range(len(losses)),
+            key=lambda idx: losses[idx],
+        )
+
+        return [
+            self._clone_individual(population[idx])
+            for idx in ranked_indices[:keep_count]
+        ]
+
+    @torch.no_grad()
+    def _crossover(
+        self,
+        parents: list[list[torch.Tensor]],
+    ) -> list[list[torch.Tensor]]:
+        child_count = self.population_size - len(parents)
+        children = []
+
+        for _ in range(child_count):
+            idx1 = torch.randint(
+                low=0,
+                high=len(parents),
+                size=(1,),
+                generator=self._generator,
+            ).item()
+
+            idx2 = torch.randint(
+                low=0,
+                high=len(parents),
+                size=(1,),
+                generator=self._generator,
+            ).item()
+
+            parent1 = parents[idx1]
+            parent2 = parents[idx2]
+
+            child = []
+
+            for p1, p2 in zip(parent1, parent2):
+                mask = (
+                    torch.rand(
+                        p1.shape,
+                        generator=self._generator,
+                        device=p1.device,
+                        dtype=p1.dtype,
+                    )
+                    < 0.5
+                )
+
+                child_param = torch.where(mask, p1, p2)
+                child.append(child_param)
+
+            children.append(child)
+
+        return children
+
+    @torch.no_grad()
+    def _mutate(
+        self,
+        population: list[list[torch.Tensor]],
+    ) -> list[list[torch.Tensor]]:
+        mutated = []
+
+        for individual in population:
+            new_individual = []
+
+            for param in individual:
+                mutation_mask = (
+                    torch.rand(
+                        param.shape,
+                        generator=self._generator,
+                        device=param.device,
+                        dtype=param.dtype,
+                    )
+                    < self.mutation_rate
+                )
+
+                noise = torch.randn(
+                    param.shape,
+                    generator=self._generator,
+                    device=param.device,
+                    dtype=param.dtype,
+                )
+
+                new_param = param + mutation_mask * self.step_size * noise
+                new_individual.append(new_param)
+
+            mutated.append(new_individual)
+
+        return mutated
+
+    @staticmethod
+    def _clone_individual(
+        individual: list[torch.Tensor],
+    ) -> list[torch.Tensor]:
+        return [param.detach().clone() for param in individual]
+
+    @torch.no_grad()
+    def restore_best(self) -> None:
+        """Restore the best parameters observed so far."""
+        if self._best_params is not None:
+            self._restore_params(self._best_params)

pyperch-0.3.0/pyperch/optim/rhc.py

@@ -0,0 +1,154 @@
+"""
+Randomized Optimization methods for PyPerch.
+
+Based on the original PyPerch optimizers by Jakub Owczarek
+(BSD 3-Clause License).
+
+These were also inspired by ABAGAIL’s randomized optimization algorithms - https://github.com/pushkar/ABAGAIL.
+
+Substantial refactoring and redesign by John Mansfield (2026).
+
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+import torch
+
+from .base import RandomizedOptimizer
+
+
+class RHC(RandomizedOptimizer):
+    """Randomized Hill Climbing optimizer for arbitrary PyTorch models.
+
+    The closure should return a scalar loss tensor. Lower loss is assumed to be
+    better. Gradients are not required.
+    """
+
+    def __init__(
+        self,
+        params,
+        step_size: float = 0.1,
+        restarts: int = 0,
+        restart_interval: int | None = None,
+        random_state: int | None = None,
+    ):
+        if step_size <= 0:
+            raise ValueError("step_size must be positive.")
+        if restarts < 0:
+            raise ValueError("restarts must be >= 0.")
+        if restart_interval is not None and restart_interval <= 0:
+            raise ValueError("restart_interval must be positive when provided.")
+
+        defaults = {"step_size": step_size}
+        super().__init__(params, defaults)
+
+        self.step_size = step_size
+        self.restarts = restarts
+        self.restart_interval = restart_interval
+        self.completed_restarts = 0
+
+        self._generator = torch.Generator()
+        if random_state is not None:
+            self._generator.manual_seed(random_state)
+
+        self._initialized = False
+        self._current_loss: float | None = None
+        self._best_params: list[torch.Tensor] | None = None
+
+    def step(self, closure: Callable[[], torch.Tensor]) -> torch.Tensor:
+        """Evaluate one candidate move and keep it if loss does not increase."""
+        if closure is None:
+            raise ValueError("RHC requires a closure that returns the loss.")
+
+        if not self._initialized or self._current_loss is None:
+            loss_tensor = self._evaluate(closure)
+            loss = float(loss_tensor.detach().item())
+            self._initialized = True
+            self._current_loss = loss
+            self._save_best_if_needed(loss)
+            return loss_tensor
+
+        old_params = self._clone_params()
+        old_loss = self._current_loss
+
+        self._propose_step()
+        self.proposed_steps += 1
+
+        candidate_loss_tensor = self._evaluate(closure)
+        candidate_loss = float(candidate_loss_tensor.detach().item())
+
+        if candidate_loss <= old_loss:
+            self._current_loss = candidate_loss
+            self.accepted_steps += 1
+            self._save_best_if_needed(candidate_loss)
+            return candidate_loss_tensor
+
+        self._restore_params(old_params)
+        self.rejected_steps += 1
+        self._maybe_restart()
+        return torch.tensor(
+            old_loss,
+            dtype=candidate_loss_tensor.dtype,
+            device=candidate_loss_tensor.device,
+        )
+
+    def _evaluate(self, closure: Callable[[], torch.Tensor]) -> torch.Tensor:
+        """Run the closure and count one objective evaluation."""
+        with torch.enable_grad():
+            loss_tensor = closure()
+        loss = float(loss_tensor.detach().item())
+        self._record_eval(loss)
+        return loss_tensor
+
+    @torch.no_grad()
+    def _propose_step(self) -> None:
+        """Add Gaussian noise to each trainable parameter."""
+        for group in self.param_groups:
+            step_size = group["step_size"]
+            for p in group["params"]:
+                if not p.requires_grad:
+                    continue
+                noise = torch.randn(
+                    p.shape,
+                    generator=self._generator,
+                    device=p.device,
+                    dtype=p.dtype,
+                )
+                p.add_(step_size * noise)
+
+    @torch.no_grad()
+    def _maybe_restart(self) -> None:
+        """Randomly reset parameters when the restart schedule is reached."""
+        if self.restart_interval is None:
+            return
+        if self.completed_restarts >= self.restarts:
+            return
+        if self.proposed_steps % self.restart_interval != 0:
+            return
+
+        for p in self._parameters():
+            noise = torch.randn(
+                p.shape,
+                generator=self._generator,
+                device=p.device,
+                dtype=p.dtype,
+            )
+            p.copy_(noise)
+
+        self.completed_restarts += 1
+        self._current_loss = None
+
+    @torch.no_grad()
+    def _save_best_if_needed(self, loss: float) -> None:
+        """Save the current parameters when they improve the best loss."""
+        if self.best_loss is None or loss <= self.best_loss:
+            self.best_loss = loss
+            self._best_params = self._clone_params()
+
+    @torch.no_grad()
+    def restore_best(self) -> None:
+        """Restore the best parameter values observed so far."""
+        if self._best_params is not None:
+            self._restore_params(self._best_params)