vectorphile 0.1.0__py3-none-any.whl

vectorphile/nn/activations.py

@@ -0,0 +1,97 @@
+"""Activation function modules."""
+import numpy as np
+from vectorphile.nn.module import Module
+from vectorphile.core.tensor import Tensor
+
+
+class ReLU(Module):
+    def __init__(self, inplace=False):
+        super().__init__()
+    def forward(self, x: Tensor) -> Tensor:
+        return x.relu()
+    def __repr__(self): return 'ReLU()'
+
+
+class LeakyReLU(Module):
+    def __init__(self, negative_slope=0.01):
+        super().__init__()
+        self.negative_slope = negative_slope
+    def forward(self, x: Tensor) -> Tensor:
+        return Tensor(np.where(x._data >= 0, x._data, self.negative_slope * x._data), device=x.device)
+    def __repr__(self): return f'LeakyReLU(negative_slope={self.negative_slope})'
+
+
+class ELU(Module):
+    def __init__(self, alpha=1.0):
+        super().__init__()
+        self.alpha = alpha
+    def forward(self, x: Tensor) -> Tensor:
+        return Tensor(np.where(x._data >= 0, x._data, self.alpha * (np.exp(x._data) - 1)), device=x.device)
+    def __repr__(self): return f'ELU(alpha={self.alpha})'
+
+
+class GELU(Module):
+    def forward(self, x: Tensor) -> Tensor:
+        d = x._data
+        return Tensor(0.5 * d * (1 + np.tanh(np.sqrt(2/np.pi) * (d + 0.044715 * d**3))), device=x.device)
+    def __repr__(self): return 'GELU()'
+
+
+class Sigmoid(Module):
+    def forward(self, x: Tensor) -> Tensor:
+        return x.sigmoid()
+    def __repr__(self): return 'Sigmoid()'
+
+
+class Tanh(Module):
+    def forward(self, x: Tensor) -> Tensor:
+        return x.tanh()
+    def __repr__(self): return 'Tanh()'
+
+
+class Softmax(Module):
+    def __init__(self, dim=-1):
+        super().__init__()
+        self.dim = dim
+    def forward(self, x: Tensor) -> Tensor:
+        from vectorphile.core.tensor import softmax
+        return softmax(x, self.dim)
+    def __repr__(self): return f'Softmax(dim={self.dim})'
+
+
+class LogSoftmax(Module):
+    def __init__(self, dim=-1):
+        super().__init__()
+        self.dim = dim
+    def forward(self, x: Tensor) -> Tensor:
+        from vectorphile.core.tensor import log_softmax
+        return log_softmax(x, self.dim)
+    def __repr__(self): return f'LogSoftmax(dim={self.dim})'
+
+
+class SELU(Module):
+    _alpha = 1.6732632423543772
+    _scale = 1.0507009873554805
+    def forward(self, x: Tensor) -> Tensor:
+        d = x._data
+        return Tensor(self._scale * np.where(d >= 0, d, self._alpha * (np.exp(d) - 1)), device=x.device)
+    def __repr__(self): return 'SELU()'
+
+
+class SiLU(Module):
+    def forward(self, x: Tensor) -> Tensor:
+        return Tensor(x._data * (1 / (1 + np.exp(-x._data))), device=x.device)
+    def __repr__(self): return 'SiLU()'
+
+
+class Mish(Module):
+    def forward(self, x: Tensor) -> Tensor:
+        return Tensor(x._data * np.tanh(np.log1p(np.exp(x._data))), device=x.device)
+    def __repr__(self): return 'Mish()'
+
+
+class Hardswish(Module):
+    def forward(self, x: Tensor) -> Tensor:
+        d = x._data
+        return Tensor(d * np.clip(d + 3, 0, 6) / 6, device=x.device)
+    def __repr__(self): return 'Hardswish()'

vectorphile/nn/layers.py

@@ -0,0 +1,231 @@
+"""Neural network layers."""
+from __future__ import annotations
+import numpy as np
+import math
+from vectorphile.nn.module import Module
+from vectorphile.core.tensor import Tensor
+from vectorphile.autograd.engine import Function
+
+
+class _LinearBackward(Function):
+    """Custom backward for y = x @ W.T  (W is weight matrix, shape out×in)."""
+    def __init__(self, x, weight):
+        super().__init__(x, weight)
+        self.x = x
+        self.weight = weight
+
+    def backward(self, grad):
+        # grad shape: (N, out)
+        # dL/dx = grad @ W        (N, in)
+        # dL/dW = grad.T @ x      (out, in)
+        dx = Tensor(grad._data @ self.weight._data)
+        dw = Tensor(grad._data.T @ self.x._data)
+        return dx, dw
+
+
+class Linear(Module):
+    def __init__(self, in_features: int, out_features: int, bias: bool = True):
+        super().__init__()
+        self.in_features = in_features
+        self.out_features = out_features
+        k = math.sqrt(1.0 / in_features)
+        self.weight = Tensor(
+            np.random.uniform(-k, k, (out_features, in_features)).astype(np.float32),
+            requires_grad=True
+        )
+        self.bias_param = None
+        if bias:
+            self.bias_param = Tensor(
+                np.zeros(out_features, dtype=np.float32), requires_grad=True
+            )
+            self._parameters['bias_param'] = self.bias_param
+        self._parameters['weight'] = self.weight
+
+    def forward(self, x: Tensor) -> Tensor:
+        # Manually compute matmul to keep weight in autograd graph
+        # x: (N, in), weight: (out, in) → out: (N, out)
+        from vectorphile.autograd.engine import MatMulBackward
+        import numpy as np
+        result_data = x._data @ self.weight._data.T
+        req_grad = x.requires_grad or self.weight.requires_grad
+        # Create a transposed view for the backward pass
+        weight_t = Tensor(self.weight._data.T, device=self.weight.device, requires_grad=False)
+        weight_t._data = self.weight._data.T  # view, not copy
+        result = Tensor(result_data, device=x.device, requires_grad=req_grad)
+        if req_grad:
+            # Custom backward: dL/dx = grad @ W,  dL/dW = grad.T @ x
+            result.grad_fn = _LinearBackward(x, self.weight)
+            result._is_leaf = False
+        if self.bias_param is not None:
+            result = result + self.bias_param
+        return result
+
+    def __repr__(self):
+        return f'Linear(in_features={self.in_features}, out_features={self.out_features}, bias={self.bias_param is not None})'
+
+
+class Conv2d(Module):
+    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, bias=True):
+        super().__init__()
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.kernel_size = kernel_size if isinstance(kernel_size, tuple) else (kernel_size, kernel_size)
+        self.stride = stride if isinstance(stride, tuple) else (stride, stride)
+        self.padding = padding if isinstance(padding, tuple) else (padding, padding)
+        kh, kw = self.kernel_size
+        k = math.sqrt(1.0 / (in_channels * kh * kw))
+        self.weight = Tensor(
+            np.random.uniform(-k, k, (out_channels, in_channels, kh, kw)).astype(np.float32),
+            requires_grad=True
+        )
+        self._parameters['weight'] = self.weight
+        self.bias_param = None
+        if bias:
+            self.bias_param = Tensor(np.zeros(out_channels, dtype=np.float32), requires_grad=True)
+            self._parameters['bias_param'] = self.bias_param
+
+    def forward(self, x: Tensor) -> Tensor:
+        N, C, H, W = x._data.shape
+        kh, kw = self.kernel_size
+        sh, sw = self.stride
+        ph, pw = self.padding
+        x_pad = np.pad(x._data, ((0,0),(0,0),(ph,ph),(pw,pw)), mode='constant')
+        Ho = (H + 2*ph - kh) // sh + 1
+        Wo = (W + 2*pw - kw) // sw + 1
+        out = np.zeros((N, self.out_channels, Ho, Wo), dtype=np.float32)
+        w = self.weight._data
+        for i in range(Ho):
+            for j in range(Wo):
+                patch = x_pad[:, :, i*sh:i*sh+kh, j*sw:j*sw+kw]
+                out[:, :, i, j] = np.einsum('nckl,ockl->no', patch, w)
+        if self.bias_param is not None:
+            out += self.bias_param._data[None, :, None, None]
+        return Tensor(out, device=x.device)
+
+    def __repr__(self):
+        return (f'Conv2d({self.in_channels}, {self.out_channels}, '
+                f'kernel_size={self.kernel_size}, stride={self.stride}, padding={self.padding})')
+
+
+class BatchNorm1d(Module):
+    def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True):
+        super().__init__()
+        self.num_features = num_features
+        self.eps = eps
+        self.momentum = momentum
+        self.affine = affine
+        if affine:
+            self.weight = Tensor(np.ones(num_features, dtype=np.float32), requires_grad=True)
+            self.bias_param = Tensor(np.zeros(num_features, dtype=np.float32), requires_grad=True)
+            self._parameters['weight'] = self.weight
+            self._parameters['bias_param'] = self.bias_param
+        self.running_mean = np.zeros(num_features, dtype=np.float32)
+        self.running_var = np.ones(num_features, dtype=np.float32)
+
+    def forward(self, x: Tensor) -> Tensor:
+        data = x._data
+        if self.training:
+            mean = data.mean(axis=0)
+            var = data.var(axis=0)
+            self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * mean
+            self.running_var = (1 - self.momentum) * self.running_var + self.momentum * var
+        else:
+            mean, var = self.running_mean, self.running_var
+        x_norm = (data - mean) / np.sqrt(var + self.eps)
+        if self.affine:
+            x_norm = x_norm * self.weight._data + self.bias_param._data
+        return Tensor(x_norm.astype(np.float32), device=x.device)
+
+
+class BatchNorm2d(Module):
+    def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True):
+        super().__init__()
+        self.num_features = num_features
+        self.eps = eps
+        self.momentum = momentum
+        self.affine = affine
+        if affine:
+            self.weight = Tensor(np.ones(num_features, dtype=np.float32), requires_grad=True)
+            self.bias_param = Tensor(np.zeros(num_features, dtype=np.float32), requires_grad=True)
+            self._parameters['weight'] = self.weight
+            self._parameters['bias_param'] = self.bias_param
+        self.running_mean = np.zeros(num_features, dtype=np.float32)
+        self.running_var = np.ones(num_features, dtype=np.float32)
+
+    def forward(self, x: Tensor) -> Tensor:
+        data = x._data  # (N, C, H, W)
+        if self.training:
+            mean = data.mean(axis=(0, 2, 3))
+            var = data.var(axis=(0, 2, 3))
+            self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * mean
+            self.running_var = (1 - self.momentum) * self.running_var + self.momentum * var
+        else:
+            mean, var = self.running_mean, self.running_var
+        x_norm = (data - mean[None,:,None,None]) / np.sqrt(var[None,:,None,None] + self.eps)
+        if self.affine:
+            x_norm = x_norm * self.weight._data[None,:,None,None] + self.bias_param._data[None,:,None,None]
+        return Tensor(x_norm.astype(np.float32), device=x.device)
+
+
+class LayerNorm(Module):
+    def __init__(self, normalized_shape, eps=1e-5, elementwise_affine=True):
+        super().__init__()
+        if isinstance(normalized_shape, int):
+            normalized_shape = (normalized_shape,)
+        self.normalized_shape = tuple(normalized_shape)
+        self.eps = eps
+        self.elementwise_affine = elementwise_affine
+        if elementwise_affine:
+            self.weight = Tensor(np.ones(normalized_shape, dtype=np.float32), requires_grad=True)
+            self.bias_param = Tensor(np.zeros(normalized_shape, dtype=np.float32), requires_grad=True)
+            self._parameters['weight'] = self.weight
+            self._parameters['bias_param'] = self.bias_param
+
+    def forward(self, x: Tensor) -> Tensor:
+        ndim = len(self.normalized_shape)
+        axes = tuple(range(-ndim, 0))
+        mean = x._data.mean(axis=axes, keepdims=True)
+        var = x._data.var(axis=axes, keepdims=True)
+        x_norm = (x._data - mean) / np.sqrt(var + self.eps)
+        if self.elementwise_affine:
+            x_norm = x_norm * self.weight._data + self.bias_param._data
+        return Tensor(x_norm.astype(np.float32), device=x.device)
+
+
+class Dropout(Module):
+    def __init__(self, p=0.5):
+        super().__init__()
+        self.p = p
+
+    def forward(self, x: Tensor) -> Tensor:
+        if not self.training or self.p == 0:
+            return x
+        mask = (np.random.rand(*x._data.shape) > self.p).astype(np.float32)
+        return Tensor(x._data * mask / (1.0 - self.p), device=x.device, requires_grad=x.requires_grad)
+
+    def __repr__(self):
+        return f'Dropout(p={self.p})'
+
+
+class Embedding(Module):
+    def __init__(self, num_embeddings, embedding_dim, padding_idx=None):
+        super().__init__()
+        self.num_embeddings = num_embeddings
+        self.embedding_dim = embedding_dim
+        self.padding_idx = padding_idx
+        self.weight = Tensor(
+            np.random.randn(num_embeddings, embedding_dim).astype(np.float32) * 0.02,
+            requires_grad=True
+        )
+        self._parameters['weight'] = self.weight
+
+    def forward(self, indices: Tensor) -> Tensor:
+        idx = indices._data.astype(int)
+        out = self.weight._data[idx]
+        if self.padding_idx is not None:
+            mask = (idx == self.padding_idx)[..., None]
+            out = np.where(mask, 0.0, out)
+        return Tensor(out.astype(np.float32), device=indices.device)
+
+    def __repr__(self):
+        return f'Embedding({self.num_embeddings}, {self.embedding_dim})'

vectorphile/nn/losses.py

@@ -0,0 +1,172 @@
+"""Loss functions."""
+import numpy as np
+from vectorphile.nn.module import Module
+from vectorphile.core.tensor import Tensor
+
+
+class MSELoss(Module):
+    def __init__(self, reduction='mean'):
+        super().__init__()
+        self.reduction = reduction
+
+    def forward(self, pred: Tensor, target: Tensor) -> Tensor:
+        diff = pred - target
+        sq = diff * diff
+        if self.reduction == 'mean':
+            return sq.mean()
+        elif self.reduction == 'sum':
+            return sq.sum()
+        return sq
+
+
+class MAELoss(Module):
+    def __init__(self, reduction='mean'):
+        super().__init__()
+        self.reduction = reduction
+
+    def forward(self, pred: Tensor, target: Tensor) -> Tensor:
+        diff = np.abs(pred._data - target._data)
+        if self.reduction == 'mean':
+            return Tensor(diff.mean(), device=pred.device, requires_grad=True)
+        elif self.reduction == 'sum':
+            return Tensor(diff.sum(), device=pred.device, requires_grad=True)
+        return Tensor(diff, device=pred.device, requires_grad=True)
+
+
+class CrossEntropyLoss(Module):
+    def __init__(self, reduction='mean', ignore_index=-100, label_smoothing=0.0):
+        super().__init__()
+        self.reduction = reduction
+        self.ignore_index = ignore_index
+        self.label_smoothing = label_smoothing
+
+    def forward(self, logits: Tensor, targets: Tensor) -> Tensor:
+        log_probs = logits._data - np.log(np.exp(logits._data).sum(axis=-1, keepdims=True) + 1e-12)
+        t = targets._data.astype(int)
+        n = log_probs.shape[0]
+        nll = -log_probs[np.arange(n), t]
+        if self.label_smoothing > 0:
+            smooth_loss = -log_probs.mean(axis=-1)
+            nll = (1 - self.label_smoothing) * nll + self.label_smoothing * smooth_loss
+        if self.reduction == 'mean':
+            return Tensor(nll.mean(), device=logits.device, requires_grad=True)
+        elif self.reduction == 'sum':
+            return Tensor(nll.sum(), device=logits.device, requires_grad=True)
+        return Tensor(nll, device=logits.device, requires_grad=True)
+
+
+class _BCEBackward:
+    """Custom backward for BCE loss that keeps the computation in the autograd graph."""
+    def __init__(self, pred, target, reduction):
+        self.inputs = (pred,)
+        self.pred = pred
+        self.target = target
+        self.reduction = reduction
+
+    def backward(self, grad):
+        p = np.clip(self.pred._data, 1e-7, 1 - 1e-7)
+        t = self.target._data
+        n = p.size if self.reduction == 'mean' else 1
+        dp = (-(t / p) + (1 - t) / (1 - p)) / n
+        return (Tensor(grad._data * dp),)
+
+
+class BCELoss(Module):
+    def __init__(self, reduction='mean'):
+        super().__init__()
+        self.reduction = reduction
+
+    def forward(self, pred: Tensor, target: Tensor) -> Tensor:
+        p = np.clip(pred._data, 1e-7, 1 - 1e-7)
+        t = target._data
+        loss_data = -(t * np.log(p) + (1 - t) * np.log(1 - p))
+        if self.reduction == 'mean':
+            val = loss_data.mean()
+        elif self.reduction == 'sum':
+            val = loss_data.sum()
+        else:
+            val = loss_data
+        result = Tensor(val, device=pred.device, requires_grad=pred.requires_grad)
+        if pred.requires_grad:
+            result.grad_fn = _BCEBackward(pred, target, self.reduction)
+            result._is_leaf = False
+        return result
+
+
+class BCEWithLogitsLoss(Module):
+    def __init__(self, reduction='mean'):
+        super().__init__()
+        self.reduction = reduction
+
+    def forward(self, logits: Tensor, target: Tensor) -> Tensor:
+        d = logits._data
+        t = target._data
+        loss = np.maximum(d, 0) - d * t + np.log(1 + np.exp(-np.abs(d)))
+        if self.reduction == 'mean':
+            return Tensor(loss.mean(), device=logits.device, requires_grad=True)
+        elif self.reduction == 'sum':
+            return Tensor(loss.sum(), device=logits.device, requires_grad=True)
+        return Tensor(loss, device=logits.device, requires_grad=True)
+
+
+class NLLLoss(Module):
+    def __init__(self, reduction='mean'):
+        super().__init__()
+        self.reduction = reduction
+
+    def forward(self, log_probs: Tensor, targets: Tensor) -> Tensor:
+        t = targets._data.astype(int)
+        n = log_probs._data.shape[0]
+        nll = -log_probs._data[np.arange(n), t]
+        if self.reduction == 'mean':
+            return Tensor(nll.mean(), device=log_probs.device, requires_grad=True)
+        elif self.reduction == 'sum':
+            return Tensor(nll.sum(), device=log_probs.device, requires_grad=True)
+        return Tensor(nll, device=log_probs.device, requires_grad=True)
+
+
+class HuberLoss(Module):
+    def __init__(self, reduction='mean', delta=1.0):
+        super().__init__()
+        self.reduction = reduction
+        self.delta = delta
+
+    def forward(self, pred: Tensor, target: Tensor) -> Tensor:
+        diff = np.abs(pred._data - target._data)
+        loss = np.where(diff < self.delta, 0.5 * diff**2, self.delta * (diff - 0.5 * self.delta))
+        if self.reduction == 'mean':
+            return Tensor(loss.mean(), device=pred.device, requires_grad=True)
+        elif self.reduction == 'sum':
+            return Tensor(loss.sum(), device=pred.device, requires_grad=True)
+        return Tensor(loss, device=pred.device, requires_grad=True)
+
+
+class KLDivLoss(Module):
+    def __init__(self, reduction='mean', log_target=False):
+        super().__init__()
+        self.reduction = reduction
+        self.log_target = log_target
+
+    def forward(self, input: Tensor, target: Tensor) -> Tensor:
+        if self.log_target:
+            loss = np.exp(target._data) * (target._data - input._data)
+        else:
+            loss = target._data * (np.log(target._data + 1e-12) - input._data)
+        if self.reduction == 'mean':
+            return Tensor(loss.mean(), device=input.device, requires_grad=True)
+        elif self.reduction == 'sum':
+            return Tensor(loss.sum(), device=input.device, requires_grad=True)
+        return Tensor(loss, device=input.device, requires_grad=True)
+
+
+class CosineSimilarityLoss(Module):
+    def __init__(self, dim=1, eps=1e-8):
+        super().__init__()
+        self.dim = dim
+        self.eps = eps
+
+    def forward(self, x1: Tensor, x2: Tensor) -> Tensor:
+        n1 = np.linalg.norm(x1._data, axis=self.dim, keepdims=True)
+        n2 = np.linalg.norm(x2._data, axis=self.dim, keepdims=True)
+        cos_sim = (x1._data * x2._data).sum(axis=self.dim) / (n1.squeeze(self.dim) * n2.squeeze(self.dim) + self.eps)
+        return Tensor((1 - cos_sim).mean(), device=x1.device, requires_grad=True)

vectorphile/nn/module.py

@@ -0,0 +1,97 @@
+"""Base Module class — mirrors PyTorch's nn.Module API."""
+from __future__ import annotations
+import numpy as np
+from typing import Dict, Iterator, List, Optional, Tuple, Any
+from collections import OrderedDict
+
+
+class Module:
+    def __init__(self):
+        self._parameters: OrderedDict = OrderedDict()
+        self._modules: OrderedDict = OrderedDict()
+        self._buffers: OrderedDict = OrderedDict()
+        self.training: bool = True
+
+    def __setattr__(self, name: str, value):
+        from vectorphile.core.tensor import Tensor
+        if isinstance(value, Tensor) and value.requires_grad:
+            self._parameters[name] = value
+        elif isinstance(value, Module):
+            self._modules[name] = value
+        super().__setattr__(name, value)
+
+    def parameters(self) -> Iterator:
+        for p in self._parameters.values():
+            yield p
+        for module in self._modules.values():
+            yield from module.parameters()
+
+    def named_parameters(self, prefix='') -> Iterator[Tuple[str, Any]]:
+        for name, p in self._parameters.items():
+            yield (f'{prefix}.{name}' if prefix else name), p
+        for mname, module in self._modules.items():
+            full = f'{prefix}.{mname}' if prefix else mname
+            yield from module.named_parameters(prefix=full)
+
+    def modules(self) -> Iterator['Module']:
+        yield self
+        for m in self._modules.values():
+            yield from m.modules()
+
+    def named_modules(self, prefix='') -> Iterator[Tuple[str, 'Module']]:
+        yield prefix or '', self
+        for name, m in self._modules.items():
+            full = f'{prefix}.{name}' if prefix else name
+            yield from m.named_modules(prefix=full)
+
+    def state_dict(self) -> OrderedDict:
+        sd = OrderedDict()
+        for name, p in self.named_parameters():
+            sd[name] = p._data.copy()
+        return sd
+
+    def load_state_dict(self, state_dict: dict, strict=True):
+        from vectorphile.core.tensor import Tensor
+        own = dict(self.named_parameters())
+        for key, val in state_dict.items():
+            if key in own:
+                own[key]._data[:] = val
+            elif strict:
+                raise KeyError(f'Unexpected key "{key}" in state_dict')
+
+    def train(self, mode=True) -> 'Module':
+        self.training = mode
+        for m in self._modules.values():
+            m.train(mode)
+        return self
+
+    def eval(self) -> 'Module':
+        return self.train(False)
+
+    def zero_grad(self):
+        for p in self.parameters():
+            p.grad = None
+
+    def forward(self, *args, **kwargs):
+        raise NotImplementedError
+
+    def __call__(self, *args, **kwargs):
+        return self.forward(*args, **kwargs)
+
+    def __repr__(self):
+        lines = [self.__class__.__name__ + '(']
+        for name, m in self._modules.items():
+            mod_str = repr(m)
+            mod_str = '  ' + mod_str.replace('\n', '\n  ')
+            lines.append(f'  ({name}): {mod_str}')
+        lines.append(')')
+        return '\n'.join(lines)
+
+    def register_buffer(self, name: str, tensor):
+        self._buffers[name] = tensor
+        object.__setattr__(self, name, tensor)
+
+    def apply(self, fn) -> 'Module':
+        for m in self.modules():
+            fn(m)
+        return self

vectorphile/nn/sequential.py

@@ -0,0 +1,47 @@
+"""Sequential container."""
+from collections import OrderedDict
+from vectorphile.nn.module import Module
+from vectorphile.core.tensor import Tensor
+
+
+class Sequential(Module):
+    def __init__(self, *args):
+        super().__init__()
+        if len(args) == 1 and isinstance(args[0], OrderedDict):
+            for key, module in args[0].items():
+                self._modules[key] = module
+                setattr(self, key, module)
+        else:
+            for i, module in enumerate(args):
+                self._modules[str(i)] = module
+                object.__setattr__(self, str(i), module)
+
+    def forward(self, x: Tensor) -> Tensor:
+        for module in self._modules.values():
+            x = module(x)
+        return x
+
+    def append(self, module: Module) -> 'Sequential':
+        idx = str(len(self._modules))
+        self._modules[idx] = module
+        object.__setattr__(self, idx, module)
+        return self
+
+    def __getitem__(self, idx):
+        if isinstance(idx, slice):
+            keys = list(self._modules.keys())[idx]
+            return Sequential(OrderedDict((k, self._modules[k]) for k in keys))
+        return list(self._modules.values())[idx]
+
+    def __len__(self):
+        return len(self._modules)
+
+    def __iter__(self):
+        return iter(self._modules.values())
+
+    def __repr__(self):
+        lines = ['Sequential(']
+        for i, (name, m) in enumerate(self._modules.items()):
+            lines.append(f'  ({name}): {repr(m)}')
+        lines.append(')')
+        return '\n'.join(lines)

vectorphile/optim/__init__.py

@@ -0,0 +1,12 @@
+from vectorphile.optim.sgd import SGD
+from vectorphile.optim.adam import Adam, AdamW
+from vectorphile.optim.rmsprop import RMSprop
+from vectorphile.optim.adagrad import Adagrad
+from vectorphile.optim.lr_scheduler import (
+    StepLR, ExponentialLR, CosineAnnealingLR, LinearLR, ReduceLROnPlateau,
+)
+
+__all__ = [
+    'SGD', 'Adam', 'AdamW', 'RMSprop', 'Adagrad',
+    'StepLR', 'ExponentialLR', 'CosineAnnealingLR', 'LinearLR', 'ReduceLROnPlateau',
+]

vectorphile/optim/adagrad.py

@@ -0,0 +1,31 @@
+"""Adagrad optimizer."""
+import numpy as np
+from vectorphile.optim.optimizer import Optimizer
+
+
+class Adagrad(Optimizer):
+    def __init__(self, params, lr=0.01, lr_decay=0.0, weight_decay=0.0, eps=1e-10):
+        defaults = dict(lr=lr, lr_decay=lr_decay, weight_decay=weight_decay, eps=eps)
+        super().__init__(params, defaults)
+        self.state = {}
+
+    def step(self):
+        for group in self.param_groups:
+            lr = group['lr']
+            lr_decay = group['lr_decay']
+            wd = group['weight_decay']
+            eps = group['eps']
+            for p in group['params']:
+                if p.grad is None:
+                    continue
+                g = p.grad._data.copy()
+                if wd != 0:
+                    g += wd * p._data
+                pid = id(p)
+                if pid not in self.state:
+                    self.state[pid] = {'step': 0, 'sum': np.zeros_like(p._data)}
+                s = self.state[pid]
+                s['step'] += 1
+                clr = lr / (1 + (s['step'] - 1) * lr_decay)
+                s['sum'] += g**2
+                p._data -= clr * g / (np.sqrt(s['sum']) + eps)