pytensorforge 0.1.0__py3-none-any.whl

src/optimizers/Adam.py

@@ -0,0 +1,98 @@
+import numpy as np
+
+from src.optimizers.Optimizer import SerialOptimizer
+
+
+class Adam(SerialOptimizer):
+
+    def __init__(
+        self,
+        lr=0.001,
+        beta1=0.9,
+        beta2=0.999,
+        eps=1e-8,
+    ):
+        self.lr = lr
+        self.beta1 = beta1
+        self.beta2 = beta2
+        self.eps = eps
+
+        self.m = {}
+        self.v = {}
+
+        self.t = 0
+
+    def step(self, params):
+
+        self.t += 1
+
+        for p in params:
+
+            if not p.requires_grad:
+                continue
+
+            pid = id(p)
+
+            if pid not in self.m:
+                self.m[pid] = np.zeros_like(p.data)
+                self.v[pid] = np.zeros_like(p.data)
+
+            self.m[pid] = (
+                self.beta1 * self.m[pid]
+                + (1 - self.beta1) * p.grad
+            )
+
+            self.v[pid] = (
+                self.beta2 * self.v[pid]
+                + (1 - self.beta2) * (p.grad ** 2)
+            )
+
+            m_hat = self.m[pid] / (1 - self.beta1 ** self.t)
+            v_hat = self.v[pid] / (1 - self.beta2 ** self.t)
+
+            p.data -= (
+                self.lr
+                * m_hat
+                / (np.sqrt(v_hat) + self.eps)
+            )
+
+    def zero_grad(self, params):
+        for p in params:
+            p.zero_grad()
+
+    # def state_dict(self):
+    #
+    #     return {
+    #         "m": self.m,
+    #         "v": self.v,
+    #         "t": self.t,
+    #         "lr": self.lr
+    #     }
+    #
+    # def load_state_dict(self, state):
+    #
+    #     self.m = state["m"]
+    #     self.v = state["v"]
+    #     self.t = state["t"]
+    #     self.lr = state["lr"]
+
+    def state_dict(self):
+        return {
+            "lr": self.lr,
+            "beta1": self.beta1,
+            "beta2": self.beta2,
+            "eps": self.eps,
+            "t": self.t,
+            "m": self.m,
+            "v": self.v,
+        }
+
+    def load_state_dict(self, state):
+
+        self.lr = float(state["lr"])
+        self.beta1 = float(state["beta1"])
+        self.beta2 = float(state["beta2"])
+        self.eps = float(state["eps"])
+        self.t = int(state["t"])
+        self.m = state["m"].item() if isinstance(state["m"], np.ndarray) else state["m"]
+        self.v = state["v"].item() if isinstance(state["v"], np.ndarray) else state["v"]

src/optimizers/AdamW.py

@@ -0,0 +1,84 @@
+import numpy as np
+
+from src.optimizers.Optimizer import SerialOptimizer
+
+
+class AdamW(SerialOptimizer):
+
+    def __init__(
+        self,
+        lr=0.001,
+        beta1=0.9,
+        beta2=0.999,
+        eps=1e-8,
+        weight_decay=0.01,
+    ):
+        self.lr = lr
+        self.beta1 = beta1
+        self.beta2 = beta2
+        self.eps = eps
+        self.weight_decay = weight_decay
+
+        self.m = {}
+        self.v = {}
+
+        self.t = 0
+
+    def step(self, params):
+
+        self.t += 1
+
+        for pid, p in enumerate(params):
+
+            if not p.requires_grad:
+                continue
+
+            if pid not in self.m:
+                self.m[pid] = np.zeros_like(p.data)
+                self.v[pid] = np.zeros_like(p.data)
+
+            p.data *= (1 - self.lr * self.weight_decay)
+
+            self.m[pid] = (
+                self.beta1 * self.m[pid]
+                + (1 - self.beta1) * p.grad
+            )
+
+            self.v[pid] = (
+                self.beta2 * self.v[pid]
+                + (1 - self.beta2) * (p.grad ** 2)
+            )
+
+            m_hat = self.m[pid] / (1 - self.beta1 ** self.t)
+            v_hat = self.v[pid] / (1 - self.beta2 ** self.t)
+
+            p.data -= (
+                self.lr
+                * m_hat
+                / (np.sqrt(v_hat) + self.eps)
+            )
+
+    def zero_grad(self, params):
+        for p in params:
+            p.zero_grad()
+
+    def state_dict(self):
+        return {
+            "t": self.t,
+            "lr": self.lr,
+            "beta1": self.beta1,
+            "beta2": self.beta2,
+            "eps": self.eps,
+            "weight_decay": self.weight_decay,
+            "m": {k: v.copy() for k, v in self.m.items()},
+            "v": {k: v.copy() for k, v in self.v.items()},
+        }
+
+    def load_state_dict(self, state):
+        self.t = state["t"]
+        self.beta1 = state["beta1"]
+        self.beta2 = state["beta2"]
+        self.eps = state["eps"]
+        self.weight_decay = state["weight_decay"]
+        self.m = {k: v.copy() for k, v in state["m"].items()}
+        self.v = {k: v.copy() for k, v in state["v"].items()}

src/optimizers/Batch.py

@@ -0,0 +1,11 @@
+from src.optimizers.Optimizer import Optimizer
+
+
+class Batch(Optimizer):
+    def __init__(self, layers, lr):
+        self.layers = layers
+        self.lr = lr
+
+    def step(self):
+        for layer in self.layers:
+            layer.step()

src/optimizers/Nesterov.py

@@ -0,0 +1,35 @@
+import numpy as np
+
+
+class Nesterov:
+
+    def __init__(self, lr=0.01, momentum=0.9):
+        self.lr = lr
+        self.momentum = momentum
+        self.velocity = {}
+
+    def step(self, params):
+
+        for p in params:
+
+            if not p.requires_grad:
+                continue
+
+            if id(p) not in self.velocity:
+                self.velocity[id(p)] = np.zeros_like(p.data)
+
+            v_prev = self.velocity[id(p)].copy()
+
+            self.velocity[id(p)] = (
+                self.momentum * self.velocity[id(p)]
+                - self.lr * p.grad
+            )
+
+            p.data += (
+                -self.momentum * v_prev
+                + (1 + self.momentum) * self.velocity[id(p)]
+            )
+
+    def zero_grad(self, params):
+        for p in params:
+            p.zero_grad()

src/optimizers/Optimizer.py

@@ -0,0 +1,18 @@
+class Optimizer:
+    def __init__(self, parameters, grads, learning_rate):
+        self.parameters = parameters
+        self.grads = grads
+        self.learning_rate = learning_rate
+
+    def zero_out(self):
+        for p in self.grads:
+            p.zeros(p.shape)
+
+    def step(self):
+        raise NotImplementedError
+
+class SerialOptimizer:
+    def state_dict(self):
+        raise NotImplementedError
+    def load_state_dict(self, state):
+        raise NotImplementedError

src/optimizers/RMSProp.py

@@ -0,0 +1,35 @@
+import numpy as np
+
+
+class RMSProp:
+
+    def __init__(self, lr=0.001, beta=0.9, eps=1e-8):
+        self.lr = lr
+        self.beta = beta
+        self.eps = eps
+        self.cache = {}
+
+    def step(self, params):
+
+        for p in params:
+
+            if not p.requires_grad:
+                continue
+
+            if id(p) not in self.cache:
+                self.cache[id(p)] = np.zeros_like(p.data)
+
+            self.cache[id(p)] = (
+                self.beta * self.cache[id(p)]
+                + (1 - self.beta) * (p.grad ** 2)
+            )
+
+            p.data -= (
+                self.lr
+                * p.grad
+                / (np.sqrt(self.cache[id(p)]) + self.eps)
+            )
+
+    def zero_grad(self, params):
+        for p in params:
+            p.zero_grad()

src/optimizers/SGD.py

@@ -0,0 +1,30 @@
+from src.optimizers.Optimizer import Optimizer
+
+
+class SGD(Optimizer):
+    def __init__(self, parameters, grads, learning_rate = 0.01):
+        super().__init__(parameters, grads, learning_rate)
+
+    def step(self):
+        for p, grad in zip(self.parameters, self.grads):
+            p.data -= self.learning_rate * grad.data
+
+class SGDescent:
+    def __init__(self, lr=0.01): self.lr = lr
+
+    def step(self, params):
+        for p in params:
+            if p.requires_grad:
+                p.data -= self.lr * p.grad
+
+    def zero_grad(self, params):
+        for p in params:
+            p.zero_grad()
+
+    def state_dict(self):
+        return {
+            "lr": self.lr
+        }
+
+    def load_state_dict(self, state):
+        self.lr = float(state["lr"])

src/optimizers/SGDMomentum.py

@@ -0,0 +1,28 @@
+import numpy as np
+
+class SGDMomentum:
+
+    def __init__(self, lr=0.01, momentum=0.9):
+        self.lr = lr
+        self.momentum = momentum
+        self.velocity = {}
+
+    def step(self, params):
+
+        for p in params:
+
+            if not p.requires_grad:
+                continue
+
+            if id(p) not in self.velocity:
+                self.velocity[id(p)] = np.zeros_like(p.data)
+
+            v = self.velocity[id(p)]
+
+            v[:] = self.momentum * v - self.lr * p.grad
+
+            p.data += v
+
+    def zero_grad(self, params):
+        for p in params:
+            p.zero_grad()

src/optimizers/__init__.py

@@ -0,0 +1,9 @@
+from .SGD import SGDescent
+from .Adam import Adam
+from .AdamW import AdamW
+
+optimizers = {
+    "sgd": SGDescent(),
+    "adam": Adam(),
+    "adamw": AdamW(),
+}

src/scaling/StandardScaler.py

@@ -0,0 +1,15 @@
+import numpy as np
+
+
+class StandardScaler:
+    def fit(self, X):
+        self.mean = np.mean(X, axis=0)
+        self.std = np.std(X, axis=0)
+        self.std[self.std == 0] = 1.0
+
+    def transform(self, X):
+        return (X - self.mean) / self.std
+
+    def fit_transform(self, X):
+        self.fit(X)
+        return self.transform(X)

src/serialization/checkpoint.py

@@ -0,0 +1,58 @@
+import pickle
+
+# How to use:
+# model = Sequential(...)
+#
+# model.compile(
+#     optimizer="adam",
+#     loss="cross_entropy",
+# )
+#
+# epoch, loss = Checkpoint.load(
+#     model,
+#     model.optimizer,
+#     "checkpoint.ptf",
+# )
+#
+# model.fit(
+#     X_train,
+#     y_train,
+#     epochs=100,
+#     initial_epoch=epoch,
+# )
+
+class Checkpoint:
+
+    @staticmethod
+    def save(
+        model,
+        optimizer,
+        epoch,
+        loss,
+        path,
+    ):
+
+        state = {
+            "epoch": epoch,
+            "loss": float(loss.data),
+            "model": model.state_dict(),
+            "optimizer": optimizer.state_dict(),
+        }
+
+        with open(path, "wb") as f:
+            pickle.dump(state, f)
+
+    @staticmethod
+    def load(
+        model,
+        optimizer,
+        path,
+    ):
+
+        with open(path, "rb") as f:
+            state = pickle.load(f)
+
+        model.load_state_dict(state["model"])
+        optimizer.load_state_dict(state["optimizer"])
+
+        return state["epoch"], state["loss"]

src/serialization/modelio.py

@@ -0,0 +1,132 @@
+# import numpy as np
+#
+#
+# def flatten(state):
+#     out = {}
+#
+#     for layer_name, params in state.items():
+#
+#         for param_name, value in params.items():
+#             out[f"{layer_name}.{param_name}"] = value
+#
+#     return out
+#
+# class ModelIO:
+#
+#     @staticmethod
+#     def save(model, path, metadata = None):
+#
+#         np.savez_compressed(
+#             path,
+#             **flatten(model.state_dict())
+#         )
+#
+#     @staticmethod
+#     def save_data(data, path):
+#
+#         np.savez_compressed(
+#             path,
+#             **flatten(data)
+#         )
+#
+#     @staticmethod
+#     def load(model, path):
+#
+#         data = np.load(path)
+#
+#         state = {}
+#
+#         for key in data.files:
+#
+#             layer, param = key.split(".")
+#
+#             if layer not in state:
+#                 state[layer] = {}
+#
+#             state[layer][param] = data[key]
+#
+#         model.load_state_dict(state)
+#
+#         return model
+
+import numpy as np
+
+
+def flatten(d, prefix=""):
+    out = {}
+
+    for k, v in d.items():
+
+        key = f"{prefix}.{k}" if prefix else k
+
+        if isinstance(v, dict):
+            out.update(flatten(v, key))
+        else:
+            out[key] = v
+
+    return out
+
+
+def unflatten(flat):
+    out = {}
+
+    for key, value in flat.items():
+
+        parts = key.split(".")
+
+        d = out
+
+        for p in parts[:-1]:
+            d = d.setdefault(p, {})
+
+        d[parts[-1]] = value
+
+    return out
+
+
+class ModelIO:
+
+    @staticmethod
+    def save(model, path, metadata=None):
+
+        state = {
+            "model": model.state_dict()
+        }
+
+        if metadata is not None:
+            state["metadata"] = metadata
+
+        np.savez_compressed(
+            path,
+            **flatten(state)
+        )
+
+    @staticmethod
+    def load(model, path):
+
+        data = np.load(path, allow_pickle=True)
+
+        flat = {
+            k: data[k]
+            for k in data.files
+        }
+
+        state = unflatten(flat)
+
+        model.load_state_dict(state["model"])
+
+        return state.get("metadata", None)
+
+    @staticmethod
+    def read(path):
+
+        data = np.load(path, allow_pickle=True)
+
+        flat = {
+            k: data[k]
+            for k in data.files
+        }
+
+        state = unflatten(flat)
+
+        return state