pytensorforge 0.1.0__py3-none-any.whl

src/models/seq/Sequential.py

@@ -0,0 +1,297 @@
+import numpy as np
+from src.loss import losses
+from src.optimizers import optimizers
+from src.serialization.checkpoint import Checkpoint
+from src.serialization.modelio import ModelIO, flatten, unflatten
+
+
+class Sequential:
+
+    def __init__(self):
+        self.layers = []
+
+        self.loss = None
+        self.optimizer = None
+        self.metrics = []
+
+    def add(self, layer):
+        self.layers.append(layer)
+
+    def __call__(self, x):
+        for layer in self.layers:
+            x = layer(x)
+            # print(layer.__class__.__name__, x.shape)
+
+        return x
+
+    def compile(
+        self,
+        loss,
+        optimizer,
+        metrics=None,
+    ):
+
+        if isinstance(loss, str):
+            try:
+                self.loss = losses[loss]
+            except KeyError:
+                raise ValueError(f"Unknown loss '{loss}'")
+        else:
+            self.loss = loss
+
+        if isinstance(optimizer, str):
+            try:
+                self.optimizer = optimizers[optimizer]
+            except KeyError:
+                raise ValueError(f"Unknown optimizer '{optimizer}'")
+        else:
+            self.optimizer = optimizer
+
+        self.metrics = metrics or []
+
+    def fit(
+        self,
+        X,
+        y,
+        epochs=5,
+        initial_epoch=0,
+        batch_size=32,
+        verbose=1,
+        checkpoint_path=None,
+        checkpoint_every=1,
+    ):
+        n = len(X)
+
+        for epoch in range(initial_epoch, epochs):
+
+            epoch_loss = 0.0
+
+            for start in range(0, n, batch_size):
+
+                end = start + batch_size
+
+                xb = X[start:end]
+                yb = y[start:end]
+
+                self.optimizer.zero_grad(self.parameters())
+
+                predictions = self(xb)
+
+                loss = self.loss(predictions, yb)
+                epoch_loss += loss.data.item()
+
+                loss.backward()
+
+                self.optimizer.step(self.parameters())
+
+            if verbose:
+                # if epoch % 20 == 0 and self.layers[1].__class__.__name__:
+                #     print(np.linalg.norm(self.layers[1].attn.Wq.grad))
+                #     print(np.linalg.norm(self.layers[1].attn.Wk.grad))
+                #     print(np.linalg.norm(self.layers[1].attn.Wv.grad))
+                #     print(np.linalg.norm(self.layers[1].attn.Wo.grad))
+
+                print(f"Epoch {epoch + 1}/{epochs} loss={epoch_loss:.4f}")
+
+            if checkpoint_path is not None:
+                if (epoch + 1) % checkpoint_every == 0:
+                    Checkpoint.save(
+                        model=self,
+                        optimizer=self.optimizer,
+                        epoch=epoch + 1,
+                        loss=epoch_loss,
+                        path=checkpoint_path,
+                    )
+
+    def parameters(self):
+        params = []
+
+        for layer in self.layers:
+            params.extend(layer.parameters())
+
+        return params
+
+    def summary(self):
+
+        print("=" * 80)
+        print(f'{"Model: Sequential":^80}')
+        print("=" * 80)
+
+        print(
+            f'{"Layer (type)":30}'
+            f'{"Output Shape":25}'
+            f'{"Param #":>15}'
+        )
+
+        print("-" * 80)
+
+        total = 0
+        trainable = 0
+        non_trainable = 0
+
+        shape = None
+
+        for layer in self.layers:
+
+            params = 0
+
+            for p in layer.parameters():
+                count = int(np.prod(p.shape))
+                params += count
+                total += count
+
+                if p.trainable:
+                    trainable += count
+                else:
+                    non_trainable += count
+
+            output_shape = (
+                str(layer.output_shape)
+                if hasattr(layer, "output_shape")
+                else "?"
+            )
+
+            print(
+                f"{layer.__class__.__name__:30}"
+                f"{output_shape:25}"
+                f"{params:>15,}"
+            )
+
+        print("-" * 80)
+
+        print(f"{'Total params:':30}{total:>20,}")
+        print(f"{'Trainable params:':30}{trainable:>20,}")
+        print(f"{'Non-trainable params:':30}{non_trainable:>20,}")
+
+        print("=" * 80)
+
+        if self.optimizer is not None:
+            print(f"Optimizer : {self.optimizer.__class__.__name__}")
+
+        if self.loss is not None:
+            print(f"Loss      : {self.loss.__class__.__name__}")
+
+        if self.metrics:
+            print(
+                "Metrics   : "
+                + ", ".join(
+                    metric.__class__.__name__
+                    if not isinstance(metric, str)
+                    else metric
+                    for metric in self.metrics
+                )
+            )
+
+        print("=" * 80)
+
+    def predict(self, X):
+        predictions = []
+        for layer in self.layers:
+            X = layer(X)
+            predictions.append(X)
+        return X
+
+    def state_dict(self):
+
+        state = {}
+
+        for i, layer in enumerate(self.layers):
+            state[f"layer_{i}"] = layer.state_dict()
+
+        return state
+
+    def load_state_dict(self, state):
+
+        for i, layer in enumerate(self.layers):
+            layer.load_state_dict(
+                state.get(f"layer_{i}", {})
+            )
+
+    def save(self, path, metadata = None):
+        ModelIO.save(self, path, metadata)
+
+    def load_model(self, path, input_shape=None):
+
+        if not self.built:
+
+            if input_shape is None:
+                raise RuntimeError(
+                    "Model must be built before loading."
+                )
+
+            self.build(input_shape)
+
+        return ModelIO.load(self, path)
+
+    def load(self, path):
+        return ModelIO.load(self, path)
+
+    def save_metadata(self, path, metadata):
+        ModelIO.save_data(metadata, path)
+
+    def load_metadata(self, path):
+        return ModelIO.load(path)
+
+    def save_checkpoint(
+            self,
+            path,
+            optimizer,
+            epoch,
+            loss,
+            metadata=None,
+    ):
+
+        checkpoint = {
+            "model": self.state_dict(),
+            "optimizer": optimizer.state_dict(),
+            "epoch": epoch,
+            "loss": loss,
+            "metadata": metadata or {},
+        }
+
+        np.savez_compressed(
+            path,
+            **flatten(checkpoint)
+        )
+
+    def load_checkpoint(
+            self,
+            path,
+            optimizer=None,
+    ):
+
+        data = np.load(path, allow_pickle=True)
+
+        flat = {
+            k: data[k]
+            for k in data.files
+        }
+
+        checkpoint = unflatten(flat)
+
+        self.load_state_dict(
+            checkpoint["model"]
+        )
+
+        if optimizer is not None:
+            optimizer.load_state_dict(
+                checkpoint["optimizer"]
+            )
+
+        return {
+            "epoch": int(checkpoint["epoch"]),
+            "loss": float(checkpoint["loss"]),
+            "metadata": checkpoint.get("metadata", {}),
+        }
+
+    def build(self, input_shape):
+
+        shape = input_shape
+
+        for layer in self.layers:
+            layer.build(shape)
+            layer.built = True
+
+            shape = layer.compute_output_shape(shape)
+
+        self.built = True

src/models/tokenizer/BPETokenizer.py

@@ -0,0 +1,228 @@
+from collections import Counter
+
+# tokenizer.fit(texts)
+#
+# tokenizer.encode(text)
+#
+# tokenizer.decode(ids)
+#
+# tokenizer.batch_encode(texts)
+#
+# tokenizer.batch_decode(batch)
+#
+# tokenizer.save(path)
+#
+# Tokenizer.load(path)
+#
+# len(tokenizer)
+#
+# tokenizer.vocab_size
+
+# tokenizer = Tokenizer(
+#     lower=True,
+#     oov_token="<unk>",
+# )
+#
+# tokenizer.fit(texts)
+#
+# ids = tokenizer.encode("What is Python?")
+# text = tokenizer.decode(ids)
+#
+# batch = tokenizer.batch_encode(texts)
+#
+# tokenizer.save("tokenizer.ptf")
+# tokenizer = Tokenizer.load("tokenizer.ptf")
+
+class BPETokenizer:
+
+    def __init__(self,
+                 vocab_size=30000,
+                 lowercase=True):
+
+        self.vocab_size = vocab_size
+        self.lowercase = lowercase
+
+        self.merges = []
+        self.word_to_index = {}
+        self.index_to_word = {}
+
+
+    def build_words(self, texts):
+
+        words = Counter()
+
+        for text in texts:
+
+            if self.lowercase:
+                text = text.lower()
+
+            for word in text.split():
+
+                chars = tuple(word) + ("</w>",)
+
+                words[chars] += 1
+
+        return words
+
+    def get_pair_counts(self, words):
+
+        pairs = Counter()
+
+        for word, freq in words.items():
+
+            for i in range(len(word) - 1):
+                pair = (word[i], word[i + 1])
+
+                pairs[pair] += freq
+
+        return pairs
+
+    def merge(self, words, pair):
+
+        merged = {}
+
+        bigram = pair
+
+        replacement = "".join(pair)
+
+        for word, freq in words.items():
+
+            new = []
+
+            i = 0
+
+            while i < len(word):
+
+                if (
+                        i < len(word) - 1
+                        and word[i] == bigram[0]
+                        and word[i + 1] == bigram[1]
+                ):
+
+                    new.append(replacement)
+
+                    i += 2
+
+                else:
+
+                    new.append(word[i])
+
+                    i += 1
+
+            merged[tuple(new)] = freq
+
+        return merged
+
+    def fit(self, texts):
+
+        words = self.build_words(texts)
+
+        while True:
+
+            pairs = self.get_pair_counts(words)
+
+            if not pairs:
+                break
+
+            best = max(
+                pairs,
+                key=pairs.get
+            )
+
+            self.merges.append(best)
+
+            words = self.merge(words, best)
+
+            vocab = set()
+
+            for word in words:
+                vocab.update(word)
+
+            if len(vocab) >= self.vocab_size:
+                break
+
+        vocab = sorted(vocab)
+
+        self.word_to_index = {
+            token: i
+            for i, token in enumerate(vocab)
+        }
+
+        self.index_to_word = {
+            i: token
+            for token, i in self.word_to_index.items()
+        }
+
+    def encode_word(self, word):
+
+        tokens = list(word)
+
+        tokens.append("</w>")
+
+        for pair in self.merges:
+
+            replacement = "".join(pair)
+
+            i = 0
+
+            merged = []
+
+            while i < len(tokens):
+
+                if (
+                        i < len(tokens) - 1
+                        and tokens[i] == pair[0]
+                        and tokens[i + 1] == pair[1]
+                ):
+
+                    merged.append(replacement)
+
+                    i += 2
+
+                else:
+
+                    merged.append(tokens[i])
+
+                    i += 1
+
+            tokens = merged
+
+        return [
+            self.word_to_index[t]
+            for t in tokens
+        ]
+
+    def encode(self, text):
+
+        if self.lowercase:
+            text = text.lower()
+
+        ids = []
+
+        for word in text.split():
+            ids.extend(self.encode_word(word))
+
+        return ids
+
+    def decode(self, ids):
+
+        tokens = [
+            self.index_to_word[i]
+            for i in ids
+        ]
+
+        text = ""
+
+        for token in tokens:
+
+            if token == "</w>":
+
+                text += " "
+
+            else:
+
+                text += token
+
+        return text.strip()
+
+

src/models/transformers/Dropout.py

@@ -0,0 +1,35 @@
+import numpy as np
+
+from src.core.Tensor import Tensor
+from src.neural.Layer import Layer
+
+
+class Dropout(Layer):
+
+    def __init__(self, p=0.5):
+        super().__init__()
+        self.p = p
+        self.training = True
+
+    def build(self, input_shape):
+        pass
+
+    def call(self, x):
+
+        if not self.training or self.p <= 0:
+            return x
+
+        keep = 1 - self.p
+
+        mask = np.random.binomial(
+            1,
+            keep,
+            size=x.shape,
+        ).astype(np.float32)
+
+        mask = Tensor(
+            mask / keep,
+            requires_grad=False,
+        )
+
+        return x * mask

src/models/transformers/LastToken.py

@@ -0,0 +1,10 @@
+from src.neural.Layer import Layer
+
+
+class LastToken(Layer):
+
+    def build(self, input_shape):
+        pass
+
+    def call(self, x):
+        return x[:, -1, :]

src/models/transformers/LayerNorm.py

@@ -0,0 +1,54 @@
+import numpy as np
+
+from src.neural.Layer import Layer
+
+
+class LayerNorm(Layer):
+    def __init__(self, d_model, eps=1e-6):
+        super().__init__()
+        self.beta = None
+        self.gamma = None
+        self.d_model = d_model
+        self.eps = eps
+
+    def call_(self, x, mask=None):
+        mean = x.mean(axis=-1, keepdims=True)
+
+        # var = ((x - mean) ** 2).mean(
+        #     axis=-1,
+        #     keepdims=True,
+        # )
+        diff = x - mean
+        var = (diff * diff).mean(axis=-1, keepdims=True)
+
+        x_hat = (x - mean) / (var + self.eps).sqrt()
+
+        return self.gamma * x_hat + self.beta
+
+    def call(self, x, mask=None):
+        # mean = x.mean(axis=-1, keepdims=True)
+        #
+        # var = ((x - mean) ** 2).mean(axis=-1, keepdims=True)
+        #
+        # y = self.gamma * (x - mean) / (var + self.eps).sqrt() + self.beta
+        # return y
+        mean = x.mean(axis=-1, keepdims=True)
+
+        var = ((x - mean) ** 2).mean(axis=-1, keepdims=True)
+
+        xhat = (x - mean) / (var + self.eps).sqrt()
+
+        return self.gamma * xhat + self.beta
+
+    def build(self, input_shape):
+        self.gamma = self.add_weight(
+            shape=(self.d_model,),
+            initializer="ones",
+            name="gamma",
+        )
+
+        self.beta = self.add_weight(
+            shape=(self.d_model,),
+            initializer="zeros",
+            name="beta",
+        )

src/models/transformers/Linear.py

@@ -0,0 +1,18 @@
+from src.neural.Dense import Dense
+
+
+class Linear(Dense):
+
+    def __init__(
+        self,
+        in_features,
+        out_features,
+        bias=True,
+    ):
+        super().__init__(
+            units=out_features,
+            activation=None,
+        )
+
+        self.in_features = in_features
+        self.use_bias = bias