PyIntell 0.1.0__py3-none-any.whl

pyintell/layers.py

@@ -0,0 +1,75 @@
+"""Neural-network layer and activation primitives."""
+
+import numpy as np
+
+
+def linear(x, weights, bias=None):
+    result = np.asarray(x) @ np.asarray(weights)
+    return result if bias is None else result + bias
+
+
+def relu(x): return np.maximum(x, 0)
+def leaky_relu(x, negative_slope=0.01):
+    x = np.asarray(x); return np.where(x >= 0, x, negative_slope * x)
+def gelu(x):
+    x = np.asarray(x); return 0.5 * x * (1.0 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * x**3)))
+def sigmoid(x): return 1.0 / (1.0 + np.exp(-np.asarray(x)))
+def tanh(x): return np.tanh(x)
+def softplus(x): return np.log1p(np.exp(-np.abs(x))) + np.maximum(x, 0)
+def silu(x): return np.asarray(x) * sigmoid(x)
+def swish(x): return silu(x)
+def mish(x): return np.asarray(x) * np.tanh(softplus(x))
+
+
+def softmax(x, axis=-1):
+    x = np.asarray(x); shifted = x - np.max(x, axis=axis, keepdims=True)
+    values = np.exp(shifted); return values / np.sum(values, axis=axis, keepdims=True)
+
+
+def log_softmax(x, axis=-1):
+    x = np.asarray(x); shifted = x - np.max(x, axis=axis, keepdims=True)
+    return shifted - np.log(np.sum(np.exp(shifted), axis=axis, keepdims=True))
+
+
+def activation(x, name="gelu"):
+    functions = {"relu": relu, "gelu": gelu, "tanh": tanh, "sigmoid": sigmoid,
+                 "leaky_relu": leaky_relu, "softplus": softplus, "silu": silu,
+                 "swish": swish, "mish": mish}
+    if name not in functions: raise ValueError(f"unknown activation: {name}")
+    return functions[name](x)
+
+
+def layer_norm(x, eps=1e-5):
+    x = np.asarray(x); mean = np.mean(x, axis=-1, keepdims=True); variance = np.var(x, axis=-1, keepdims=True)
+    return (x - mean) / np.sqrt(variance + eps)
+
+
+def batch_norm(x, eps=1e-5):
+    x = np.asarray(x); mean = np.mean(x, axis=0, keepdims=True); variance = np.var(x, axis=0, keepdims=True)
+    return (x - mean) / np.sqrt(variance + eps)
+
+
+def rms_norm(x, eps=1e-8):
+    x = np.asarray(x); return x / np.sqrt(np.mean(x * x, axis=-1, keepdims=True) + eps)
+
+
+def dropout(x, probability=0.0, training=True):
+    if not training or probability <= 0: return x
+    if probability >= 1: raise ValueError("dropout probability must be less than 1")
+    mask = (np.random.random(np.asarray(x).shape) >= probability) / (1 - probability)
+    return np.asarray(x) * mask
+
+
+def flatten_layer(x): return np.asarray(x).reshape(-1)
+
+
+def feedforward(x, hidden_size=None, activation_name="gelu"):
+    width = x.shape[-1] if hidden_size is None else hidden_size; input_size = x.shape[-1]
+    w1 = np.random.randn(input_size, width).astype(np.float32) * 0.02
+    w2 = np.random.randn(width, input_size).astype(np.float32) * 0.02
+    return linear(activation(linear(x, w1), activation_name), w2)
+
+
+def mlp(x, hidden_size=None, activation_name="gelu"): return feedforward(x, hidden_size, activation_name)
+def residual(x, function): return np.asarray(x) + function(x)
+def residual_block(x, function): return residual(x, function)

pyintell/loss.py

@@ -0,0 +1,39 @@
+"""Loss functions."""
+
+import numpy as np
+
+
+def cross_entropy(logits, targets):
+    logits = np.asarray(logits, dtype=np.float64); targets = np.asarray(targets, dtype=np.int64)
+    shifted = logits - np.max(logits, axis=-1, keepdims=True)
+    log_probs = shifted - np.log(np.sum(np.exp(shifted), axis=-1, keepdims=True))
+    flat = log_probs.reshape(-1, log_probs.shape[-1]); target_flat = targets.reshape(-1)
+    return float(-np.mean(flat[np.arange(target_flat.size), target_flat]))
+
+def binary_cross_entropy(predictions, targets, eps=1e-12):
+    p = np.clip(np.asarray(predictions), eps, 1-eps); t = np.asarray(targets)
+    return float(-np.mean(t*np.log(p) + (1-t)*np.log(1-p)))
+
+def mse(predictions, targets): return float(np.mean((np.asarray(predictions) - np.asarray(targets)) ** 2))
+def mae(predictions, targets): return float(np.mean(np.abs(np.asarray(predictions) - np.asarray(targets))))
+def huber_loss(predictions, targets, delta=1.0):
+    error = np.abs(np.asarray(predictions) - np.asarray(targets)); quadratic = np.minimum(error, delta)
+    linear = error - quadratic; return float(np.mean(0.5*quadratic**2 + delta*linear))
+
+def kl_divergence(p, q, eps=1e-12):
+    p = np.clip(np.asarray(p), eps, None); q = np.clip(np.asarray(q), eps, None)
+    return float(np.sum(p * (np.log(p) - np.log(q))))
+
+def contrastive_loss(a, b, target, margin=1.0):
+    distance = np.linalg.norm(np.asarray(a)-np.asarray(b), axis=-1)
+    target = np.asarray(target); return float(np.mean(target*distance**2 + (1-target)*np.maximum(0, margin-distance)**2))
+
+def label_smoothing(targets, classes, smoothing=0.1):
+    result = np.full((len(targets), classes), smoothing/(classes-1), dtype=np.float32)
+    result[np.arange(len(targets)), np.asarray(targets)] = 1-smoothing
+    return result
+
+def loss(predictions, targets, kind="cross_entropy"):
+    functions = {"cross_entropy": cross_entropy, "mse": mse, "mae": mae, "binary_cross_entropy": binary_cross_entropy, "huber": huber_loss}
+    if kind not in functions: raise ValueError(f"unknown loss: {kind}")
+    return functions[kind](predictions, targets)

pyintell/model.py

@@ -0,0 +1,224 @@
+"""Trainable language-model object returned by pyintell.build."""
+
+import time
+import numpy as np
+
+from .embeddings import embedding, positional_embedding
+from .transformer import transformer, init_block
+from .tokenization import encode, decode
+from .generation import sample
+from .optim import Optimizer
+from .focus import build_focus_config, normalize_focus
+
+
+class Model:
+    """A compact, persistent NumPy Transformer language model.
+
+    Alpha note: this class provides the framework and introspection APIs. Its
+    training implementation is intentionally simple and is not production AI
+    training yet.
+    """
+    def __init__(self, vocab, reverse_vocab, dataset, parameters, focus, dtype, settings):
+        self.vocab = vocab
+        self.reverse_vocab = reverse_vocab
+        self.dataset = dataset
+        self.requested_parameters = int(parameters)
+        self.parameters = self.requested_parameters
+        self.focus = tuple(normalize_focus(focus))
+        self.requested_dtype = str(dtype)
+        storage_dtype = {"bfloat16": "float16", "int4": "uint8"}.get(self.requested_dtype, self.requested_dtype)
+        self.dtype = np.dtype(storage_dtype)
+        self.settings = dict(settings)
+        self.focus_config = dict(self.settings.get("focus_config", build_focus_config(self.focus)))
+        self.focus_priorities = dict(self.focus_config.get("priorities", {}))
+        self.model_name = str(self.settings.get("model_name", "pyintell_model"))
+        self.platform = str(self.settings.get("platform", "auto"))
+        self.device = self.settings.get("device")
+        self.layers = int(self.settings["layers"])
+        self.heads = int(self.settings["heads"])
+        self.embedding_size = int(self.settings["embedding_size"])
+        self.context_length = int(self.settings["context_length"])
+        self.hidden_size = int(self.settings["hidden_size"])
+        seed = self.settings.get("seed", None)
+        self.rng = np.random.default_rng(seed)
+        self.embedding_weights = embedding(np.arange(len(vocab)), dimensions=self.embedding_size,
+                                           vocab_size=len(vocab)).astype(self.dtype)
+        self.position_weights = positional_embedding(self.context_length, self.embedding_size).astype(self.dtype)
+        self.blocks = [init_block(self.embedding_size, self.heads, self.hidden_size, self.rng)
+                       for _ in range(self.layers)]
+        self.output_weights = (self.rng.standard_normal((self.embedding_size, len(vocab))) *
+                               (1.0 / np.sqrt(max(self.embedding_size, 1)))).astype(np.float32)
+        self.output_bias = np.zeros(len(vocab), dtype=np.float32)
+        self.training_steps = 0
+        self.training_history = []
+        self.optimizer = None
+        self.created_at = time.time()
+
+    def _named_parameters(self):
+        params = {"embedding": self.embedding_weights, "position": self.position_weights,
+                  "output": self.output_weights, "output_bias": self.output_bias}
+        for i, block in enumerate(self.blocks):
+            for name, value in block.items():
+                params[f"blocks.{i}.{name}"] = value
+        return params
+
+    def named_parameters(self):
+        return self._named_parameters()
+
+    def parameter_count(self):
+        return int(sum(value.size for value in self._named_parameters().values()))
+
+    def parameters_info(self):
+        return {name: {"shape": tuple(value.shape), "dtype": str(value.dtype), "count": int(value.size)}
+                for name, value in self._named_parameters().items()}
+
+    def get_focus(self):
+        """Return the normalized capabilities selected for this model."""
+        return list(self.focus)
+
+    def set_focus(self, focus):
+        """Change focus metadata without pretending to retrain the model.
+
+        In 0.1.x this does not modify existing weights. Future training systems
+        can use the resulting configuration to implement specialization.
+        """
+        config = build_focus_config(focus)
+        self.focus = tuple(config["focuses"])
+        self.focus_config = config
+        self.focus_priorities = dict(config["priorities"])
+        self.settings["focus_config"] = config
+        return self.get_focus()
+
+    def focus_scores(self):
+        """Return normalized capability priorities produced by the builder."""
+        return dict(self.focus_priorities)
+
+    def focus_info(self):
+        """Return focus names and their current alpha priority metadata."""
+        ranked = sorted(self.focus_priorities.items(), key=lambda item: (-item[1], item[0]))
+        return {
+            "focuses": self.get_focus(),
+            "priorities": self.focus_scores(),
+            "top_capabilities": [name for name, _ in ranked[:5]],
+        }
+
+    def forward(self, token_ids):
+        ids = np.asarray(token_ids, dtype=np.int64)
+        if ids.ndim != 1:
+            raise ValueError("token_ids must be a one-dimensional sequence")
+        if len(ids) == 0:
+            raise ValueError("token_ids must not be empty")
+        if len(ids) > self.context_length:
+            raise ValueError("sequence exceeds model context length")
+        if ids.min() < 0 or ids.max() >= len(self.vocab):
+            raise ValueError("token ID outside vocabulary")
+        x = self.embedding_weights[ids].astype(np.float32) + self.position_weights[:len(ids)].astype(np.float32)
+        return transformer(x, layers=self.layers, heads=self.heads, hidden_size=self.hidden_size,
+                           causal=True, weights=self.blocks)
+
+    def logits(self, token_ids):
+        return self.forward(token_ids) @ self.output_weights + self.output_bias
+
+    @staticmethod
+    def _softmax(logits):
+        shifted = logits - np.max(logits, axis=-1, keepdims=True)
+        probabilities = np.exp(np.clip(shifted, -80.0, 80.0))
+        return probabilities / np.maximum(np.sum(probabilities, axis=-1, keepdims=True), 1e-12)
+
+    def _examples(self, dataset):
+        for sequence in dataset:
+            ids = list(sequence)
+            if len(ids) < 2:
+                continue
+            for end in range(1, len(ids)):
+                start = max(0, end - self.context_length)
+                yield ids[start:end], ids[end]
+
+    def evaluate(self, dataset=None, **kwargs):
+        data = self.dataset if dataset is None else dataset
+        total_loss = 0.0; correct = 0; count = 0
+        for inputs, target in self._examples(data):
+            probabilities = self._softmax(self.logits(inputs)[-1])
+            target = int(target)
+            if target < 0 or target >= len(self.vocab):
+                raise ValueError("target token ID outside vocabulary")
+            total_loss -= float(np.log(np.clip(probabilities[target], 1e-12, 1.0)))
+            correct += int(np.argmax(probabilities) == target)
+            count += 1
+        if count == 0:
+            raise ValueError("dataset must contain sequences with at least two token IDs")
+        loss = total_loss / count
+        return {"loss": loss, "accuracy": correct / count, "samples": count,
+                "perplexity": float(np.exp(min(loss, 20.0)))}
+
+    def train(self, dataset=None, epochs=1, learning_rate=None, optimizer=None, **kwargs):
+        """Train the model's currently supported trainable output head."""
+        data = self.dataset if dataset is None else dataset
+        epochs = int(epochs if epochs is not None else self.settings.get("epochs", 1))
+        learning_rate = float(learning_rate if learning_rate is not None else self.settings.get("learning_rate", 3e-4))
+        optimizer = optimizer or self.settings.get("optimizer", "adamw")
+        if epochs < 1:
+            raise ValueError("epochs must be at least 1")
+        examples = list(self._examples(data))
+        if not examples:
+            raise ValueError("dataset must contain sequences with at least two token IDs")
+        trainable = {"output": self.output_weights, "output_bias": self.output_bias}
+        if self.optimizer is None or self.optimizer.kind != str(optimizer).lower() or self.optimizer.lr != learning_rate:
+            self.optimizer = Optimizer(trainable, kind=optimizer, learning_rate=learning_rate,
+                                       weight_decay=float(kwargs.get("weight_decay", self.settings.get("weight_decay", 0.0))),
+                                       clip_norm=kwargs.get("clip_norm", self.settings.get("gradient_clip")))
+        before = self.evaluate(data)
+        history = []
+        for _ in range(epochs):
+            gradients = {"output": np.zeros_like(self.output_weights), "output_bias": np.zeros_like(self.output_bias)}
+            total_loss = 0.0
+            for inputs, target in examples:
+                hidden = self.forward(inputs)[-1].astype(np.float32)
+                probabilities = self._softmax(hidden @ self.output_weights + self.output_bias)
+                target = int(target)
+                total_loss -= float(np.log(np.clip(probabilities[target], 1e-12, 1.0)))
+                probabilities[target] -= 1.0
+                gradients["output"] += np.outer(hidden, probabilities)
+                gradients["output_bias"] += probabilities
+            gradients = {k: v / len(examples) for k, v in gradients.items()}
+            self.optimizer.step(gradients)
+            self.training_steps += 1
+            history.append(total_loss / len(examples))
+        after = self.evaluate(data)
+        record = {"loss": history[-1], "loss_history": history, "epochs": epochs, "samples": len(examples),
+                  "steps": self.training_steps, "before": before, "after": after,
+                  "loss_decreased": after["loss"] < before["loss"],
+                  "accuracy_improved": after["accuracy"] > before["accuracy"]}
+        self.training_history.append(record)
+        return record
+
+    def generate(self, prompt, max_tokens=50, temperature=1.0, top_k=None, **kwargs):
+        if isinstance(max_tokens, bool) or not isinstance(max_tokens, (int, np.integer)):
+            raise TypeError("max_tokens must be an integer")
+        max_tokens = int(max_tokens)
+        if max_tokens < 0:
+            raise ValueError("max_tokens must not be negative")
+        if isinstance(prompt, str):
+            unknown_token = "<unk>" if "<unk>" in self.vocab else None
+            ids = encode(prompt, self.vocab, unknown_token=unknown_token)
+            if any(token_id is None for token_id in ids):
+                raise ValueError("prompt contains unknown tokens and vocabulary has no '<unk>' token")
+        else:
+            ids = list(prompt)
+        if not ids:
+            raise ValueError("prompt must contain at least one known token")
+        for _ in range(max_tokens):
+            context = ids[-self.context_length:]
+            ids.append(sample(self.logits(context)[-1], temperature=temperature, top_k=top_k))
+        return decode(ids, self.reverse_vocab) if isinstance(prompt, str) else ids
+
+    def summary(self):
+        actual = self.parameter_count()
+        return {"model_name": self.model_name, "platform": self.platform, "device": self.device,
+                "parameters": actual, "requested_parameters": self.requested_parameters,
+                "focus": list(self.focus), "focus_scores": self.focus_scores(),
+                "dtype": self.requested_dtype, "layers": self.layers,
+                "heads": self.heads, "embedding_size": self.embedding_size, "hidden_size": self.hidden_size,
+                "context_length": self.context_length, "vocab_size": len(self.vocab),
+                "training_steps": self.training_steps, "training_history_length": len(self.training_history),
+                "settings": dict(self.settings)}

pyintell/optim.py

@@ -0,0 +1,94 @@
+"""Practical NumPy optimizers for pyintell parameters."""
+
+import numpy as np
+
+
+class Optimizer:
+    """Stateful optimizer supporting SGD, Adam, AdamW, RMSProp and Adagrad."""
+    def __init__(self, parameters, kind="adamw", learning_rate=3e-4, weight_decay=0.0,
+                 beta1=0.9, beta2=0.999, eps=1e-8, momentum=0.0, clip_norm=None):
+        self.parameters = parameters
+        self.kind = str(kind).lower()
+        self.lr = float(learning_rate)
+        self.weight_decay = float(weight_decay)
+        self.beta1, self.beta2, self.eps = float(beta1), float(beta2), float(eps)
+        self.momentum, self.clip_norm = float(momentum), clip_norm
+        self.step_count = 0
+        self.m = {}
+        self.v = {}
+
+    def step(self, gradients):
+        """Apply one update to a dict of named NumPy arrays."""
+        self.step_count += 1
+        total = 0.0
+        for name, grad in gradients.items():
+            if name not in self.parameters:
+                continue
+            g = np.asarray(grad, dtype=np.float32)
+            if self.clip_norm is not None:
+                total += float(np.sum(g * g))
+        scale = 1.0
+        if self.clip_norm is not None:
+            norm = np.sqrt(total)
+            if norm > float(self.clip_norm):
+                scale = float(self.clip_norm) / max(norm, 1e-12)
+        for name, grad in gradients.items():
+            if name not in self.parameters:
+                continue
+            p = self.parameters[name]
+            g = np.asarray(grad, dtype=np.float32) * scale
+            if self.kind == "sgd":
+                if self.momentum:
+                    self.m[name] = self.momentum * self.m.get(name, np.zeros_like(g)) + g
+                    g = self.m[name]
+                update = g
+            elif self.kind in ("adam", "adamw"):
+                m = self.m[name] = self.beta1 * self.m.get(name, np.zeros_like(g)) + (1 - self.beta1) * g
+                v = self.v[name] = self.beta2 * self.v.get(name, np.zeros_like(g)) + (1 - self.beta2) * (g * g)
+                mh = m / (1 - self.beta1 ** self.step_count)
+                vh = v / (1 - self.beta2 ** self.step_count)
+                update = mh / (np.sqrt(vh) + self.eps)
+                if self.kind == "adamw" and self.weight_decay:
+                    p *= (1.0 - self.lr * self.weight_decay)
+            elif self.kind == "rmsprop":
+                v = self.v[name] = 0.99 * self.v.get(name, np.zeros_like(g)) + 0.01 * (g * g)
+                update = g / (np.sqrt(v) + self.eps)
+            elif self.kind == "adagrad":
+                v = self.v[name] = self.v.get(name, np.zeros_like(g)) + g * g
+                update = g / (np.sqrt(v) + self.eps)
+            else:
+                raise ValueError(f"unknown optimizer: {self.kind}")
+            if self.kind != "adamw" and self.weight_decay:
+                update = update + self.weight_decay * p
+            p[...] = p - self.lr * update
+        return self
+
+    def zero_grad(self):
+        return {name: np.zeros_like(value, dtype=np.float32) for name, value in self.parameters.items()}
+
+    def state_dict(self):
+        return {"kind": self.kind, "learning_rate": self.lr, "weight_decay": self.weight_decay,
+                "step": self.step_count, "m": self.m, "v": self.v}
+
+
+def optimizer(parameters=None, kind="adamw", learning_rate=3e-4, weight_decay=0.0, **kwargs):
+    return Optimizer(parameters or {}, kind, learning_rate, weight_decay, **kwargs)
+
+
+def sgd(parameters=None, learning_rate=1e-2, **kwargs): return optimizer(parameters, "sgd", learning_rate, **kwargs)
+def adam(parameters=None, learning_rate=3e-4, **kwargs): return optimizer(parameters, "adam", learning_rate, **kwargs)
+def adamw(parameters=None, learning_rate=3e-4, **kwargs): return optimizer(parameters, "adamw", learning_rate, **kwargs)
+def rmsprop(parameters=None, learning_rate=1e-3, **kwargs): return optimizer(parameters, "rmsprop", learning_rate, **kwargs)
+def adagrad(parameters=None, learning_rate=1e-2, **kwargs): return optimizer(parameters, "adagrad", learning_rate, **kwargs)
+
+def update_weights(weights, gradients, learning_rate=3e-4, weight_decay=0.0):
+    return np.asarray(weights) - learning_rate * (np.asarray(gradients) + weight_decay * np.asarray(weights))
+
+def step(weights, gradients, learning_rate=3e-4, weight_decay=0.0):
+    return update_weights(weights, gradients, learning_rate, weight_decay)
+
+def zero_grad(gradients):
+    return {k: np.zeros_like(v) for k, v in gradients.items()} if isinstance(gradients, dict) else np.zeros_like(gradients)
+
+def learning_rate(optimizer_state):
+    return float(optimizer_state.lr if hasattr(optimizer_state, "lr") else optimizer_state.get("learning_rate", 0.0))

pyintell/quantization.py

@@ -0,0 +1,25 @@
+"""Simple NumPy quantization and compression helpers."""
+
+import numpy as np
+
+
+def quantize(x, bits=8):
+    x = np.asarray(x, dtype=np.float32)
+    if bits == 8:
+        scale = max(float(np.max(np.abs(x))) / 127.0, 1e-12); return np.round(x / scale).astype(np.int8), scale
+    if bits == 4:
+        scale = max(float(np.max(np.abs(x))) / 7.0, 1e-12); return np.clip(np.round(x / scale), -8, 7).astype(np.int8), scale
+    raise ValueError("bits must be 8 or 4")
+
+def dequantize(values, scale): return np.asarray(values, dtype=np.float32) * scale
+def int8(x): return quantize(x, 8)
+def int4(x): return quantize(x, 4)
+def float16(x): return np.asarray(x, dtype=np.float16)
+def bfloat16(x): return np.asarray(x, dtype=np.float32)
+def prune(x, threshold=0.0): return np.where(np.abs(x) < threshold, 0, x)
+def sparsify(x, threshold=0.0): return prune(x, threshold)
+def compress(x): return np.asarray(x).tobytes()
+def decompress(data, dtype=np.float32): return np.frombuffer(data, dtype=dtype)
+def low_rank(x, rank):
+    u, s, vh = np.linalg.svd(np.asarray(x), full_matrices=False); return u[:, :rank], s[:rank], vh[:rank, :]
+def factorize(x, rank): return low_rank(x, rank)

pyintell/scheduling.py

@@ -0,0 +1,16 @@
+"""Learning-rate and training scheduling helpers."""
+
+import math
+
+
+def learning_rate(initial, step=0): return float(initial)
+def lr_scheduler(initial, kind="constant", total_steps=1, step=0):
+    if kind == "constant": return float(initial)
+    if kind == "linear": return linear_decay(initial, total_steps, step)
+    if kind == "cosine": return cosine_decay(initial, total_steps, step)
+    raise ValueError(f"unknown scheduler: {kind}")
+
+def constant_lr(initial, step=0): return float(initial)
+def linear_decay(initial, total_steps, step): return max(0.0, initial * (1.0 - min(step, total_steps) / max(total_steps, 1)))
+def cosine_decay(initial, total_steps, step): return initial * 0.5 * (1 + math.cos(math.pi * min(step, total_steps) / max(total_steps, 1)))
+def warmup(initial, warmup_steps, step): return initial * min(1.0, (step + 1) / max(warmup_steps, 1))