PyIntell 0.1.0__py3-none-any.whl

pyintell/serialization.py

@@ -0,0 +1,313 @@
+"""Model serialization and named-model management."""
+
+import json
+import os
+import pickle
+from pathlib import Path
+
+_REGISTRY_PATH = Path(os.path.expanduser("~/.pyintell/models.json"))
+_CURRENT_MODEL = None
+
+
+def set_current_model(model):
+    """Set the model used by save_model when no explicit model is supplied."""
+    global _CURRENT_MODEL
+    _CURRENT_MODEL = model
+    return model
+
+
+def state_dict(model):
+    return {
+        name: value.copy()
+        for name, value in vars(model).items()
+        if hasattr(value, "shape") and hasattr(value, "copy")
+    }
+
+
+def load_state_dict(model, state):
+    for name, value in state.items():
+        setattr(model, name, value)
+    return model
+
+
+def save_weights(model, path):
+    import numpy as np
+    np.savez(path, **state_dict(model))
+
+
+def load_weights(model, path):
+    import numpy as np
+    with np.load(path) as data:
+        return load_state_dict(model, {key: data[key] for key in data.files})
+
+
+def _read_registry():
+    try:
+        with _REGISTRY_PATH.open("r", encoding="utf-8") as file:
+            data = json.load(file)
+        return data if isinstance(data, dict) else {}
+    except (FileNotFoundError, json.JSONDecodeError, OSError):
+        return {}
+
+
+def _write_registry(registry):
+    _REGISTRY_PATH.parent.mkdir(parents=True, exist_ok=True)
+    temporary = _REGISTRY_PATH.with_suffix(".tmp")
+    with temporary.open("w", encoding="utf-8") as file:
+        json.dump(registry, file, ensure_ascii=False, indent=2)
+    os.replace(temporary, _REGISTRY_PATH)
+
+
+def _validate_model_name(model_name):
+    if not isinstance(model_name, str) or not model_name.strip():
+        raise TypeError("model_name must be a non-empty string")
+    name = model_name.strip()
+    if name in {".", ".."} or any(char in name for char in "\\/"):
+        raise ValueError("model_name must not contain path separators")
+    return name
+
+
+def _model_file(directory, name):
+    """Return the canonical saved-model path inside a directory."""
+    directory = Path(directory).expanduser()
+    directory.mkdir(parents=True, exist_ok=True)
+    if not directory.is_dir():
+        raise NotADirectoryError(f"path must be a directory: {directory}")
+    return directory / f"{name}.pyintell"
+
+
+def save_model(model_name, path=None, model=None):
+    """Save a model under a unique name.
+
+    ``path`` is the directory where ``<model_name>.pyintell`` is created.
+    If ``model`` is omitted, the most recently built/loaded model is used.
+    """
+    global _CURRENT_MODEL
+    name = _validate_model_name(model_name)
+    if model is None:
+        model = _CURRENT_MODEL
+    if model is None:
+        raise RuntimeError("no active model; build or load a model first")
+    if path is None:
+        raise TypeError("path is required")
+
+    registry = _read_registry()
+    if name in registry:
+        raise FileExistsError(f"model name '{name}' already exists")
+
+    model_path = _model_file(path, name)
+    if model_path.exists():
+        raise FileExistsError(f"model file already exists: {model_path}")
+
+    # Store the name on the model so the active-model state is unambiguous.
+    model.model_name = name
+
+    temporary = model_path.with_suffix(model_path.suffix + ".tmp")
+    try:
+        with temporary.open("wb") as file:
+            pickle.dump(model, file, protocol=pickle.HIGHEST_PROTOCOL)
+        os.replace(temporary, model_path)
+
+        registry[name] = {"path": str(model_path.resolve())}
+        try:
+            _write_registry(registry)
+        except Exception:
+            try:
+                model_path.unlink()
+            except OSError:
+                pass
+            raise
+    finally:
+        try:
+            temporary.unlink()
+        except OSError:
+            pass
+
+    _CURRENT_MODEL = model
+    return str(model_path)
+
+
+def load_model(model_name):
+    """Load a named model and make it the active model."""
+    global _CURRENT_MODEL
+    name = _validate_model_name(model_name)
+    registry = _read_registry()
+    if name not in registry:
+        raise FileNotFoundError(f"model '{name}' was not found")
+
+    model_path = Path(registry[name].get("path", "")).expanduser()
+    if not model_path.is_file():
+        raise FileNotFoundError(f"model file for '{name}' no longer exists: {model_path}")
+
+    with model_path.open("rb") as file:
+        model = pickle.load(file)
+    model.model_name = name
+    _CURRENT_MODEL = model
+    return model
+
+
+def get_model(model_name):
+    """Load a named model without changing the active model."""
+    name = _validate_model_name(model_name)
+    registry = _read_registry()
+    if name not in registry:
+        raise FileNotFoundError(f"model '{name}' was not found")
+    model_path = Path(registry[name].get("path", "")).expanduser()
+    if not model_path.is_file():
+        raise FileNotFoundError(f"model file for '{name}' no longer exists: {model_path}")
+    with model_path.open("rb") as file:
+        model = pickle.load(file)
+    model.model_name = name
+    return model
+
+
+def delete_model(model_name):
+    """Delete a named model from disk and remove it from the registry."""
+    global _CURRENT_MODEL
+    name = _validate_model_name(model_name)
+    registry = _read_registry()
+    if name not in registry:
+        raise FileNotFoundError(f"model '{name}' was not found")
+
+    model_path = Path(registry[name].get("path", "")).expanduser()
+    if not model_path.is_file():
+        del registry[name]
+        _write_registry(registry)
+        raise FileNotFoundError(f"model file for '{name}' no longer exists: {model_path}")
+
+    model_path.unlink()
+    del registry[name]
+    _write_registry(registry)
+
+    if _CURRENT_MODEL is not None and getattr(_CURRENT_MODEL, "model_name", None) == name:
+        _CURRENT_MODEL = None
+    return True
+
+
+def edit_model(model_name, **changes):
+    """Edit supported metadata of a saved model."""
+    global _CURRENT_MODEL
+    name = _validate_model_name(model_name)
+    registry = _read_registry()
+    if name not in registry:
+        raise FileNotFoundError(f"model '{name}' was not found")
+    if not changes:
+        raise ValueError("at least one model field must be provided")
+
+    model_path = Path(registry[name].get("path", "")).expanduser()
+    if not model_path.is_file():
+        raise FileNotFoundError(f"model file for '{name}' no longer exists: {model_path}")
+
+    with model_path.open("rb") as file:
+        model = pickle.load(file)
+
+    allowed = {"focus", "parameters", "settings", "model_name"}
+    unknown = set(changes) - allowed
+    if unknown:
+        raise TypeError(f"unsupported model field(s): {', '.join(sorted(unknown))}")
+
+    if "focus" in changes:
+        focus = changes["focus"]
+        if isinstance(focus, str):
+            model.focus = (focus,)
+        elif isinstance(focus, (list, tuple, set)) and focus:
+            model.focus = tuple(focus)
+        else:
+            raise TypeError("focus must be a non-empty string, list, tuple, or set")
+
+    if "parameters" in changes:
+        parameters = changes["parameters"]
+        if isinstance(parameters, bool) or not isinstance(parameters, (int, float)):
+            raise TypeError("parameters must be a number")
+        if parameters < 0:
+            raise ValueError("parameters must not be negative")
+        model.parameters = int(parameters)
+
+    if "settings" in changes:
+        settings = changes["settings"]
+        if not isinstance(settings, dict):
+            raise TypeError("settings must be a dictionary")
+        protected = {"layers", "heads", "embedding_size", "hidden_size", "context_length"}
+        changed_architecture = protected.intersection(settings)
+        if changed_architecture:
+            raise ValueError("cannot edit architecture settings in place; rebuild the model instead")
+        model.settings.update(settings)
+
+    new_name = name
+    if "model_name" in changes:
+        new_name = _validate_model_name(changes["model_name"])
+        if new_name != name and new_name in registry:
+            raise FileExistsError(f"model name '{new_name}' already exists")
+        new_path = model_path.with_name(f"{new_name}.pymodel")
+        if new_path != model_path and new_path.exists():
+            raise FileExistsError(f"model file already exists: {new_path}")
+    else:
+        new_path = model_path
+
+    model.model_name = new_name
+    temporary = new_path.with_suffix(new_path.suffix + ".tmp")
+    with temporary.open("wb") as file:
+        pickle.dump(model, file, protocol=pickle.HIGHEST_PROTOCOL)
+    os.replace(temporary, new_path)
+
+    if new_name != name:
+        del registry[name]
+        registry[new_name] = {"path": str(new_path.resolve())}
+        if model_path != new_path and model_path.exists():
+            model_path.unlink()
+    else:
+        registry[name] = {"path": str(new_path.resolve())}
+
+    _write_registry(registry)
+    _CURRENT_MODEL = model
+    return model
+
+
+def save(model, path):
+    """Backward-compatible anonymous model save."""
+    with open(path, "wb") as file:
+        pickle.dump(model, file, protocol=pickle.HIGHEST_PROTOCOL)
+
+
+def load(path):
+    """Backward-compatible anonymous model load."""
+    with open(path, "rb") as file:
+        return pickle.load(file)
+
+
+def checkpoint(model, path):
+    return save(model, path)
+
+
+def save_checkpoint(model, path):
+    return save(model, path)
+
+
+def load_checkpoint(path):
+    return load(path)
+
+
+def save_vocab(vocabulary, path):
+    with open(path, "w", encoding="utf-8") as file:
+        json.dump(vocabulary, file, ensure_ascii=False, indent=2)
+
+
+def load_vocab(path):
+    with open(path, "r", encoding="utf-8") as file:
+        return json.load(file)
+
+
+def serialize(value):
+    return pickle.dumps(value)
+
+
+def deserialize(value):
+    return pickle.loads(value)
+
+
+def export_model(model, path):
+    return save(model, path)
+
+
+def import_model(path):
+    return load(path)

pyintell/system.py

@@ -0,0 +1,122 @@
+"""System resource inspection used by model builders."""
+
+import os
+import platform
+import shutil
+
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
+
+def ram():
+    """Return total, available, and used system RAM in bytes."""
+    if psutil is not None:
+        try:
+            info = psutil.virtual_memory()
+            return {"total": info.total, "available": info.available, "used": info.used}
+        except Exception:
+            pass
+
+    # Android/Pydroid may not expose virtual memory through psutil.
+    try:
+        values = {}
+        with open("/proc/meminfo", "r", encoding="utf-8") as file:
+            for line in file:
+                key, value = line.split(":", 1)
+                parts = value.strip().split()
+                if parts:
+                    values[key] = int(parts[0]) * 1024
+
+        total = values.get("MemTotal")
+        available = values.get("MemAvailable")
+        if available is None:
+            available = values.get("MemFree", 0) + values.get("Buffers", 0) + values.get("Cached", 0)
+        used = total - available if total is not None and available is not None else None
+        return {"total": total, "available": available, "used": used}
+    except (OSError, ValueError, IndexError):
+        return {"total": None, "available": None, "used": None}
+
+
+def cpu_info():
+    return {"name": platform.processor(), "count": os.cpu_count(), "architecture": platform.machine()}
+
+
+def storage_info(path="."):
+    usage = shutil.disk_usage(path)
+    return {"total": usage.total, "free": usage.free, "used": usage.used}
+
+
+def memory_info():
+    process = None
+    if psutil is not None:
+        try:
+            process = psutil.Process(os.getpid()).memory_info().rss
+        except Exception:
+            pass
+    return {"ram": ram(), "process": process}
+
+
+def device_info():
+    return {"cpu": cpu_info(), "gpu": gpu_info()}
+
+
+def device_count():
+    return os.cpu_count() or 1
+
+
+def gpu_info():
+    try:
+        import torch
+        if torch.cuda.is_available():
+            return {"available": True, "count": torch.cuda.device_count(), "name": torch.cuda.get_device_name(0)}
+    except Exception:
+        pass
+    return {"available": False, "count": 0, "name": None}
+
+
+def is_gpu_available():
+    return bool(gpu_info()["available"])
+
+
+def cuda():
+    return is_gpu_available()
+
+
+def cpu():
+    return True
+
+
+def clear_cache():
+    try:
+        import torch
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    except Exception:
+        pass
+
+
+def free_memory():
+    return ram().get("available")
+
+
+def gpu_memory():
+    return gpu_info()
+
+
+def cpu_memory():
+    return ram()
+
+
+def system_info():
+    return {
+        "platform": platform.platform(),
+        "system": platform.system(),
+        "machine": platform.machine(),
+        "processor": platform.processor(),
+        "cpu_count": os.cpu_count(),
+        "ram": ram(),
+        "storage": storage_info(),
+        "gpu": gpu_info(),
+    }

pyintell/tokenization.py

@@ -0,0 +1,147 @@
+"""Basic vocabulary and tokenization utilities."""
+
+
+def tokenizer(text):
+    if not isinstance(text, str):
+        raise TypeError("text must be a string")
+    return text.split()
+
+
+def tokenize(text):
+    return tokenizer(text)
+
+
+def detokenize(tokens):
+    return " ".join(tokens)
+
+
+def vocab(tokens):
+    if isinstance(tokens, str):
+        tokens = tokenizer(tokens)
+    if tokens and isinstance(tokens[0], (list, tuple)):
+        tokens = [t for seq in tokens for t in seq]
+    result = {}
+    for token in tokens:
+        if token not in result:
+            result[token] = len(result)
+    return result
+
+
+def build_vocab(tokens):
+    return vocab(tokens)
+
+
+def update_vocab(vocabulary, tokens):
+    result = dict(vocabulary)
+    new = vocab(tokens)
+    for token in new:
+        if token not in result:
+            result[token] = len(result)
+    return result
+
+
+def merge_vocab(*vocabularies):
+    result = {}
+    for vocabulary in vocabularies:
+        for token in vocabulary:
+            if token not in result:
+                result[token] = len(result)
+    return result
+
+
+def reverse_vocab(vocabulary):
+    return {index: token for token, index in vocabulary.items()}
+
+
+def vocab_size(vocabulary):
+    return len(vocabulary)
+
+
+def token_id(token, vocabulary, default=None):
+    return vocabulary.get(token, default)
+
+
+def id_token(index, reverse):
+    return reverse[index]
+
+
+def token_exists(token, vocabulary):
+    return token in vocabulary
+
+
+def add_token(vocabulary, token):
+    result = dict(vocabulary)
+    if token not in result:
+        result[token] = len(result)
+    return result
+
+
+def remove_token(vocabulary, token):
+    result = dict(vocabulary)
+    result.pop(token, None)
+    return result
+
+
+def special_tokens(pad="<PAD>", unk="<UNK>", bos="<BOS>", eos="<EOS>"):
+    return {"pad": pad, "unk": unk, "bos": bos, "eos": eos}
+
+
+def add_special_token(vocabulary, token):
+    return add_token(vocabulary, token)
+
+
+def encode(text, vocabulary, unknown_token=None):
+    tokens = tokenizer(text)
+    unknown_id = vocabulary.get(unknown_token) if unknown_token is not None else None
+    return [vocabulary[token] if token in vocabulary else unknown_id for token in tokens]
+
+
+def decode(ids, vocabulary):
+    reverse = vocabulary if vocabulary and all(isinstance(key, int) for key in vocabulary) else reverse_vocab(vocabulary)
+    return " ".join(reverse[index] for index in ids)
+
+
+def encode_batch(texts, vocabulary, unknown_token=None):
+    return [encode(text, vocabulary, unknown_token) for text in texts]
+
+
+def decode_batch(ids, reverse):
+    return [decode(item, reverse) for item in ids]
+
+
+def tokenize_batch(texts):
+    return [tokenizer(text) for text in texts]
+
+
+def detokenize_batch(batch):
+    return [detokenize(tokens) for tokens in batch]
+
+
+def normalize_text(text):
+    return " ".join(str(text).split())
+
+
+def clean_text(text):
+    return normalize_text(text)
+
+
+def split_text(text, separator=None):
+    return str(text).split(separator)
+
+
+def truncate(text, length):
+    return str(text)[:length]
+
+
+def pad_sequence(sequence, length, value=0, pad_id=None):
+    """Pad or truncate a sequence to ``length``.
+
+    ``value`` is the original padding argument. ``pad_id`` is supported as
+    an API-compatible alias used by the public test suite and is preferred
+    when supplied.
+    """
+    if pad_id is not None:
+        value = pad_id
+
+    result = list(sequence)[:length]
+    return result + [value] * max(0, length - len(result))

pyintell/training.py

@@ -0,0 +1,15 @@
+"""Training helpers."""
+
+
+def train(model, dataset, epochs=1, **kwargs):
+    """Run a model's training method when available."""
+    if hasattr(model, "train"):
+        return model.train(dataset, epochs=epochs, **kwargs)
+    raise TypeError("model must provide a train() method")
+
+
+def evaluate(model, dataset, **kwargs):
+    """Evaluate a model using its evaluate() method."""
+    if hasattr(model, "evaluate"):
+        return model.evaluate(dataset, **kwargs)
+    raise TypeError("model must provide an evaluate() method")

pyintell/transformer.py

@@ -0,0 +1,90 @@
+"""Persistent, parameterized Transformer building blocks."""
+
+import numpy as np
+
+from .attention import multihead_attention, cross_attention, causal_mask
+
+
+def _norm(x, eps=1e-5):
+    x = np.asarray(x, dtype=np.float32)
+    return (x - x.mean(axis=-1, keepdims=True)) / np.sqrt(x.var(axis=-1, keepdims=True) + eps)
+
+
+def _gelu(x):
+    return 0.5 * x * (1.0 + np.tanh(np.sqrt(2.0 / np.pi) * (x + 0.044715 * x ** 3)))
+
+
+def init_block(d_model, heads, hidden_size, rng=None):
+    """Create one persistent Transformer block state."""
+    rng = np.random.default_rng() if rng is None else rng
+    scale = 1.0 / np.sqrt(max(d_model, 1))
+    return {
+        "q": (rng.standard_normal((d_model, d_model)) * scale).astype(np.float32),
+        "k": (rng.standard_normal((d_model, d_model)) * scale).astype(np.float32),
+        "v": (rng.standard_normal((d_model, d_model)) * scale).astype(np.float32),
+        "o": (rng.standard_normal((d_model, d_model)) * scale).astype(np.float32),
+        "ff1": (rng.standard_normal((d_model, hidden_size)) * scale).astype(np.float32),
+        "ff2": (rng.standard_normal((hidden_size, d_model)) * (1.0 / np.sqrt(hidden_size))).astype(np.float32),
+        "ff1_bias": np.zeros(hidden_size, dtype=np.float32),
+        "ff2_bias": np.zeros(d_model, dtype=np.float32),
+        "q_bias": np.zeros(d_model, dtype=np.float32),
+        "k_bias": np.zeros(d_model, dtype=np.float32),
+        "v_bias": np.zeros(d_model, dtype=np.float32),
+        "o_bias": np.zeros(d_model, dtype=np.float32),
+        "norm1_scale": np.ones(d_model, dtype=np.float32),
+        "norm1_bias": np.zeros(d_model, dtype=np.float32),
+        "norm2_scale": np.ones(d_model, dtype=np.float32),
+        "norm2_bias": np.zeros(d_model, dtype=np.float32),
+    }
+
+
+def transformer_block(x, heads=8, hidden_size=None, causal=True, weights=None):
+    """Apply one pre-norm Transformer block."""
+    d_model = x.shape[-1]
+    hidden_size = int(hidden_size or d_model * 4)
+    if weights is None:
+        weights = init_block(d_model, heads, hidden_size)
+    n1 = _norm(x) * weights["norm1_scale"] + weights["norm1_bias"]
+    mask = causal_mask(x.shape[-2]) if causal else None
+    x = x + multihead_attention(n1, heads=heads, mask=mask, weights=weights)
+    n2 = _norm(x) * weights["norm2_scale"] + weights["norm2_bias"]
+    ff = _gelu(n2 @ weights["ff1"] + weights["ff1_bias"]) @ weights["ff2"] + weights["ff2_bias"]
+    return x + ff
+
+
+def transformer(x, layers=4, heads=8, hidden_size=None, causal=True, weights=None):
+    """Stack persistent Transformer blocks."""
+    hidden_size = int(hidden_size or x.shape[-1] * 4)
+    if weights is None:
+        weights = [init_block(x.shape[-1], heads, hidden_size) for _ in range(int(layers))]
+    if len(weights) != int(layers):
+        raise ValueError("number of Transformer weight sets must equal layers")
+    for block in weights:
+        x = transformer_block(x, heads=heads, hidden_size=hidden_size, causal=causal, weights=block)
+    return _norm(x)
+
+
+def encoder(x, layers=4, heads=8, hidden_size=None, weights=None):
+    for _ in []:
+        pass
+    hidden_size = int(hidden_size or x.shape[-1] * 4)
+    if weights is None:
+        weights = [init_block(x.shape[-1], heads, hidden_size) for _ in range(int(layers))]
+    for block in weights:
+        x = transformer_block(x, heads=heads, hidden_size=hidden_size, causal=False, weights=block)
+    return _norm(x)
+
+
+def decoder(x, context=None, memory=None, layers=4, heads=8, hidden_size=None, weights=None):
+    if context is not None and memory is not None:
+        raise ValueError("provide either context or memory, not both")
+    if memory is not None:
+        context = memory
+    hidden_size = int(hidden_size or x.shape[-1] * 4)
+    if weights is None:
+        weights = [init_block(x.shape[-1], heads, hidden_size) for _ in range(int(layers))]
+    for block in weights:
+        x = transformer_block(x, heads=heads, hidden_size=hidden_size, causal=True, weights=block)
+        if context is not None:
+            x = x + cross_attention(_norm(x), context)
+    return _norm(x)