tensorless-pytorch 0.1.0__py3-none-any.whl

tensorless/__init__.py

@@ -0,0 +1,41 @@
+"""
+Tensorless PyTorch
+==========
+
+ML with maximum automation and minimum setup.
+
+    import tensorless as tl
+
+    tl.train("./data")
+    tl.run("model.tl")
+
+See https://github.com/DeveloperPuneet/Tensorless-Pytorch for full documentation.
+"""
+
+from .api import train, pretrain, run, load, inspect
+from .config import TrainConfig
+from .errors import (
+    TensorlessError,
+    DataError,
+    ConfigError,
+    ModelError,
+    CheckpointError,
+    SerializationError,
+)
+from ._version import __version__
+
+__all__ = [
+    "train",
+    "pretrain",
+    "run",
+    "load",
+    "inspect",
+    "TrainConfig",
+    "TensorlessError",
+    "DataError",
+    "ConfigError",
+    "ModelError",
+    "CheckpointError",
+    "SerializationError",
+    "__version__",
+]

tensorless/_version.py

@@ -0,0 +1,6 @@
+__version__ = "0.1.0"
+
+# Bump this whenever the .tl serialization layout changes in a
+# backward-incompatible way. Stored inside every .tl file so old
+# files can be detected and (eventually) migrated.
+TL_FORMAT_VERSION = 1

tensorless/api.py

@@ -0,0 +1,246 @@
+"""Public Tensorless PyTorch API.
+
+    import tensorless as tl
+
+    tl.train("./data")            # full auto
+    tl.train("./data", d_model=512, layers=6, batch_size=32)  # override anything
+    tl.inspect("./data")
+    model = tl.load("model.tl")
+    tl.run("model.tl")
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import importlib.resources
+import os
+from typing import Any, Optional
+
+from .config import TrainConfig
+from .errors import ConfigError, DataError, ModelError
+from .data.loader import load_dataset
+from .data.fingerprint import fingerprint_path
+from .data.inspector import inspect_path, InspectionReport
+from .auto.config import resolve_config
+from .checkpoint.manager import CheckpointManager
+from .training.trainer import run_training
+from .serialization.tl_format import save_tl, load_tl
+from .runtime import LoadedModel, load_model
+from ._version import __version__ as _tl_version, TL_FORMAT_VERSION as _tl_format_version
+
+_KNOWN_FIELDS = {f.name for f in dataclasses.fields(TrainConfig)}
+
+
+def _build_train_config(**kwargs: Any) -> TrainConfig:
+    unknown = set(kwargs) - _KNOWN_FIELDS
+    if unknown:
+        raise ConfigError(
+            f"Unknown train() argument(s): {sorted(unknown)}. "
+            f"Valid arguments: {sorted(_KNOWN_FIELDS)}"
+        )
+    return TrainConfig(**kwargs)
+
+
+def inspect(path: str) -> InspectionReport:
+    """Inspect a dataset: detected task, size, problems, recommendations.
+
+    Does not train anything. Prints a human-readable report and also
+    returns a structured `InspectionReport` object.
+    """
+    report = inspect_path(path)
+    print(report)
+    return report
+
+
+def train(path: str, **kwargs: Any) -> LoadedModel:
+    """Train a model on the dataset at `path`, fully automatically by
+    default. Any field of `TrainConfig` can be overridden via keyword
+    argument, e.g. `tl.train("./data", d_model=512, layers=6)`.
+
+    Implements the "Smart Auto Check":
+      - if an up-to-date trained model already exists for this exact
+        dataset -> return it without retraining
+      - if training was interrupted on this exact dataset -> resume
+      - if the dataset changed -> retrain (or raise, if
+        `ask_on_data_change=True`), unless `force=True` is passed
+    """
+    user_cfg = _build_train_config(**kwargs)
+    out = user_cfg.out or "model.tl"
+    checkpoint_dir = user_cfg.checkpoint_dir or (out + ".ckpt")
+    checkpoint_mgr = CheckpointManager(checkpoint_dir)
+
+    if not os.path.exists(path):
+        raise DataError(f"Path '{path}' does not exist.")
+
+    fingerprint = fingerprint_path(path)
+
+    resume_state = None
+
+    if not user_cfg.force:
+        # 1. Is there already a complete, up-to-date .tl file?
+        try:
+            existing = load_tl(out)
+        except Exception:
+            existing = None
+        if existing is not None:
+            if existing.get("dataset_fingerprint") == fingerprint and existing.get("training_complete"):
+                if user_cfg.verbose:
+                    print(
+                        f"[tensorless] '{out}' already exists and matches this "
+                        f"dataset (fingerprint {fingerprint[:12]}...) -- using "
+                        f"the existing model. Pass force=True to retrain."
+                    )
+                return LoadedModel(existing)
+
+        # 2. Is there an interrupted / matching checkpoint to resume from?
+        if checkpoint_mgr.exists() and user_cfg.resume is not False:
+            ckpt = checkpoint_mgr.load()
+            if ckpt.get("dataset_fingerprint") == fingerprint:
+                if ckpt.get("training_complete"):
+                    # Training finished but the final .tl wasn't written
+                    # (e.g. process died right after the last checkpoint).
+                    # No need to retrain -- just package the .tl file.
+                    return _finalize_from_checkpoint(ckpt, out, user_cfg.verbose)
+                resume_state = ckpt
+                if user_cfg.verbose:
+                    print(
+                        f"[tensorless] found an interrupted checkpoint for this "
+                        f"exact dataset -- resuming training."
+                    )
+            else:
+                if user_cfg.ask_on_data_change:
+                    raise ConfigError(
+                        "The dataset has changed since the existing checkpoint "
+                        "was created. Pass force=True to retrain from scratch, "
+                        "or ask_on_data_change=False to retrain automatically."
+                    )
+                if user_cfg.verbose:
+                    print(
+                        "[tensorless] dataset has changed since the last "
+                        "checkpoint -- retraining from scratch."
+                    )
+                checkpoint_mgr.clear()
+        elif checkpoint_mgr.exists():
+            if user_cfg.verbose:
+                print("[tensorless] resume=False -- ignoring the existing checkpoint.")
+            checkpoint_mgr.clear()
+    else:
+        checkpoint_mgr.clear()
+
+    ds = load_dataset(path)
+    resolved = resolve_config(ds, user_cfg)
+    cfg = resolved.to_dict()
+
+    if resume_state is not None:
+        # Resumed runs must keep the exact architecture/config used
+        # originally, regardless of any new overrides, so the checkpoint
+        # can actually be loaded.
+        cfg = resume_state["config"]
+
+    result = run_training(
+        ds=ds,
+        cfg=cfg,
+        checkpoint_mgr=checkpoint_mgr,
+        dataset_fingerprint=fingerprint,
+        resume_state=resume_state,
+        log_fn=print if cfg.get("verbose", True) else (lambda *a, **k: None),
+    )
+
+    payload = {
+        "tl_format_version": _tl_format_version,
+        "tensorless_version": _tl_version,
+        "task": cfg["task"],
+        "model_type": cfg["model_type"],
+        "config": cfg,
+        "meta": result["meta"],
+        "model_state_dict": result["model_state_dict"],
+        "tokenizer_state": result["tokenizer"].state_dict() if result["tokenizer"] else None,
+        "preprocessor_state": result["preprocessor"].state_dict() if result["preprocessor"] else None,
+        "dataset_fingerprint": fingerprint,
+        "training_complete": True,
+        "metrics": result["metrics"],
+    }
+    save_tl(out, payload)
+    if cfg.get("verbose", True):
+        print(f"[tensorless] saved trained model to '{out}'")
+
+    return LoadedModel(payload)
+
+
+def pretrain(
+    out: str = "english_pretrained.tl", language: str = "english", **kwargs: Any
+) -> LoadedModel:
+    """Pretrain a small language model on the built-in starter corpus."""
+    if language.lower() != "english":
+        raise ValueError("The built-in pretraining corpus currently supports only 'english'.")
+    corpus = importlib.resources.files("tensorless.data").joinpath("english_grammar.txt")
+    options = {"task": "text-generation", "out": out, **kwargs}
+    return train(str(corpus), **options)
+
+
+def _finalize_from_checkpoint(ckpt: dict, out: str, verbose: bool) -> LoadedModel:
+    payload = {
+        "tl_format_version": _tl_format_version,
+        "tensorless_version": _tl_version,
+        "task": ckpt["config"]["task"],
+        "model_type": ckpt["config"]["model_type"],
+        "config": ckpt["config"],
+        "meta": ckpt["meta"],
+        "model_state_dict": ckpt["model_state_dict"],
+        "tokenizer_state": ckpt.get("tokenizer_state"),
+        "preprocessor_state": ckpt.get("preprocessor_state"),
+        "dataset_fingerprint": ckpt["dataset_fingerprint"],
+        "training_complete": True,
+        "metrics": {},
+    }
+    save_tl(out, payload)
+    if verbose:
+        print(f"[tensorless] finalized already-complete checkpoint into '{out}'")
+    return LoadedModel(payload)
+
+
+def load(path: str, device: Optional[str] = None) -> LoadedModel:
+    """Load a trained `.tl` model for inference."""
+    return load_model(path, device=device)
+
+
+def run(path: str, prompt: Optional[str] = None) -> Any:
+    """Run a trained `.tl` model.
+
+    For text-generation models with no prompt given, starts an
+    interactive terminal chat. Otherwise runs one generation/prediction
+    and returns/prints the result.
+    """
+    model = load_model(path)
+    if model.task == "text-generation":
+        if prompt is None:
+            model.chat()
+            return None
+        result = model.generate(prompt)
+        print(result)
+        return result
+    elif model.task == "text-classification":
+        if prompt is None:
+            print(
+                f"Loaded a '{model.task}' model. Use tl.load('{path}').predict(text) "
+                f"to classify text, or pass prompt=... / --prompt on the CLI."
+            )
+            return None
+        result = model.predict(prompt)
+        print(result)
+        return result
+    else:
+        # tabular classification/regression: needs a structured record, not
+        # free text, so the CLI --prompt shortcut doesn't apply here.
+        if prompt is not None:
+            raise ModelError(
+                f"'{path}' is a tabular '{model.task}' model, which expects a "
+                f"structured record (e.g. {{'age': 30, 'income': 90000}}), not "
+                f"free text. Use tl.load('{path}').predict({{...}}) from Python "
+                f"instead of --prompt on the CLI."
+            )
+        print(
+            f"Loaded a tabular '{model.task}' model. Use "
+            f"tl.load('{path}').predict({{...}}) to run predictions."
+        )
+        return None

tensorless/auto/__init__.py

@@ -0,0 +1,4 @@
+from .detector import detect_task
+from .config import resolve_config
+
+__all__ = ["detect_task", "resolve_config"]

tensorless/auto/config.py

@@ -0,0 +1,135 @@
+"""Automatic configuration.
+
+Turns a `Dataset` + a user-supplied `TrainConfig` (with mostly-None fields)
+into a fully-resolved `ResolvedConfig`. Every automatic choice is driven by
+simple, explainable heuristics based on dataset size -- this is not meant
+to be state-of-the-art NAS, it's meant to produce a *sane, working* default
+so `tl.train("./data")` just works.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict
+
+from ..config import TrainConfig, ResolvedConfig
+from ..data.loader import Dataset
+from ..devices.device import auto_select_device
+from .detector import detect_task, target_column
+
+
+def _auto_model_size(n_examples: int, kind: str):
+    """Return (d_model, layers, heads, ff_mult) scaled to dataset size.
+
+    Small datasets get small models (avoids absurd overfitting / long CPU
+    training times); larger datasets get bigger models. These are
+    deliberately modest sizes suitable for CPU-friendly experimentation --
+    users can always override with layers=, d_model=, etc.
+    """
+    if kind in ("text", "text_labeled"):
+        if n_examples < 200:
+            return 64, 2, 2, 2
+        elif n_examples < 5000:
+            return 128, 4, 4, 2
+        elif n_examples < 50000:
+            return 256, 6, 8, 4
+        else:
+            return 384, 8, 8, 4
+    else:  # tabular
+        if n_examples < 500:
+            return 32, 2, 1, 2
+        elif n_examples < 20000:
+            return 64, 3, 1, 2
+        else:
+            return 128, 4, 1, 2
+
+
+def _auto_batch_size(n_examples: int, max_seq_len: int) -> int:
+    token_budget = 8192
+    sequence_batch = max(1, token_budget // max_seq_len)
+    if n_examples < 200:
+        return min(8, sequence_batch)
+    elif n_examples < 2000:
+        return min(16, sequence_batch)
+    elif n_examples < 20000:
+        return min(32, sequence_batch)
+    else:
+        return min(64, sequence_batch)
+
+
+def _auto_epochs(n_examples: int) -> int:
+    if n_examples < 200:
+        return 40
+    elif n_examples < 2000:
+        return 20
+    elif n_examples < 20000:
+        return 10
+    else:
+        return 5
+
+
+def _effective_text_size(ds: Dataset) -> int:
+    """Estimate useful training examples for raw corpora."""
+    if ds.kind in ("text", "text_labeled"):
+        return max(len(ds), sum(len(text) for text in ds.texts) // 200)
+    return len(ds)
+
+
+def _auto_vocab_size(ds: Dataset) -> int:
+    if ds.kind not in ("text", "text_labeled"):
+        return 1000
+    unique_chars = len(set("".join(ds.texts)))
+    return min(4096, max(64, unique_chars * 8))
+
+
+def resolve_config(ds: Dataset, user: TrainConfig) -> ResolvedConfig:
+    n = _effective_text_size(ds)
+    task = user.task or detect_task(ds)
+    model_type = user.model_type or (
+        "transformer" if task in ("text-generation", "text-classification") else "mlp"
+    )
+    tokenizer = user.tokenizer or "bpe"
+    if tokenizer not in ("char", "bpe"):
+        raise ValueError("tokenizer must be 'char' or 'bpe'")
+
+    d_model, layers, heads, ff_mult = _auto_model_size(n, ds.kind)
+
+    device, precision = auto_select_device(user.device, user.precision)
+
+    out = user.out or "model.tl"
+    checkpoint_dir = user.checkpoint_dir or (out + ".ckpt")
+
+    max_seq_len = user.max_seq_len or (256 if ds.kind in ("text", "text_labeled") else 1)
+    resolved = ResolvedConfig(
+        out=out,
+        force=bool(user.force),
+        resume=user.resume,
+        ask_on_data_change=bool(user.ask_on_data_change),
+        task=task,
+        model_type=model_type,
+        d_model=user.d_model or d_model,
+        layers=user.layers or layers,
+        heads=user.heads or heads,
+        ff_mult=user.ff_mult or ff_mult,
+        dropout=user.dropout if user.dropout is not None else 0.1,
+        max_seq_len=max_seq_len,
+        tokenizer=tokenizer,
+        bpe_vocab_size=user.bpe_vocab_size if user.bpe_vocab_size is not None else _auto_vocab_size(ds),
+        optimizer=user.optimizer or "adamw",
+        learning_rate=user.learning_rate or (3e-4 if model_type == "transformer" else 1e-3),
+        weight_decay=user.weight_decay if user.weight_decay is not None else 0.01,
+        batch_size=user.batch_size if user.batch_size is not None else _auto_batch_size(n, max_seq_len),
+        epochs=user.epochs or _auto_epochs(n),
+        max_steps=user.max_steps,
+        grad_clip=user.grad_clip if user.grad_clip is not None else 1.0,
+        warmup_steps=user.warmup_steps if user.warmup_steps is not None else min(100, max(1, n // 10)),
+        val_split=user.val_split if user.val_split is not None else (0.1 if n >= 50 else 0.0),
+        patience=user.patience if user.patience is not None else 3,
+        min_delta=user.min_delta if user.min_delta is not None else 1e-4,
+        device=device,
+        precision=precision,
+        checkpoint_every=user.checkpoint_every or 50,
+        checkpoint_dir=checkpoint_dir,
+        seed=user.seed,
+        verbose=user.verbose,
+    )
+    return resolved

tensorless/auto/detector.py

@@ -0,0 +1,95 @@
+"""Automatic task-type detection.
+
+Given a loaded `Dataset`, decide what kind of ML task it represents:
+
+  - "text-generation"  : raw text corpus -> train a language model
+  - "text-classification": (text, label) pairs -> classify text
+  - "classification"    : tabular data with a categorical target
+  - "regression"        : tabular data with a numeric target
+
+The detector is intentionally simple and explainable: every decision can
+be described in one sentence, which is important because `tl.inspect()`
+reports *why* a task was chosen.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from ..data.loader import Dataset
+from ..errors import DataError
+
+_TARGET_CANDIDATES = ("label", "target", "class", "category", "y", "output")
+
+
+def _looks_numeric(values) -> bool:
+    n_ok = 0
+    n_total = 0
+    for v in values:
+        if v is None or v == "":
+            continue
+        n_total += 1
+        try:
+            float(v)
+            n_ok += 1
+        except (TypeError, ValueError):
+            pass
+    if n_total == 0:
+        return False
+    return (n_ok / n_total) > 0.95
+
+
+def _find_target_column(ds: Dataset) -> Optional[str]:
+    for cand in _TARGET_CANDIDATES:
+        for col in ds.columns:
+            if col.lower() == cand:
+                return col
+    # Fall back to the last column, a common convention in tabular datasets.
+    return ds.columns[-1] if ds.columns else None
+
+
+def detect_task(ds: Dataset) -> str:
+    """Return one of "text-generation", "text-classification",
+    "classification", "regression".
+    """
+    if ds.kind == "text":
+        return "text-generation"
+
+    if ds.kind == "text_labeled":
+        return "text-classification"
+
+    if ds.kind == "tabular":
+        if not ds.records:
+            raise DataError("Tabular dataset has no rows.")
+        target = _find_target_column(ds)
+        if target is None:
+            raise DataError(
+                "Could not find a target/label column in the tabular data. "
+                "Tensorless PyTorch looks for a column named one of: "
+                f"{', '.join(_TARGET_CANDIDATES)} (or uses the last column)."
+            )
+        values = [r.get(target) for r in ds.records]
+        if _looks_numeric(values):
+            unique_vals = set(values)
+            # Small number of distinct numeric values that are all
+            # integer-like -> more likely classification (e.g. 0/1 labels)
+            # than regression.
+            if len(unique_vals) <= min(10, max(2, len(values) // 20)):
+                try:
+                    all_int_like = all(float(v) == int(float(v)) for v in unique_vals if v not in (None, ""))
+                except (TypeError, ValueError):
+                    all_int_like = False
+                if all_int_like:
+                    return "classification"
+            return "regression"
+        return "classification"
+
+    raise DataError(f"Unknown dataset kind: {ds.kind}")
+
+
+def target_column(ds: Dataset) -> Optional[str]:
+    """Public helper mirroring `_find_target_column`, used by the tabular
+    model pipeline so detection and training agree on which column is
+    the target.
+    """
+    return _find_target_column(ds)

tensorless/checkpoint/__init__.py

@@ -0,0 +1,3 @@
+from .manager import CheckpointManager
+
+__all__ = ["CheckpointManager"]

tensorless/checkpoint/manager.py

@@ -0,0 +1,66 @@
+"""Checkpoint management.
+
+Handles all the state needed to resume training safely and transparently:
+model weights, optimizer state, scheduler state, epoch/step counters, the
+resolved training config, tokenizer/preprocessor state, the dataset
+fingerprint used for training, and the best-metric-so-far for early
+stopping.
+
+Users never touch this directly -- `tl.train()` decides automatically
+whether to create, update, or resume from a checkpoint (see
+`training/trainer.py` and the "Smart Auto Check" logic in `api.py`).
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import tempfile
+from typing import Any, Dict, Optional
+
+import torch
+
+from ..errors import CheckpointError
+
+CHECKPOINT_FILENAME = "checkpoint.pt"
+
+
+class CheckpointManager:
+    def __init__(self, checkpoint_dir: str):
+        self.checkpoint_dir = checkpoint_dir
+        self.path = os.path.join(checkpoint_dir, CHECKPOINT_FILENAME)
+
+    def exists(self) -> bool:
+        return os.path.isfile(self.path)
+
+    def save(self, state: Dict[str, Any]) -> None:
+        """Atomically write `state` to the checkpoint file.
+
+        Writes to a temp file first and renames it into place, so a crash
+        or interruption mid-write never leaves a corrupt checkpoint that
+        would block resumption.
+        """
+        os.makedirs(self.checkpoint_dir, exist_ok=True)
+        fd, tmp_path = tempfile.mkstemp(dir=self.checkpoint_dir, suffix=".tmp")
+        os.close(fd)
+        try:
+            torch.save(state, tmp_path)
+            shutil.move(tmp_path, self.path)
+        except Exception as e:
+            if os.path.exists(tmp_path):
+                os.remove(tmp_path)
+            raise CheckpointError(f"Failed to save checkpoint to '{self.path}': {e}") from e
+
+    def load(self, map_location: Optional[str] = None) -> Dict[str, Any]:
+        if not self.exists():
+            raise CheckpointError(f"No checkpoint found at '{self.path}'.")
+        try:
+            return torch.load(self.path, map_location=map_location, weights_only=False)
+        except Exception as e:
+            raise CheckpointError(
+                f"Checkpoint at '{self.path}' is corrupt or incompatible: {e}"
+            ) from e
+
+    def clear(self) -> None:
+        if os.path.isdir(self.checkpoint_dir):
+            shutil.rmtree(self.checkpoint_dir, ignore_errors=True)

tensorless/cli/__init__.py

@@ -0,0 +1,3 @@
+from .main import main
+
+__all__ = ["main"]