tensorless 0.6.0__tar.gz → 0.7.0__tar.gz

{tensorless-0.6.0/tensorless.egg-info → tensorless-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorless
-Version: 0.6.0
+Version: 0.7.0
 Summary: ML with maximum automation and minimum setup.
 Author: Tensorless Contributors
 License: MIT
@@ -105,4 +105,3 @@ without a PyTorch dependency. Accelerator cache helpers are available as
 
 See the [documentation](docs/quickstart.md) for data formats, configuration,
 checkpointing, and the command-line interface.
-pypi-AgEIcHlwaS5vcmcCJDJmYzZiYmQ1LTI1YTAtNGNlZi05OWE2LTliMjg3ZjY5MThiZQACElsxLFsidGVuc29ybGVzcyJdXQACLFsyLFsiNjcxZGZmNDQtZmVmMC00MWNiLWIzYzYtZWQzMTI3OWU4NWM5Il1dAAAGIDZAp3AAlp0idrTOMPJ227qF_7W0LDsyUpi6pfqf0aV0

{tensorless-0.6.0 → tensorless-0.7.0}/README.md

@@ -78,5 +78,4 @@ without a PyTorch dependency. Accelerator cache helpers are available as
 `tensorless.devices.clear_memory()` and `tensorless.devices.memory_stats()`.
 
 See the [documentation](docs/quickstart.md) for data formats, configuration,
-checkpointing, and the command-line interface.
-pypi-AgEIcHlwaS5vcmcCJDJmYzZiYmQ1LTI1YTAtNGNlZi05OWE2LTliMjg3ZjY5MThiZQACElsxLFsidGVuc29ybGVzcyJdXQACLFsyLFsiNjcxZGZmNDQtZmVmMC00MWNiLWIzYzYtZWQzMTI3OWU4NWM5Il1dAAAGIDZAp3AAlp0idrTOMPJ227qF_7W0LDsyUpi6pfqf0aV0
+checkpointing, and the command-line interface.

{tensorless-0.6.0 → tensorless-0.7.0}/docs/training.md

@@ -12,6 +12,28 @@ Returns a `LoadedModel` (see [inference.md](inference.md)) ready for
 predictions, and writes `model.tl` (plus a `model.tl.ckpt/` checkpoint
 directory) to the current directory.
 
+## Pretraining and fine-tuning
+
+Use the built-in corpus (or any text corpus) to create a base model, then
+continue training its learned weights on your own data:
+
+```python
+base = tl.pretrain(out="english_pretrained.tl", epochs=10)
+model = tl.train(
+    "./my_text.txt",
+    pretrained="english_pretrained.tl",
+    out="my_model.tl",
+    epochs=5,
+    learning_rate=1e-4,
+)
+```
+
+The source tokenizer and compatible architecture are reused automatically.
+The fine-tuning run starts a fresh optimizer and scheduler; interrupted
+fine-tuning still resumes from its own checkpoint. Use
+`tl.load_pretrained("english_pretrained.tl")` when you want to load the base
+model directly for inference.
+
 ## Supported data formats
 
 | Format | Notes |

{tensorless-0.6.0 → tensorless-0.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tensorless"
-version = "0.6.0"
+version = "0.7.0"
 description = "ML with maximum automation and minimum setup."
 readme = "README.md"
 requires-python = ">=3.9"

{tensorless-0.6.0 → tensorless-0.7.0}/tensorless/__init__.py

@@ -12,7 +12,7 @@ ML with maximum automation and minimum setup.
 See https://github.com/tensorless/tensorless for full documentation.
 """
 
-from .api import train, pretrain, run, load, inspect
+from .api import train, pretrain, run, load, load_pretrained, inspect
 from .config import TrainConfig
 from .errors import (
     TensorlessError,
@@ -29,6 +29,7 @@ __all__ = [
     "pretrain",
     "run",
     "load",
+    "load_pretrained",
     "inspect",
     "TrainConfig",
     "TensorlessError",

{tensorless-0.6.0 → tensorless-0.7.0}/tensorless/api.py

@@ -55,7 +55,8 @@ def inspect(path: str) -> InspectionReport:
 def train(path: str, **kwargs: Any) -> LoadedModel:
     """Train a model on the dataset at `path`, fully automatically by
     default. Any field of `TrainConfig` can be overridden via keyword
-    argument, e.g. `tl.train("./data", d_model=512, layers=6)`.
+    argument, e.g. `tl.train("./data", d_model=512, layers=6)`. Pass
+    `pretrained="base.tl"` to fine-tune an existing Tensorless model.
 
     Implements the "Smart Auto Check":
       - if an up-to-date trained model already exists for this exact
@@ -65,6 +66,14 @@ def train(path: str, **kwargs: Any) -> LoadedModel:
         `ask_on_data_change=True`), unless `force=True` is passed
     """
     user_cfg = _build_train_config(**kwargs)
+    pretrained_state = None
+    if user_cfg.pretrained:
+        try:
+            pretrained_state = load_tl(user_cfg.pretrained)
+        except Exception as exc:
+            raise ModelError(
+                f"Could not load pretrained model '{user_cfg.pretrained}': {exc}"
+            ) from exc
     out = user_cfg.out or "model.tl"
     checkpoint_dir = user_cfg.checkpoint_dir or (out + ".ckpt")
     checkpoint_mgr = CheckpointManager(checkpoint_dir)
@@ -130,6 +139,16 @@ def train(path: str, **kwargs: Any) -> LoadedModel:
     ds = load_dataset(path)
     resolved = resolve_config(ds, user_cfg)
     cfg = resolved.to_dict()
+    if pretrained_state is not None:
+        source_cfg = pretrained_state["config"]
+        for field in ("d_model", "layers", "heads", "ff_mult", "max_seq_len", "tokenizer", "bpe_vocab_size"):
+            if field not in user_cfg.overrides() and field in source_cfg:
+                cfg[field] = source_cfg[field]
+        if pretrained_state["task"] != cfg["task"] or pretrained_state["model_type"] != cfg["model_type"]:
+            raise ModelError(
+                "Pretrained model and target training data must use the same "
+                "task and model_type."
+            )
 
     if resume_state is not None:
         # Resumed runs must keep the exact architecture/config used
@@ -143,6 +162,7 @@ def train(path: str, **kwargs: Any) -> LoadedModel:
         checkpoint_mgr=checkpoint_mgr,
         dataset_fingerprint=fingerprint,
         resume_state=resume_state,
+        pretrained_state=pretrained_state if resume_state is None else None,
         log_fn=print if cfg.get("verbose", True) else (lambda *a, **k: None),
     )
 
@@ -205,6 +225,11 @@ def load(path: str, device: Optional[str] = None) -> LoadedModel:
     return load_model(path, device=device)
 
 
+def load_pretrained(path: str, device: Optional[str] = None) -> LoadedModel:
+    """Load a portable model intended to be used as a fine-tuning base."""
+    return load_model(path, device=device)
+
+
 def run(path: str, prompt: Optional[str] = None) -> Any:
     """Run a trained `.tl` model.
 

{tensorless-0.6.0 → tensorless-0.7.0}/tensorless/auto/config.py

@@ -101,6 +101,7 @@ def resolve_config(ds: Dataset, user: TrainConfig) -> ResolvedConfig:
     max_seq_len = user.max_seq_len or (256 if ds.kind in ("text", "text_labeled") else 1)
     resolved = ResolvedConfig(
         out=out,
+        pretrained=user.pretrained,
         force=bool(user.force),
         resume=user.resume,
         ask_on_data_change=bool(user.ask_on_data_change),

{tensorless-0.6.0 → tensorless-0.7.0}/tensorless/config.py

@@ -20,6 +20,7 @@ class TrainConfig:
 
     # --- output / lifecycle ---
     out: Optional[str] = None                 # output .tl path, default "model.tl"
+    pretrained: Optional[str] = None          # optional .tl weights to fine-tune
     force: bool = False                        # force retraining even if unchanged
     resume: Optional[bool] = None               # force/forbid resume (None = auto)
     ask_on_data_change: bool = False            # raise instead of auto-retrain on data change
@@ -81,6 +82,7 @@ class ResolvedConfig:
     """
 
     out: str
+    pretrained: Optional[str]
     force: bool
     resume: Optional[bool]
     ask_on_data_change: bool

{tensorless-0.6.0 → tensorless-0.7.0}/tensorless/engine.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from typing import Dict, Iterable, Iterator
+import math
 import numpy as np
 
 
@@ -226,10 +227,12 @@ class LambdaScheduler:
     def _factor(self, step):
         if self.warmup_steps and step < self.warmup_steps:
             return (step + 1) / self.warmup_steps
-        return max(
-            0.1,
-            1.0 - (step - self.warmup_steps) / max(1, self.total_steps - self.warmup_steps),
+        progress = min(
+            1.0,
+            max(0.0, (step - self.warmup_steps) /
+                max(1, self.total_steps - self.warmup_steps)),
         )
+        return 0.1 + 0.9 * 0.5 * (1.0 + math.cos(math.pi * progress))
 
     def step(self):
         self.step_count += 1

{tensorless-0.6.0 → tensorless-0.7.0}/tensorless/serialization/tl_format.py

@@ -56,6 +56,7 @@ def _migrate_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
 
     config = migrated.get("config")
     if isinstance(config, dict):
+        config.setdefault("pretrained", None)
         config.setdefault("tokenizer", "char")
         config.setdefault("bpe_vocab_size", 1000)
         config.setdefault("precision", "fp32")

{tensorless-0.6.0 → tensorless-0.7.0}/tensorless/training/trainer.py

@@ -53,13 +53,15 @@ def _show_progress(epoch, epochs, step, total_steps, loss):
 
 def run_training(ds: Dataset, cfg: Dict[str, Any], checkpoint_mgr: CheckpointManager,
                  dataset_fingerprint: str, resume_state: Optional[Dict[str, Any]] = None,
+                 pretrained_state: Optional[Dict[str, Any]] = None,
                  log_fn=print) -> Dict[str, Any]:
     np.random.seed(cfg["seed"])
     task, model_type = cfg["task"], cfg["model_type"]
     device = get_device(cfg["device"])
     if cfg["verbose"]: log_fn(f"[tensorless] task={task} model={model_type} device={cfg['device']} precision={cfg['precision']}")
-    tokenizer = tokenizer_from_state_dict(resume_state["tokenizer_state"]) if resume_state and resume_state.get("tokenizer_state") else None
-    preprocessor = TabularPreprocessor.from_state_dict(resume_state["preprocessor_state"]) if resume_state and resume_state.get("preprocessor_state") else None
+    source_state = resume_state or pretrained_state
+    tokenizer = tokenizer_from_state_dict(source_state["tokenizer_state"]) if source_state and source_state.get("tokenizer_state") else None
+    preprocessor = TabularPreprocessor.from_state_dict(source_state["preprocessor_state"]) if source_state and source_state.get("preprocessor_state") else None
     if task == "text-generation": prepared = dp.prepare_text_generation(ds, cfg, tokenizer=tokenizer)
     elif task == "text-classification": prepared = dp.prepare_text_classification(ds, cfg, tokenizer=tokenizer, classes=resume_state["meta"]["classes"] if resume_state else None)
     elif task in ("classification", "regression"): prepared = dp.prepare_tabular(ds, cfg, task, preprocessor)
@@ -71,6 +73,14 @@ def run_training(ds: Dataset, cfg: Dict[str, Any], checkpoint_mgr: CheckpointMan
         elif device == "mps":
             backend = "mlx"
     model = build_model(task, model_type, cfg, prepared.meta, backend=backend)
+    if pretrained_state is not None:
+        try:
+            model.load_state_dict(pretrained_state["model_state_dict"])
+        except (KeyError, TypeError, ValueError) as exc:
+            raise ValueError(
+                "Pretrained weights are incompatible with the target model. "
+                "Keep the task, tokenizer, and architecture dimensions compatible."
+            ) from exc
     optimizer = _build_optimizer(model, cfg)
     total_steps = cfg.get("max_steps") or max(1, len(prepared.train_loader)) * cfg["epochs"]
     scheduler = LambdaScheduler(optimizer, cfg["warmup_steps"], total_steps)
@@ -99,7 +109,7 @@ def run_training(ds: Dataset, cfg: Dict[str, Any], checkpoint_mgr: CheckpointMan
             if not np.isfinite(last_train_loss):
                 raise FloatingPointError(f"Non-finite training loss at step {global_step + 1}: {last_train_loss}")
             if cfg["grad_clip"]:
-                norm = np.sqrt(sum(float(np.sum(p.grad * p.grad)) for p in model.parameters()))
+                norm = np.sqrt(sum(float(np.sum(p.grad.astype(np.float64) ** 2)) for p in model.parameters()))
                 if not np.isfinite(norm):
                     raise FloatingPointError(f"Non-finite gradient norm at step {global_step + 1}: {norm}")
                 if norm > cfg["grad_clip"]:

{tensorless-0.6.0 → tensorless-0.7.0/tensorless.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorless
-Version: 0.6.0
+Version: 0.7.0
 Summary: ML with maximum automation and minimum setup.
 Author: Tensorless Contributors
 License: MIT
@@ -105,4 +105,3 @@ without a PyTorch dependency. Accelerator cache helpers are available as
 
 See the [documentation](docs/quickstart.md) for data formats, configuration,
 checkpointing, and the command-line interface.
-pypi-AgEIcHlwaS5vcmcCJDJmYzZiYmQ1LTI1YTAtNGNlZi05OWE2LTliMjg3ZjY5MThiZQACElsxLFsidGVuc29ybGVzcyJdXQACLFsyLFsiNjcxZGZmNDQtZmVmMC00MWNiLWIzYzYtZWQzMTI3OWU4NWM5Il1dAAAGIDZAp3AAlp0idrTOMPJ227qF_7W0LDsyUpi6pfqf0aV0

{tensorless-0.6.0 → tensorless-0.7.0}/tests/test_train_text_generation.py

@@ -59,6 +59,21 @@ def test_builtin_english_pretraining(workdir):
     assert model.tokenizer is not None
 
 
+def test_finetuning_uses_pretrained_weights(text_corpus, workdir):
+    tl.pretrain(
+        out="base.tl", epochs=1, max_steps=1, max_seq_len=32,
+        d_model=16, layers=1, heads=2, batch_size=2, checkpoint_every=1,
+        verbose=False,
+    )
+    model = tl.train(
+        text_corpus, pretrained="base.tl", out="finetuned.tl",
+        epochs=1, max_steps=1, batch_size=2, checkpoint_every=1,
+        verbose=False,
+    )
+    assert model.config["d_model"] == 16
+    assert tl.load_pretrained("base.tl").task == "text-generation"
+
+
 def test_long_text_generation_is_streamed_in_batches():
     ds = Dataset(kind="text", source="memory", texts=["the quick brown fox " * 2000])
     cfg = resolve_config(ds, TrainConfig(max_seq_len=32, tokenizer="char"))