tensorless 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tensorless-0.3.0/tensorless.egg-info → tensorless-0.4.0}/PKG-INFO +4 -4
- {tensorless-0.3.0 → tensorless-0.4.0}/README.md +2 -2
- {tensorless-0.3.0 → tensorless-0.4.0}/docs/api_reference.md +1 -1
- {tensorless-0.3.0 → tensorless-0.4.0}/docs/architecture.md +1 -1
- tensorless-0.4.0/docs/installation.md +29 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/docs/tl_format.md +4 -4
- {tensorless-0.3.0 → tensorless-0.4.0}/docs/troubleshooting.md +5 -7
- {tensorless-0.3.0 → tensorless-0.4.0}/pyproject.toml +2 -2
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/auto/config.py +1 -1
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/checkpoint/manager.py +5 -3
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/data/tabular.py +9 -9
- tensorless-0.4.0/tensorless/devices/__init__.py +3 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/devices/device.py +9 -40
- tensorless-0.4.0/tensorless/engine.py +187 -0
- tensorless-0.4.0/tensorless/models/mlp.py +71 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/models/registry.py +1 -3
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/models/transformer.py +91 -11
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/runtime.py +12 -15
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/serialization/tl_format.py +5 -3
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/training/data_prep.py +47 -30
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/training/trainer.py +93 -10
- {tensorless-0.3.0 → tensorless-0.4.0/tensorless.egg-info}/PKG-INFO +4 -4
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless.egg-info/SOURCES.txt +1 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless.egg-info/requires.txt +1 -1
- {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_end_to_end.py +2 -7
- {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_serialization.py +9 -15
- tensorless-0.3.0/docs/installation.md +0 -43
- tensorless-0.3.0/tensorless/devices/__init__.py +0 -3
- tensorless-0.3.0/tensorless/models/mlp.py +0 -64
- {tensorless-0.3.0 → tensorless-0.4.0}/LICENSE +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/MANIFEST.in +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/docs/automatic_mode.md +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/docs/checkpointing.md +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/docs/cli.md +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/docs/configuration.md +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/docs/contributing.md +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/docs/examples.md +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/docs/inference.md +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/docs/limitations.md +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/docs/quickstart.md +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/docs/roadmap.md +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/docs/training.md +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/docs/tutorial.md +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/examples/tabular_classification_example.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/examples/tabular_regression_example.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/examples/text_classification_example.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/examples/text_generation_example.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/setup.cfg +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/__init__.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/_version.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/api.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/auto/__init__.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/auto/detector.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/checkpoint/__init__.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/cli/__init__.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/cli/main.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/config.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/data/__init__.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/data/english_grammar.txt +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/data/fingerprint.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/data/inspector.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/data/loader.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/errors.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/models/__init__.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/serialization/__init__.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/tokenization/__init__.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/tokenization/bpe_tokenizer.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/tokenization/char_tokenizer.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/training/__init__.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/training/early_stopping.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless.egg-info/dependency_links.txt +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless.egg-info/entry_points.txt +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tensorless.egg-info/top_level.txt +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_auto_detection.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_checkpoint_resume.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_cli.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_data_loading.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_fingerprint.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_train_tabular.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_train_text_classification.py +0 -0
- {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_train_text_generation.py +0 -0
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tensorless
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: ML with maximum automation and minimum setup.
|
|
5
5
|
Author: Tensorless Contributors
|
|
6
6
|
License: MIT
|
|
7
7
|
Requires-Python: >=3.9
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
9
9
|
License-File: LICENSE
|
|
10
|
-
Requires-Dist:
|
|
10
|
+
Requires-Dist: numpy>=1.24
|
|
11
11
|
Provides-Extra: dev
|
|
12
12
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
13
13
|
Dynamic: license-file
|
|
14
14
|
|
|
15
15
|
# Tensorless
|
|
16
16
|
|
|
17
|
-
Tensorless trains small
|
|
17
|
+
Tensorless trains small native NumPy models with sensible defaults. It supports
|
|
18
18
|
text generation, text classification, tabular classification, and regression.
|
|
19
19
|
|
|
20
20
|
## Install
|
|
@@ -37,7 +37,7 @@ tokenizer; use `tokenizer="char"` for a character-level model. Tensorless
|
|
|
37
37
|
derives model size, batch size, epochs, validation, device, and BPE vocabulary
|
|
38
38
|
size from the data, while every setting can be overridden.
|
|
39
39
|
|
|
40
|
-
Long text is tokenized lazily and fed through
|
|
40
|
+
Long text is tokenized lazily and fed through the native engine in fixed-size batches.
|
|
41
41
|
The automatic batch size uses a token budget; reduce `batch_size` if your
|
|
42
42
|
available memory is limited.
|
|
43
43
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Tensorless
|
|
2
2
|
|
|
3
|
-
Tensorless trains small
|
|
3
|
+
Tensorless trains small native NumPy models with sensible defaults. It supports
|
|
4
4
|
text generation, text classification, tabular classification, and regression.
|
|
5
5
|
|
|
6
6
|
## Install
|
|
@@ -23,7 +23,7 @@ tokenizer; use `tokenizer="char"` for a character-level model. Tensorless
|
|
|
23
23
|
derives model size, batch size, epochs, validation, device, and BPE vocabulary
|
|
24
24
|
size from the data, while every setting can be overridden.
|
|
25
25
|
|
|
26
|
-
Long text is tokenized lazily and fed through
|
|
26
|
+
Long text is tokenized lazily and fed through the native engine in fixed-size batches.
|
|
27
27
|
The automatic batch size uses a token budget; reduce `batch_size` if your
|
|
28
28
|
available memory is limited.
|
|
29
29
|
|
|
@@ -59,7 +59,7 @@ Returned by both `train()` and `load()`.
|
|
|
59
59
|
| `.info()` | all | Dict summary: task, model type, versions, config, metrics, param count |
|
|
60
60
|
|
|
61
61
|
Attributes: `.task`, `.model_type`, `.config`, `.meta`, `.metrics`,
|
|
62
|
-
`.dataset_fingerprint`, `.model` (the underlying
|
|
62
|
+
`.dataset_fingerprint`, `.model` (the underlying native Tensorless model),
|
|
63
63
|
`.tokenizer` (`CharTokenizer` or `None`), `.preprocessor`
|
|
64
64
|
(`TabularPreprocessor` or `None`).
|
|
65
65
|
|
|
@@ -37,7 +37,7 @@ tensorless/
|
|
|
37
37
|
├── serialization/
|
|
38
38
|
│ └── tl_format.py save_tl/load_tl: the .tl file format
|
|
39
39
|
├── devices/
|
|
40
|
-
│ └── device.py
|
|
40
|
+
│ └── device.py native CPU device resolution
|
|
41
41
|
└── cli/
|
|
42
42
|
└── main.py argparse-based CLI
|
|
43
43
|
```
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Installation
|
|
2
|
+
|
|
3
|
+
## Requirements
|
|
4
|
+
|
|
5
|
+
- Python 3.9 or later
|
|
6
|
+
- NumPy 1.24 or later (installed automatically as a dependency)
|
|
7
|
+
- Tensorless currently runs its native vectorized engine on CPU; accelerator
|
|
8
|
+
backends are planned without changing the public API.
|
|
9
|
+
|
|
10
|
+
## Verify your installation
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
python -c "import tensorless as tl; print(tl.__version__)"
|
|
14
|
+
tensorless --help
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
You should see a version string printed and the CLI's help text.
|
|
18
|
+
|
|
19
|
+
## Runtime support
|
|
20
|
+
|
|
21
|
+
The native engine uses NumPy vectorized CPU kernels and automatically resolves
|
|
22
|
+
the device to CPU. You can still specify `device="cpu"` explicitly in
|
|
23
|
+
`tl.train(...)`; the device option remains forward-compatible with future
|
|
24
|
+
accelerator backends.
|
|
25
|
+
|
|
26
|
+
## Troubleshooting installation
|
|
27
|
+
|
|
28
|
+
See [troubleshooting.md](troubleshooting.md#installation-issues) for
|
|
29
|
+
common installation problems.
|
|
@@ -6,9 +6,9 @@ original training data or code beyond having Tensorless installed.
|
|
|
6
6
|
|
|
7
7
|
## Structure
|
|
8
8
|
|
|
9
|
-
Under the hood, `.tl` is a
|
|
10
|
-
|
|
11
|
-
dictionary:
|
|
9
|
+
Under the hood, `.tl` is a portable Python pickle file written by the native
|
|
10
|
+
Tensorless serializer (see `tensorless/serialization/tl_format.py`) containing
|
|
11
|
+
a dictionary:
|
|
12
12
|
|
|
13
13
|
```python
|
|
14
14
|
{
|
|
@@ -18,7 +18,7 @@ dictionary:
|
|
|
18
18
|
"model_type": "transformer", # or mlp
|
|
19
19
|
"config": { ... }, # full resolved TrainConfig used
|
|
20
20
|
"meta": { ... }, # vocab_size / n_classes / column info -- whatever the model needs to rebuild
|
|
21
|
-
|
|
21
|
+
"model_state_dict": { ... }, # native NumPy model weights
|
|
22
22
|
"tokenizer_state": { ... } | None, # CharTokenizer vocab, for text tasks
|
|
23
23
|
"preprocessor_state": { ... } | None, # TabularPreprocessor state, for tabular tasks
|
|
24
24
|
"dataset_fingerprint": "...", # fingerprint of the training dataset
|
|
@@ -2,11 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
## Installation issues
|
|
4
4
|
|
|
5
|
-
**`ModuleNotFoundError: No module named '
|
|
6
|
-
|
|
7
|
-
install
|
|
8
|
-
[pytorch.org/get-started](https://pytorch.org/get-started/locally/) for
|
|
9
|
-
your platform, then reinstall Tensorless.
|
|
5
|
+
**`ModuleNotFoundError: No module named 'numpy'`**
|
|
6
|
+
NumPy is installed automatically with `pip install -e .`. If it is missing,
|
|
7
|
+
install it with `python -m pip install numpy`, then reinstall Tensorless.
|
|
10
8
|
|
|
11
9
|
**`tensorless: command not found` after installing**
|
|
12
10
|
Make sure the Python environment's `bin`/`Scripts` directory is on your
|
|
@@ -69,8 +67,8 @@ fingerprint across runs.
|
|
|
69
67
|
|
|
70
68
|
**`CheckpointError: Checkpoint at '...' is corrupt or incompatible.`**
|
|
71
69
|
The checkpoint file was likely truncated by an interruption during the
|
|
72
|
-
(non-atomic part of the) write, or created by an incompatible
|
|
73
|
-
version. Delete the `.ckpt` directory and retrain — you'll lose progress
|
|
70
|
+
(non-atomic part of the) write, or created by an incompatible Tensorless
|
|
71
|
+
version. Delete the `.ckpt` directory and retrain — you'll lose progress
|
|
74
72
|
on the interrupted run, but the checkpoint being unreadable means it
|
|
75
73
|
can't be safely resumed regardless.
|
|
76
74
|
|
|
@@ -4,14 +4,14 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "tensorless"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.0"
|
|
8
8
|
description = "ML with maximum automation and minimum setup."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
11
11
|
license = { text = "MIT" }
|
|
12
12
|
authors = [{ name = "Tensorless Contributors" }]
|
|
13
13
|
dependencies = [
|
|
14
|
-
"
|
|
14
|
+
"numpy>=1.24",
|
|
15
15
|
]
|
|
16
16
|
|
|
17
17
|
[project.optional-dependencies]
|
|
@@ -115,7 +115,7 @@ def resolve_config(ds: Dataset, user: TrainConfig) -> ResolvedConfig:
|
|
|
115
115
|
tokenizer=tokenizer,
|
|
116
116
|
bpe_vocab_size=user.bpe_vocab_size if user.bpe_vocab_size is not None else _auto_vocab_size(ds),
|
|
117
117
|
optimizer=user.optimizer or "adamw",
|
|
118
|
-
learning_rate=user.learning_rate or (3e-4 if model_type == "transformer" else 1e-3),
|
|
118
|
+
learning_rate=user.learning_rate or (3e-3 if task == "text-classification" else (3e-4 if model_type == "transformer" else 1e-3)),
|
|
119
119
|
weight_decay=user.weight_decay if user.weight_decay is not None else 0.01,
|
|
120
120
|
batch_size=user.batch_size if user.batch_size is not None else _auto_batch_size(n, max_seq_len),
|
|
121
121
|
epochs=user.epochs or _auto_epochs(n),
|
|
@@ -18,7 +18,7 @@ import shutil
|
|
|
18
18
|
import tempfile
|
|
19
19
|
from typing import Any, Dict, Optional
|
|
20
20
|
|
|
21
|
-
import
|
|
21
|
+
import pickle
|
|
22
22
|
|
|
23
23
|
from ..errors import CheckpointError
|
|
24
24
|
|
|
@@ -44,7 +44,8 @@ class CheckpointManager:
|
|
|
44
44
|
fd, tmp_path = tempfile.mkstemp(dir=self.checkpoint_dir, suffix=".tmp")
|
|
45
45
|
os.close(fd)
|
|
46
46
|
try:
|
|
47
|
-
|
|
47
|
+
with open(tmp_path, "wb") as handle:
|
|
48
|
+
pickle.dump(state, handle, protocol=pickle.HIGHEST_PROTOCOL)
|
|
48
49
|
shutil.move(tmp_path, self.path)
|
|
49
50
|
except Exception as e:
|
|
50
51
|
if os.path.exists(tmp_path):
|
|
@@ -55,7 +56,8 @@ class CheckpointManager:
|
|
|
55
56
|
if not self.exists():
|
|
56
57
|
raise CheckpointError(f"No checkpoint found at '{self.path}'.")
|
|
57
58
|
try:
|
|
58
|
-
|
|
59
|
+
with open(self.path, "rb") as handle:
|
|
60
|
+
return pickle.load(handle)
|
|
59
61
|
except Exception as e:
|
|
60
62
|
raise CheckpointError(
|
|
61
63
|
f"Checkpoint at '{self.path}' is corrupt or incompatible: {e}"
|
|
@@ -17,7 +17,7 @@ from datetime import datetime
|
|
|
17
17
|
from dataclasses import dataclass, field
|
|
18
18
|
from typing import Any, Dict, List, Optional, Tuple
|
|
19
19
|
|
|
20
|
-
import
|
|
20
|
+
import numpy as np
|
|
21
21
|
|
|
22
22
|
_MISSING = "<missing>"
|
|
23
23
|
_UNK = "<unk>"
|
|
@@ -132,13 +132,13 @@ class TabularPreprocessor:
|
|
|
132
132
|
|
|
133
133
|
def transform(
|
|
134
134
|
self, records: List[Dict[str, Any]], with_target: bool = True
|
|
135
|
-
) -> Dict[str,
|
|
135
|
+
) -> Dict[str, np.ndarray]:
|
|
136
136
|
n = len(records)
|
|
137
137
|
num_cols = self.numeric_columns
|
|
138
138
|
cat_cols = self.categorical_columns
|
|
139
139
|
|
|
140
|
-
numeric =
|
|
141
|
-
categorical =
|
|
140
|
+
numeric = np.zeros((n, max(len(num_cols), 1)), dtype=np.float32)
|
|
141
|
+
categorical = np.zeros((n, max(len(cat_cols), 1)), dtype=np.int64)
|
|
142
142
|
|
|
143
143
|
for i, r in enumerate(records):
|
|
144
144
|
for j, col in enumerate(num_cols):
|
|
@@ -158,14 +158,14 @@ class TabularPreprocessor:
|
|
|
158
158
|
|
|
159
159
|
if with_target and self.target_column is not None:
|
|
160
160
|
if self.task == "regression":
|
|
161
|
-
target =
|
|
161
|
+
target = np.zeros(n, dtype=np.float32)
|
|
162
162
|
for i, r in enumerate(records):
|
|
163
163
|
v = _try_float(r.get(self.target_column))
|
|
164
164
|
v = self.target_mean if v is None else v
|
|
165
165
|
target[i] = (v - self.target_mean) / self.target_std
|
|
166
166
|
out["target"] = target
|
|
167
167
|
else:
|
|
168
|
-
target =
|
|
168
|
+
target = np.zeros(n, dtype=np.int64)
|
|
169
169
|
for i, r in enumerate(records):
|
|
170
170
|
raw = str(r.get(self.target_column))
|
|
171
171
|
idx = self.classes.index(raw) if raw in self.classes else 0
|
|
@@ -174,10 +174,10 @@ class TabularPreprocessor:
|
|
|
174
174
|
|
|
175
175
|
return out
|
|
176
176
|
|
|
177
|
-
def inverse_target(self, values:
|
|
177
|
+
def inverse_target(self, values: np.ndarray) -> List[Any]:
|
|
178
178
|
if self.task == "regression":
|
|
179
|
-
return [(v
|
|
180
|
-
return [self.classes[int(v
|
|
179
|
+
return [float(v * self.target_std + self.target_mean) for v in values]
|
|
180
|
+
return [self.classes[int(v)] for v in values]
|
|
181
181
|
|
|
182
182
|
def categorical_vocab_sizes(self) -> List[int]:
|
|
183
183
|
return [len(self.column_stats[c].vocab) for c in self.categorical_columns]
|
|
@@ -11,37 +11,21 @@ from __future__ import annotations
|
|
|
11
11
|
|
|
12
12
|
from typing import Optional, Tuple
|
|
13
13
|
|
|
14
|
-
import torch
|
|
15
|
-
|
|
16
14
|
|
|
17
15
|
def _tpu_available() -> bool:
|
|
18
|
-
|
|
19
|
-
import torch_xla.core.xla_model as xm # noqa: F401
|
|
20
|
-
|
|
21
|
-
return True
|
|
22
|
-
except Exception:
|
|
23
|
-
return False
|
|
16
|
+
return False
|
|
24
17
|
|
|
25
18
|
|
|
26
19
|
def _cuda_available() -> bool:
|
|
27
|
-
|
|
28
|
-
return torch.cuda.is_available() and torch.cuda.device_count() > 0
|
|
29
|
-
except Exception:
|
|
30
|
-
return False
|
|
20
|
+
return False
|
|
31
21
|
|
|
32
22
|
|
|
33
23
|
def _mps_available() -> bool:
|
|
34
|
-
|
|
35
|
-
return torch.backends.mps.is_available()
|
|
36
|
-
except Exception:
|
|
37
|
-
return False
|
|
24
|
+
return False
|
|
38
25
|
|
|
39
26
|
|
|
40
27
|
def _cuda_supports_bf16() -> bool:
|
|
41
|
-
|
|
42
|
-
return torch.cuda.is_bf16_supported()
|
|
43
|
-
except Exception:
|
|
44
|
-
return False
|
|
28
|
+
return False
|
|
45
29
|
|
|
46
30
|
|
|
47
31
|
def auto_select_device(user_device: Optional[str], user_precision: Optional[str]) -> Tuple[str, str]:
|
|
@@ -85,23 +69,8 @@ def auto_select_device(user_device: Optional[str], user_precision: Optional[str]
|
|
|
85
69
|
return device, precision
|
|
86
70
|
|
|
87
71
|
|
|
88
|
-
def
|
|
89
|
-
"""
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
if device == "tpu":
|
|
94
|
-
import torch_xla.core.xla_model as xm
|
|
95
|
-
|
|
96
|
-
return xm.xla_device()
|
|
97
|
-
if device == "cuda":
|
|
98
|
-
if not _cuda_available():
|
|
99
|
-
return torch.device("cpu")
|
|
100
|
-
return torch.device("cuda")
|
|
101
|
-
if device == "mps":
|
|
102
|
-
if not _mps_available():
|
|
103
|
-
return torch.device("cpu")
|
|
104
|
-
return torch.device("mps")
|
|
105
|
-
return torch.device("cpu")
|
|
106
|
-
except Exception:
|
|
107
|
-
return torch.device("cpu")
|
|
72
|
+
def get_device(device: str) -> str:
|
|
73
|
+
"""Return the native engine device name (currently CPU only)."""
|
|
74
|
+
return "cpu"
|
|
75
|
+
|
|
76
|
+
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Small NumPy training engine used by Tensorless models."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Dict, Iterable, Iterator
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Parameter:
|
|
10
|
+
def __init__(self, data):
|
|
11
|
+
self.data = np.asarray(data, dtype=np.float32)
|
|
12
|
+
self.grad = np.zeros_like(self.data)
|
|
13
|
+
|
|
14
|
+
def numel(self) -> int:
|
|
15
|
+
return int(self.data.size)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Module:
|
|
19
|
+
def parameters(self) -> Iterator[Parameter]:
|
|
20
|
+
for value in self.__dict__.values():
|
|
21
|
+
if isinstance(value, Parameter):
|
|
22
|
+
yield value
|
|
23
|
+
elif isinstance(value, Module):
|
|
24
|
+
yield from value.parameters()
|
|
25
|
+
elif isinstance(value, (list, tuple)):
|
|
26
|
+
for item in value:
|
|
27
|
+
if isinstance(item, Parameter):
|
|
28
|
+
yield item
|
|
29
|
+
elif isinstance(item, Module):
|
|
30
|
+
yield from item.parameters()
|
|
31
|
+
|
|
32
|
+
def named_parameters(self, prefix: str = ""):
|
|
33
|
+
for name, value in self.__dict__.items():
|
|
34
|
+
full = f"{prefix}.{name}" if prefix else name
|
|
35
|
+
if isinstance(value, Parameter):
|
|
36
|
+
yield full, value
|
|
37
|
+
elif isinstance(value, Module):
|
|
38
|
+
yield from value.named_parameters(full)
|
|
39
|
+
elif isinstance(value, (list, tuple)):
|
|
40
|
+
for i, item in enumerate(value):
|
|
41
|
+
if isinstance(item, Parameter):
|
|
42
|
+
yield f"{full}.{i}", item
|
|
43
|
+
elif isinstance(item, Module):
|
|
44
|
+
yield from item.named_parameters(f"{full}.{i}")
|
|
45
|
+
|
|
46
|
+
def state_dict(self) -> Dict[str, np.ndarray]:
|
|
47
|
+
return {name: parameter.data.copy() for name, parameter in self.named_parameters()}
|
|
48
|
+
|
|
49
|
+
def load_state_dict(self, state: Dict[str, np.ndarray]) -> None:
|
|
50
|
+
current = dict(self.named_parameters())
|
|
51
|
+
missing = [name for name in current if name not in state]
|
|
52
|
+
if missing:
|
|
53
|
+
raise ValueError(f"Missing model parameters: {missing}")
|
|
54
|
+
for name, parameter in current.items():
|
|
55
|
+
value = np.asarray(state[name], dtype=np.float32)
|
|
56
|
+
if value.shape != parameter.data.shape:
|
|
57
|
+
raise ValueError(f"Shape mismatch for {name}: {value.shape} != {parameter.data.shape}")
|
|
58
|
+
parameter.data[...] = value
|
|
59
|
+
|
|
60
|
+
def train(self, mode: bool = True):
|
|
61
|
+
self.training = mode
|
|
62
|
+
return self
|
|
63
|
+
|
|
64
|
+
def eval(self):
|
|
65
|
+
return self.train(False)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def softmax_cross_entropy(logits: np.ndarray, targets: np.ndarray, ignore_index=None):
|
|
69
|
+
flat = logits.reshape(-1, logits.shape[-1]).astype(np.float64)
|
|
70
|
+
target = targets.reshape(-1).astype(np.int64)
|
|
71
|
+
valid = np.ones(len(target), dtype=bool) if ignore_index is None else target != ignore_index
|
|
72
|
+
if not np.any(valid):
|
|
73
|
+
return 0.0, np.zeros_like(logits)
|
|
74
|
+
shifted = flat - flat.max(axis=1, keepdims=True)
|
|
75
|
+
probs = np.exp(shifted)
|
|
76
|
+
probs /= probs.sum(axis=1, keepdims=True)
|
|
77
|
+
rows = np.arange(len(target))[valid]
|
|
78
|
+
loss = -np.log(np.maximum(probs[rows, target[valid]], 1e-12)).mean()
|
|
79
|
+
grad = np.zeros_like(flat, dtype=np.float32)
|
|
80
|
+
grad[valid] = probs[valid].astype(np.float32)
|
|
81
|
+
grad[rows, target[valid]] -= 1.0
|
|
82
|
+
grad[valid] /= valid.sum()
|
|
83
|
+
return float(loss), grad.reshape(logits.shape)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class Optimizer:
|
|
87
|
+
def __init__(self, parameters: Iterable[Parameter], lr: float, weight_decay: float = 0.0):
|
|
88
|
+
self.parameters_list = list(parameters)
|
|
89
|
+
self.lr = lr
|
|
90
|
+
self.weight_decay = weight_decay
|
|
91
|
+
self.step_count = 0
|
|
92
|
+
|
|
93
|
+
def zero_grad(self):
|
|
94
|
+
for parameter in self.parameters_list:
|
|
95
|
+
parameter.grad.fill(0.0)
|
|
96
|
+
|
|
97
|
+
def state_dict(self):
|
|
98
|
+
return {"step": self.step_count, "lr": self.lr, "weight_decay": self.weight_decay}
|
|
99
|
+
|
|
100
|
+
def load_state_dict(self, state):
|
|
101
|
+
self.step_count = int(state.get("step", 0))
|
|
102
|
+
self.lr = float(state.get("lr", self.lr))
|
|
103
|
+
self.weight_decay = float(state.get("weight_decay", self.weight_decay))
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class SGD(Optimizer):
|
|
107
|
+
def __init__(self, parameters, lr, weight_decay=0.0, momentum=0.9):
|
|
108
|
+
super().__init__(parameters, lr, weight_decay)
|
|
109
|
+
self.momentum = momentum
|
|
110
|
+
self.velocity = [np.zeros_like(p.data) for p in self.parameters_list]
|
|
111
|
+
|
|
112
|
+
def step(self):
|
|
113
|
+
self.step_count += 1
|
|
114
|
+
for p, velocity in zip(self.parameters_list, self.velocity):
|
|
115
|
+
velocity *= self.momentum
|
|
116
|
+
velocity += p.grad + self.weight_decay * p.data
|
|
117
|
+
p.data -= self.lr * velocity
|
|
118
|
+
|
|
119
|
+
def state_dict(self):
|
|
120
|
+
state = super().state_dict()
|
|
121
|
+
state["velocity"] = [v.copy() for v in self.velocity]
|
|
122
|
+
return state
|
|
123
|
+
|
|
124
|
+
def load_state_dict(self, state):
|
|
125
|
+
super().load_state_dict(state)
|
|
126
|
+
for target, source in zip(self.velocity, state.get("velocity", [])):
|
|
127
|
+
target[...] = source
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class Adam(Optimizer):
|
|
131
|
+
def __init__(self, parameters, lr, weight_decay=0.0, decoupled=False):
|
|
132
|
+
super().__init__(parameters, lr, weight_decay)
|
|
133
|
+
self.decoupled = decoupled
|
|
134
|
+
self.m = [np.zeros_like(p.data) for p in self.parameters_list]
|
|
135
|
+
self.v = [np.zeros_like(p.data) for p in self.parameters_list]
|
|
136
|
+
|
|
137
|
+
def step(self):
|
|
138
|
+
self.step_count += 1
|
|
139
|
+
for p, m, v in zip(self.parameters_list, self.m, self.v):
|
|
140
|
+
if self.decoupled:
|
|
141
|
+
p.data *= 1.0 - self.lr * self.weight_decay
|
|
142
|
+
gradient = p.grad
|
|
143
|
+
else:
|
|
144
|
+
gradient = p.grad + self.weight_decay * p.data
|
|
145
|
+
m *= 0.9
|
|
146
|
+
m += 0.1 * gradient
|
|
147
|
+
v *= 0.999
|
|
148
|
+
v += 0.001 * gradient * gradient
|
|
149
|
+
m_hat = m / (1.0 - 0.9 ** self.step_count)
|
|
150
|
+
v_hat = v / (1.0 - 0.999 ** self.step_count)
|
|
151
|
+
p.data -= self.lr * m_hat / (np.sqrt(v_hat) + 1e-8)
|
|
152
|
+
|
|
153
|
+
def state_dict(self):
|
|
154
|
+
state = super().state_dict()
|
|
155
|
+
state.update({"m": [x.copy() for x in self.m], "v": [x.copy() for x in self.v]})
|
|
156
|
+
return state
|
|
157
|
+
|
|
158
|
+
def load_state_dict(self, state):
|
|
159
|
+
super().load_state_dict(state)
|
|
160
|
+
for target, source in zip(self.m, state.get("m", [])):
|
|
161
|
+
target[...] = source
|
|
162
|
+
for target, source in zip(self.v, state.get("v", [])):
|
|
163
|
+
target[...] = source
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class LambdaScheduler:
|
|
167
|
+
def __init__(self, optimizer, warmup_steps, total_steps):
|
|
168
|
+
self.optimizer = optimizer
|
|
169
|
+
self.warmup_steps = warmup_steps
|
|
170
|
+
self.total_steps = total_steps
|
|
171
|
+
self.step_count = 0
|
|
172
|
+
self.base_lr = optimizer.lr
|
|
173
|
+
|
|
174
|
+
def step(self):
|
|
175
|
+
self.step_count += 1
|
|
176
|
+
step = self.step_count - 1
|
|
177
|
+
if self.warmup_steps and step < self.warmup_steps:
|
|
178
|
+
factor = (step + 1) / self.warmup_steps
|
|
179
|
+
else:
|
|
180
|
+
factor = max(0.1, 1.0 - (step - self.warmup_steps) / max(1, self.total_steps - self.warmup_steps))
|
|
181
|
+
self.optimizer.lr = self.base_lr * factor
|
|
182
|
+
|
|
183
|
+
def state_dict(self):
|
|
184
|
+
return {"step_count": self.step_count}
|
|
185
|
+
|
|
186
|
+
def load_state_dict(self, state):
|
|
187
|
+
self.step_count = int(state.get("step_count", 0))
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""NumPy MLP for tabular classification and regression."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import List
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
from ..engine import Module, Parameter
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _embedding_dim(vocab_size: int) -> int:
|
|
12
|
+
return max(2, min(32, round(1.6 * (vocab_size ** 0.56))))
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class TabularMLP(Module):
|
|
16
|
+
def __init__(self, n_numeric, categorical_vocab_sizes: List[int], d_model, layers, dropout, task, n_classes=0):
|
|
17
|
+
self.training = True
|
|
18
|
+
self.task = task
|
|
19
|
+
self.n_numeric = n_numeric
|
|
20
|
+
self.categorical_vocab_sizes = categorical_vocab_sizes
|
|
21
|
+
rng = np.random.default_rng()
|
|
22
|
+
self.embeddings = [Parameter(rng.normal(0, .02, (v, _embedding_dim(v))).astype(np.float32)) for v in categorical_vocab_sizes]
|
|
23
|
+
in_dim = n_numeric + sum(_embedding_dim(v) for v in categorical_vocab_sizes)
|
|
24
|
+
self.weights, self.biases = [], []
|
|
25
|
+
for i in range(layers):
|
|
26
|
+
fan_in = in_dim if i == 0 else d_model
|
|
27
|
+
self.weights.append(Parameter(rng.normal(0, .02, (fan_in, d_model)).astype(np.float32)))
|
|
28
|
+
self.biases.append(Parameter(np.zeros(d_model, dtype=np.float32)))
|
|
29
|
+
out_dim = n_classes if task == "classification" else 1
|
|
30
|
+
self.head_weight = Parameter(rng.normal(0, .02, (d_model if layers else in_dim, out_dim)).astype(np.float32))
|
|
31
|
+
self.head_bias = Parameter(np.zeros(out_dim, dtype=np.float32))
|
|
32
|
+
|
|
33
|
+
def forward(self, numeric, categorical, cache=False):
|
|
34
|
+
parts = [numeric.astype(np.float32)] if self.n_numeric else []
|
|
35
|
+
for i, embedding in enumerate(self.embeddings):
|
|
36
|
+
parts.append(embedding.data[categorical[:, i]])
|
|
37
|
+
x = np.concatenate(parts, axis=1) if parts else numeric.astype(np.float32)
|
|
38
|
+
activations = [x]
|
|
39
|
+
for weight, bias in zip(self.weights, self.biases):
|
|
40
|
+
x = np.maximum(0, x @ weight.data + bias.data)
|
|
41
|
+
activations.append(x)
|
|
42
|
+
out = x @ self.head_weight.data + self.head_bias.data
|
|
43
|
+
if self.task == "regression": out = out[:, 0]
|
|
44
|
+
return (out, (activations, categorical)) if cache else out
|
|
45
|
+
|
|
46
|
+
def loss_and_backward(self, numeric, categorical, target):
|
|
47
|
+
logits, (activations, categorical) = self.forward(numeric, categorical, cache=True)
|
|
48
|
+
if self.task == "regression":
|
|
49
|
+
error = logits - target
|
|
50
|
+
loss = float(np.mean(error * error))
|
|
51
|
+
grad = (2.0 / len(target)) * error[:, None]
|
|
52
|
+
else:
|
|
53
|
+
shifted = logits - logits.max(1, keepdims=True)
|
|
54
|
+
probs = np.exp(shifted); probs /= probs.sum(1, keepdims=True)
|
|
55
|
+
loss = float(-np.log(np.maximum(probs[np.arange(len(target)), target], 1e-12)).mean())
|
|
56
|
+
grad = probs; grad[np.arange(len(target)), target] -= 1; grad /= len(target)
|
|
57
|
+
self.head_weight.grad[...] = activations[-1].T @ grad
|
|
58
|
+
self.head_bias.grad[...] = grad.sum(0)
|
|
59
|
+
dx = grad @ self.head_weight.data.T
|
|
60
|
+
for i in range(len(self.weights) - 1, -1, -1):
|
|
61
|
+
dx = dx * (activations[i + 1] > 0)
|
|
62
|
+
self.weights[i].grad[...] = activations[i].T @ dx
|
|
63
|
+
self.biases[i].grad[...] = dx.sum(0)
|
|
64
|
+
dx = dx @ self.weights[i].data.T
|
|
65
|
+
offset = self.n_numeric
|
|
66
|
+
for i, embedding in enumerate(self.embeddings):
|
|
67
|
+
width = embedding.data.shape[1]
|
|
68
|
+
embedding.grad.fill(0)
|
|
69
|
+
np.add.at(embedding.grad, categorical[:, i], dx[:, offset:offset + width])
|
|
70
|
+
offset += width
|
|
71
|
+
return loss
|
|
@@ -11,14 +11,12 @@ from __future__ import annotations
|
|
|
11
11
|
|
|
12
12
|
from typing import Any, Dict
|
|
13
13
|
|
|
14
|
-
import torch.nn as nn
|
|
15
|
-
|
|
16
14
|
from .transformer import TinyTransformer
|
|
17
15
|
from .mlp import TabularMLP
|
|
18
16
|
from ..errors import ModelError
|
|
19
17
|
|
|
20
18
|
|
|
21
|
-
def build_model(task: str, model_type: str, cfg: Dict[str, Any], meta: Dict[str, Any])
|
|
19
|
+
def build_model(task: str, model_type: str, cfg: Dict[str, Any], meta: Dict[str, Any]):
|
|
22
20
|
"""Build a fresh, randomly-initialized model.
|
|
23
21
|
|
|
24
22
|
`cfg` is the resolved training config (dict). `meta` carries
|