tensorless 0.3.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. {tensorless-0.3.0/tensorless.egg-info → tensorless-0.4.0}/PKG-INFO +4 -4
  2. {tensorless-0.3.0 → tensorless-0.4.0}/README.md +2 -2
  3. {tensorless-0.3.0 → tensorless-0.4.0}/docs/api_reference.md +1 -1
  4. {tensorless-0.3.0 → tensorless-0.4.0}/docs/architecture.md +1 -1
  5. tensorless-0.4.0/docs/installation.md +29 -0
  6. {tensorless-0.3.0 → tensorless-0.4.0}/docs/tl_format.md +4 -4
  7. {tensorless-0.3.0 → tensorless-0.4.0}/docs/troubleshooting.md +5 -7
  8. {tensorless-0.3.0 → tensorless-0.4.0}/pyproject.toml +2 -2
  9. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/auto/config.py +1 -1
  10. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/checkpoint/manager.py +5 -3
  11. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/data/tabular.py +9 -9
  12. tensorless-0.4.0/tensorless/devices/__init__.py +3 -0
  13. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/devices/device.py +9 -40
  14. tensorless-0.4.0/tensorless/engine.py +187 -0
  15. tensorless-0.4.0/tensorless/models/mlp.py +71 -0
  16. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/models/registry.py +1 -3
  17. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/models/transformer.py +91 -11
  18. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/runtime.py +12 -15
  19. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/serialization/tl_format.py +5 -3
  20. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/training/data_prep.py +47 -30
  21. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/training/trainer.py +93 -10
  22. {tensorless-0.3.0 → tensorless-0.4.0/tensorless.egg-info}/PKG-INFO +4 -4
  23. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless.egg-info/SOURCES.txt +1 -0
  24. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless.egg-info/requires.txt +1 -1
  25. {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_end_to_end.py +2 -7
  26. {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_serialization.py +9 -15
  27. tensorless-0.3.0/docs/installation.md +0 -43
  28. tensorless-0.3.0/tensorless/devices/__init__.py +0 -3
  29. tensorless-0.3.0/tensorless/models/mlp.py +0 -64
  30. {tensorless-0.3.0 → tensorless-0.4.0}/LICENSE +0 -0
  31. {tensorless-0.3.0 → tensorless-0.4.0}/MANIFEST.in +0 -0
  32. {tensorless-0.3.0 → tensorless-0.4.0}/docs/automatic_mode.md +0 -0
  33. {tensorless-0.3.0 → tensorless-0.4.0}/docs/checkpointing.md +0 -0
  34. {tensorless-0.3.0 → tensorless-0.4.0}/docs/cli.md +0 -0
  35. {tensorless-0.3.0 → tensorless-0.4.0}/docs/configuration.md +0 -0
  36. {tensorless-0.3.0 → tensorless-0.4.0}/docs/contributing.md +0 -0
  37. {tensorless-0.3.0 → tensorless-0.4.0}/docs/examples.md +0 -0
  38. {tensorless-0.3.0 → tensorless-0.4.0}/docs/inference.md +0 -0
  39. {tensorless-0.3.0 → tensorless-0.4.0}/docs/limitations.md +0 -0
  40. {tensorless-0.3.0 → tensorless-0.4.0}/docs/quickstart.md +0 -0
  41. {tensorless-0.3.0 → tensorless-0.4.0}/docs/roadmap.md +0 -0
  42. {tensorless-0.3.0 → tensorless-0.4.0}/docs/training.md +0 -0
  43. {tensorless-0.3.0 → tensorless-0.4.0}/docs/tutorial.md +0 -0
  44. {tensorless-0.3.0 → tensorless-0.4.0}/examples/tabular_classification_example.py +0 -0
  45. {tensorless-0.3.0 → tensorless-0.4.0}/examples/tabular_regression_example.py +0 -0
  46. {tensorless-0.3.0 → tensorless-0.4.0}/examples/text_classification_example.py +0 -0
  47. {tensorless-0.3.0 → tensorless-0.4.0}/examples/text_generation_example.py +0 -0
  48. {tensorless-0.3.0 → tensorless-0.4.0}/setup.cfg +0 -0
  49. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/__init__.py +0 -0
  50. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/_version.py +0 -0
  51. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/api.py +0 -0
  52. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/auto/__init__.py +0 -0
  53. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/auto/detector.py +0 -0
  54. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/checkpoint/__init__.py +0 -0
  55. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/cli/__init__.py +0 -0
  56. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/cli/main.py +0 -0
  57. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/config.py +0 -0
  58. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/data/__init__.py +0 -0
  59. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/data/english_grammar.txt +0 -0
  60. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/data/fingerprint.py +0 -0
  61. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/data/inspector.py +0 -0
  62. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/data/loader.py +0 -0
  63. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/errors.py +0 -0
  64. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/models/__init__.py +0 -0
  65. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/serialization/__init__.py +0 -0
  66. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/tokenization/__init__.py +0 -0
  67. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/tokenization/bpe_tokenizer.py +0 -0
  68. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/tokenization/char_tokenizer.py +0 -0
  69. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/training/__init__.py +0 -0
  70. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless/training/early_stopping.py +0 -0
  71. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless.egg-info/dependency_links.txt +0 -0
  72. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless.egg-info/entry_points.txt +0 -0
  73. {tensorless-0.3.0 → tensorless-0.4.0}/tensorless.egg-info/top_level.txt +0 -0
  74. {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_auto_detection.py +0 -0
  75. {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_checkpoint_resume.py +0 -0
  76. {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_cli.py +0 -0
  77. {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_data_loading.py +0 -0
  78. {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_fingerprint.py +0 -0
  79. {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_train_tabular.py +0 -0
  80. {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_train_text_classification.py +0 -0
  81. {tensorless-0.3.0 → tensorless-0.4.0}/tests/test_train_text_generation.py +0 -0
@@ -1,20 +1,20 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tensorless
3
- Version: 0.3.0
3
+ Version: 0.4.0
4
4
  Summary: ML with maximum automation and minimum setup.
5
5
  Author: Tensorless Contributors
6
6
  License: MIT
7
7
  Requires-Python: >=3.9
8
8
  Description-Content-Type: text/markdown
9
9
  License-File: LICENSE
10
- Requires-Dist: torch>=2.0
10
+ Requires-Dist: numpy>=1.24
11
11
  Provides-Extra: dev
12
12
  Requires-Dist: pytest>=7.0; extra == "dev"
13
13
  Dynamic: license-file
14
14
 
15
15
  # Tensorless
16
16
 
17
- Tensorless trains small PyTorch models with sensible defaults. It supports
17
+ Tensorless trains small native NumPy models with sensible defaults. It supports
18
18
  text generation, text classification, tabular classification, and regression.
19
19
 
20
20
  ## Install
@@ -37,7 +37,7 @@ tokenizer; use `tokenizer="char"` for a character-level model. Tensorless
37
37
  derives model size, batch size, epochs, validation, device, and BPE vocabulary
38
38
  size from the data, while every setting can be overridden.
39
39
 
40
- Long text is tokenized lazily and fed through PyTorch in fixed-size batches.
40
+ Long text is tokenized lazily and fed through the native engine in fixed-size batches.
41
41
  The automatic batch size uses a token budget; reduce `batch_size` if your
42
42
  available memory is limited.
43
43
 
@@ -1,6 +1,6 @@
1
1
  # Tensorless
2
2
 
3
- Tensorless trains small PyTorch models with sensible defaults. It supports
3
+ Tensorless trains small native NumPy models with sensible defaults. It supports
4
4
  text generation, text classification, tabular classification, and regression.
5
5
 
6
6
  ## Install
@@ -23,7 +23,7 @@ tokenizer; use `tokenizer="char"` for a character-level model. Tensorless
23
23
  derives model size, batch size, epochs, validation, device, and BPE vocabulary
24
24
  size from the data, while every setting can be overridden.
25
25
 
26
- Long text is tokenized lazily and fed through PyTorch in fixed-size batches.
26
+ Long text is tokenized lazily and fed through the native engine in fixed-size batches.
27
27
  The automatic batch size uses a token budget; reduce `batch_size` if your
28
28
  available memory is limited.
29
29
 
@@ -59,7 +59,7 @@ Returned by both `train()` and `load()`.
59
59
  | `.info()` | all | Dict summary: task, model type, versions, config, metrics, param count |
60
60
 
61
61
  Attributes: `.task`, `.model_type`, `.config`, `.meta`, `.metrics`,
62
- `.dataset_fingerprint`, `.model` (the underlying `torch.nn.Module`),
62
+ `.dataset_fingerprint`, `.model` (the underlying native Tensorless model),
63
63
  `.tokenizer` (`CharTokenizer` or `None`), `.preprocessor`
64
64
  (`TabularPreprocessor` or `None`).
65
65
 
@@ -37,7 +37,7 @@ tensorless/
37
37
  ├── serialization/
38
38
  │ └── tl_format.py save_tl/load_tl: the .tl file format
39
39
  ├── devices/
40
- │ └── device.py hardware auto-detection + torch.device resolution
40
+ │ └── device.py native CPU device resolution
41
41
  └── cli/
42
42
  └── main.py argparse-based CLI
43
43
  ```
@@ -0,0 +1,29 @@
1
+ # Installation
2
+
3
+ ## Requirements
4
+
5
+ - Python 3.9 or later
6
+ - NumPy 1.24 or later (installed automatically as a dependency)
7
+ - Tensorless currently runs its native vectorized engine on CPU; accelerator
8
+ backends are planned without changing the public API.
9
+
10
+ ## Verify your installation
11
+
12
+ ```bash
13
+ python -c "import tensorless as tl; print(tl.__version__)"
14
+ tensorless --help
15
+ ```
16
+
17
+ You should see a version string printed and the CLI's help text.
18
+
19
+ ## Runtime support
20
+
21
+ The native engine uses NumPy vectorized CPU kernels and automatically resolves
22
+ the device to CPU. You can still specify `device="cpu"` explicitly in
23
+ `tl.train(...)`; the device option remains forward-compatible with future
24
+ accelerator backends.
25
+
26
+ ## Troubleshooting installation
27
+
28
+ See [troubleshooting.md](troubleshooting.md#installation-issues) for
29
+ common installation problems.
@@ -6,9 +6,9 @@ original training data or code beyond having Tensorless installed.
6
6
 
7
7
  ## Structure
8
8
 
9
- Under the hood, `.tl` is a `torch.save`/`torch.load`-compatible pickle
10
- archive (see `tensorless/serialization/tl_format.py`) containing a
11
- dictionary:
9
+ Under the hood, `.tl` is a portable Python pickle file written by the native
10
+ Tensorless serializer (see `tensorless/serialization/tl_format.py`) containing
11
+ a dictionary:
12
12
 
13
13
  ```python
14
14
  {
@@ -18,7 +18,7 @@ dictionary:
18
18
  "model_type": "transformer", # or mlp
19
19
  "config": { ... }, # full resolved TrainConfig used
20
20
  "meta": { ... }, # vocab_size / n_classes / column info -- whatever the model needs to rebuild
21
- "model_state_dict": { ... }, # PyTorch model weights
21
+ "model_state_dict": { ... }, # native NumPy model weights
22
22
  "tokenizer_state": { ... } | None, # CharTokenizer vocab, for text tasks
23
23
  "preprocessor_state": { ... } | None, # TabularPreprocessor state, for tabular tasks
24
24
  "dataset_fingerprint": "...", # fingerprint of the training dataset
@@ -2,11 +2,9 @@
2
2
 
3
3
  ## Installation issues
4
4
 
5
- **`ModuleNotFoundError: No module named 'torch'`**
6
- Torch is a dependency and should install automatically with `pip
7
- install -e .`. If it didn't, install it manually following
8
- [pytorch.org/get-started](https://pytorch.org/get-started/locally/) for
9
- your platform, then reinstall Tensorless.
5
+ **`ModuleNotFoundError: No module named 'numpy'`**
6
+ NumPy is installed automatically with `pip install -e .`. If it is missing,
7
+ install it with `python -m pip install numpy`, then reinstall Tensorless.
10
8
 
11
9
  **`tensorless: command not found` after installing**
12
10
  Make sure the Python environment's `bin`/`Scripts` directory is on your
@@ -69,8 +67,8 @@ fingerprint across runs.
69
67
 
70
68
  **`CheckpointError: Checkpoint at '...' is corrupt or incompatible.`**
71
69
  The checkpoint file was likely truncated by an interruption during the
72
- (non-atomic part of the) write, or created by an incompatible PyTorch
73
- version. Delete the `.ckpt` directory and retrain — you'll lose progress
70
+ (non-atomic part of the) write, or created by an incompatible Tensorless
71
+ version. Delete the `.ckpt` directory and retrain — you'll lose progress
74
72
  on the interrupted run, but the checkpoint being unreadable means it
75
73
  can't be safely resumed regardless.
76
74
 
@@ -4,14 +4,14 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "tensorless"
7
- version = "0.3.0"
7
+ version = "0.4.0"
8
8
  description = "ML with maximum automation and minimum setup."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.9"
11
11
  license = { text = "MIT" }
12
12
  authors = [{ name = "Tensorless Contributors" }]
13
13
  dependencies = [
14
- "torch>=2.0",
14
+ "numpy>=1.24",
15
15
  ]
16
16
 
17
17
  [project.optional-dependencies]
@@ -115,7 +115,7 @@ def resolve_config(ds: Dataset, user: TrainConfig) -> ResolvedConfig:
115
115
  tokenizer=tokenizer,
116
116
  bpe_vocab_size=user.bpe_vocab_size if user.bpe_vocab_size is not None else _auto_vocab_size(ds),
117
117
  optimizer=user.optimizer or "adamw",
118
- learning_rate=user.learning_rate or (3e-4 if model_type == "transformer" else 1e-3),
118
+ learning_rate=user.learning_rate or (3e-3 if task == "text-classification" else (3e-4 if model_type == "transformer" else 1e-3)),
119
119
  weight_decay=user.weight_decay if user.weight_decay is not None else 0.01,
120
120
  batch_size=user.batch_size if user.batch_size is not None else _auto_batch_size(n, max_seq_len),
121
121
  epochs=user.epochs or _auto_epochs(n),
@@ -18,7 +18,7 @@ import shutil
18
18
  import tempfile
19
19
  from typing import Any, Dict, Optional
20
20
 
21
- import torch
21
+ import pickle
22
22
 
23
23
  from ..errors import CheckpointError
24
24
 
@@ -44,7 +44,8 @@ class CheckpointManager:
44
44
  fd, tmp_path = tempfile.mkstemp(dir=self.checkpoint_dir, suffix=".tmp")
45
45
  os.close(fd)
46
46
  try:
47
- torch.save(state, tmp_path)
47
+ with open(tmp_path, "wb") as handle:
48
+ pickle.dump(state, handle, protocol=pickle.HIGHEST_PROTOCOL)
48
49
  shutil.move(tmp_path, self.path)
49
50
  except Exception as e:
50
51
  if os.path.exists(tmp_path):
@@ -55,7 +56,8 @@ class CheckpointManager:
55
56
  if not self.exists():
56
57
  raise CheckpointError(f"No checkpoint found at '{self.path}'.")
57
58
  try:
58
- return torch.load(self.path, map_location=map_location, weights_only=False)
59
+ with open(self.path, "rb") as handle:
60
+ return pickle.load(handle)
59
61
  except Exception as e:
60
62
  raise CheckpointError(
61
63
  f"Checkpoint at '{self.path}' is corrupt or incompatible: {e}"
@@ -17,7 +17,7 @@ from datetime import datetime
17
17
  from dataclasses import dataclass, field
18
18
  from typing import Any, Dict, List, Optional, Tuple
19
19
 
20
- import torch
20
+ import numpy as np
21
21
 
22
22
  _MISSING = "<missing>"
23
23
  _UNK = "<unk>"
@@ -132,13 +132,13 @@ class TabularPreprocessor:
132
132
 
133
133
  def transform(
134
134
  self, records: List[Dict[str, Any]], with_target: bool = True
135
- ) -> Dict[str, torch.Tensor]:
135
+ ) -> Dict[str, np.ndarray]:
136
136
  n = len(records)
137
137
  num_cols = self.numeric_columns
138
138
  cat_cols = self.categorical_columns
139
139
 
140
- numeric = torch.zeros(n, max(len(num_cols), 1), dtype=torch.float32)
141
- categorical = torch.zeros(n, max(len(cat_cols), 1), dtype=torch.long)
140
+ numeric = np.zeros((n, max(len(num_cols), 1)), dtype=np.float32)
141
+ categorical = np.zeros((n, max(len(cat_cols), 1)), dtype=np.int64)
142
142
 
143
143
  for i, r in enumerate(records):
144
144
  for j, col in enumerate(num_cols):
@@ -158,14 +158,14 @@ class TabularPreprocessor:
158
158
 
159
159
  if with_target and self.target_column is not None:
160
160
  if self.task == "regression":
161
- target = torch.zeros(n, dtype=torch.float32)
161
+ target = np.zeros(n, dtype=np.float32)
162
162
  for i, r in enumerate(records):
163
163
  v = _try_float(r.get(self.target_column))
164
164
  v = self.target_mean if v is None else v
165
165
  target[i] = (v - self.target_mean) / self.target_std
166
166
  out["target"] = target
167
167
  else:
168
- target = torch.zeros(n, dtype=torch.long)
168
+ target = np.zeros(n, dtype=np.int64)
169
169
  for i, r in enumerate(records):
170
170
  raw = str(r.get(self.target_column))
171
171
  idx = self.classes.index(raw) if raw in self.classes else 0
@@ -174,10 +174,10 @@ class TabularPreprocessor:
174
174
 
175
175
  return out
176
176
 
177
- def inverse_target(self, values: torch.Tensor) -> List[Any]:
177
+ def inverse_target(self, values: np.ndarray) -> List[Any]:
178
178
  if self.task == "regression":
179
- return [(v.item() * self.target_std + self.target_mean) for v in values]
180
- return [self.classes[int(v.item())] for v in values]
179
+ return [float(v * self.target_std + self.target_mean) for v in values]
180
+ return [self.classes[int(v)] for v in values]
181
181
 
182
182
  def categorical_vocab_sizes(self) -> List[int]:
183
183
  return [len(self.column_stats[c].vocab) for c in self.categorical_columns]
@@ -0,0 +1,3 @@
1
+ from .device import auto_select_device, get_device
2
+
3
+ __all__ = ["auto_select_device", "get_device"]
@@ -11,37 +11,21 @@ from __future__ import annotations
11
11
 
12
12
  from typing import Optional, Tuple
13
13
 
14
- import torch
15
-
16
14
 
17
15
  def _tpu_available() -> bool:
18
- try:
19
- import torch_xla.core.xla_model as xm # noqa: F401
20
-
21
- return True
22
- except Exception:
23
- return False
16
+ return False
24
17
 
25
18
 
26
19
  def _cuda_available() -> bool:
27
- try:
28
- return torch.cuda.is_available() and torch.cuda.device_count() > 0
29
- except Exception:
30
- return False
20
+ return False
31
21
 
32
22
 
33
23
  def _mps_available() -> bool:
34
- try:
35
- return torch.backends.mps.is_available()
36
- except Exception:
37
- return False
24
+ return False
38
25
 
39
26
 
40
27
  def _cuda_supports_bf16() -> bool:
41
- try:
42
- return torch.cuda.is_bf16_supported()
43
- except Exception:
44
- return False
28
+ return False
45
29
 
46
30
 
47
31
  def auto_select_device(user_device: Optional[str], user_precision: Optional[str]) -> Tuple[str, str]:
@@ -85,23 +69,8 @@ def auto_select_device(user_device: Optional[str], user_precision: Optional[str]
85
69
  return device, precision
86
70
 
87
71
 
88
- def get_torch_device(device: str) -> torch.device:
89
- """Convert our string device name into a torch.device, with a
90
- runtime fallback to CPU if the requested backend is unavailable.
91
- """
92
- try:
93
- if device == "tpu":
94
- import torch_xla.core.xla_model as xm
95
-
96
- return xm.xla_device()
97
- if device == "cuda":
98
- if not _cuda_available():
99
- return torch.device("cpu")
100
- return torch.device("cuda")
101
- if device == "mps":
102
- if not _mps_available():
103
- return torch.device("cpu")
104
- return torch.device("mps")
105
- return torch.device("cpu")
106
- except Exception:
107
- return torch.device("cpu")
72
+ def get_device(device: str) -> str:
73
+ """Return the native engine device name (currently CPU only)."""
74
+ return "cpu"
75
+
76
+
@@ -0,0 +1,187 @@
1
+ """Small NumPy training engine used by Tensorless models."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Dict, Iterable, Iterator
6
+ import numpy as np
7
+
8
+
9
+ class Parameter:
10
+ def __init__(self, data):
11
+ self.data = np.asarray(data, dtype=np.float32)
12
+ self.grad = np.zeros_like(self.data)
13
+
14
+ def numel(self) -> int:
15
+ return int(self.data.size)
16
+
17
+
18
+ class Module:
19
+ def parameters(self) -> Iterator[Parameter]:
20
+ for value in self.__dict__.values():
21
+ if isinstance(value, Parameter):
22
+ yield value
23
+ elif isinstance(value, Module):
24
+ yield from value.parameters()
25
+ elif isinstance(value, (list, tuple)):
26
+ for item in value:
27
+ if isinstance(item, Parameter):
28
+ yield item
29
+ elif isinstance(item, Module):
30
+ yield from item.parameters()
31
+
32
+ def named_parameters(self, prefix: str = ""):
33
+ for name, value in self.__dict__.items():
34
+ full = f"{prefix}.{name}" if prefix else name
35
+ if isinstance(value, Parameter):
36
+ yield full, value
37
+ elif isinstance(value, Module):
38
+ yield from value.named_parameters(full)
39
+ elif isinstance(value, (list, tuple)):
40
+ for i, item in enumerate(value):
41
+ if isinstance(item, Parameter):
42
+ yield f"{full}.{i}", item
43
+ elif isinstance(item, Module):
44
+ yield from item.named_parameters(f"{full}.{i}")
45
+
46
+ def state_dict(self) -> Dict[str, np.ndarray]:
47
+ return {name: parameter.data.copy() for name, parameter in self.named_parameters()}
48
+
49
+ def load_state_dict(self, state: Dict[str, np.ndarray]) -> None:
50
+ current = dict(self.named_parameters())
51
+ missing = [name for name in current if name not in state]
52
+ if missing:
53
+ raise ValueError(f"Missing model parameters: {missing}")
54
+ for name, parameter in current.items():
55
+ value = np.asarray(state[name], dtype=np.float32)
56
+ if value.shape != parameter.data.shape:
57
+ raise ValueError(f"Shape mismatch for {name}: {value.shape} != {parameter.data.shape}")
58
+ parameter.data[...] = value
59
+
60
+ def train(self, mode: bool = True):
61
+ self.training = mode
62
+ return self
63
+
64
+ def eval(self):
65
+ return self.train(False)
66
+
67
+
68
+ def softmax_cross_entropy(logits: np.ndarray, targets: np.ndarray, ignore_index=None):
69
+ flat = logits.reshape(-1, logits.shape[-1]).astype(np.float64)
70
+ target = targets.reshape(-1).astype(np.int64)
71
+ valid = np.ones(len(target), dtype=bool) if ignore_index is None else target != ignore_index
72
+ if not np.any(valid):
73
+ return 0.0, np.zeros_like(logits)
74
+ shifted = flat - flat.max(axis=1, keepdims=True)
75
+ probs = np.exp(shifted)
76
+ probs /= probs.sum(axis=1, keepdims=True)
77
+ rows = np.arange(len(target))[valid]
78
+ loss = -np.log(np.maximum(probs[rows, target[valid]], 1e-12)).mean()
79
+ grad = np.zeros_like(flat, dtype=np.float32)
80
+ grad[valid] = probs[valid].astype(np.float32)
81
+ grad[rows, target[valid]] -= 1.0
82
+ grad[valid] /= valid.sum()
83
+ return float(loss), grad.reshape(logits.shape)
84
+
85
+
86
+ class Optimizer:
87
+ def __init__(self, parameters: Iterable[Parameter], lr: float, weight_decay: float = 0.0):
88
+ self.parameters_list = list(parameters)
89
+ self.lr = lr
90
+ self.weight_decay = weight_decay
91
+ self.step_count = 0
92
+
93
+ def zero_grad(self):
94
+ for parameter in self.parameters_list:
95
+ parameter.grad.fill(0.0)
96
+
97
+ def state_dict(self):
98
+ return {"step": self.step_count, "lr": self.lr, "weight_decay": self.weight_decay}
99
+
100
+ def load_state_dict(self, state):
101
+ self.step_count = int(state.get("step", 0))
102
+ self.lr = float(state.get("lr", self.lr))
103
+ self.weight_decay = float(state.get("weight_decay", self.weight_decay))
104
+
105
+
106
+ class SGD(Optimizer):
107
+ def __init__(self, parameters, lr, weight_decay=0.0, momentum=0.9):
108
+ super().__init__(parameters, lr, weight_decay)
109
+ self.momentum = momentum
110
+ self.velocity = [np.zeros_like(p.data) for p in self.parameters_list]
111
+
112
+ def step(self):
113
+ self.step_count += 1
114
+ for p, velocity in zip(self.parameters_list, self.velocity):
115
+ velocity *= self.momentum
116
+ velocity += p.grad + self.weight_decay * p.data
117
+ p.data -= self.lr * velocity
118
+
119
+ def state_dict(self):
120
+ state = super().state_dict()
121
+ state["velocity"] = [v.copy() for v in self.velocity]
122
+ return state
123
+
124
+ def load_state_dict(self, state):
125
+ super().load_state_dict(state)
126
+ for target, source in zip(self.velocity, state.get("velocity", [])):
127
+ target[...] = source
128
+
129
+
130
+ class Adam(Optimizer):
131
+ def __init__(self, parameters, lr, weight_decay=0.0, decoupled=False):
132
+ super().__init__(parameters, lr, weight_decay)
133
+ self.decoupled = decoupled
134
+ self.m = [np.zeros_like(p.data) for p in self.parameters_list]
135
+ self.v = [np.zeros_like(p.data) for p in self.parameters_list]
136
+
137
+ def step(self):
138
+ self.step_count += 1
139
+ for p, m, v in zip(self.parameters_list, self.m, self.v):
140
+ if self.decoupled:
141
+ p.data *= 1.0 - self.lr * self.weight_decay
142
+ gradient = p.grad
143
+ else:
144
+ gradient = p.grad + self.weight_decay * p.data
145
+ m *= 0.9
146
+ m += 0.1 * gradient
147
+ v *= 0.999
148
+ v += 0.001 * gradient * gradient
149
+ m_hat = m / (1.0 - 0.9 ** self.step_count)
150
+ v_hat = v / (1.0 - 0.999 ** self.step_count)
151
+ p.data -= self.lr * m_hat / (np.sqrt(v_hat) + 1e-8)
152
+
153
+ def state_dict(self):
154
+ state = super().state_dict()
155
+ state.update({"m": [x.copy() for x in self.m], "v": [x.copy() for x in self.v]})
156
+ return state
157
+
158
+ def load_state_dict(self, state):
159
+ super().load_state_dict(state)
160
+ for target, source in zip(self.m, state.get("m", [])):
161
+ target[...] = source
162
+ for target, source in zip(self.v, state.get("v", [])):
163
+ target[...] = source
164
+
165
+
166
+ class LambdaScheduler:
167
+ def __init__(self, optimizer, warmup_steps, total_steps):
168
+ self.optimizer = optimizer
169
+ self.warmup_steps = warmup_steps
170
+ self.total_steps = total_steps
171
+ self.step_count = 0
172
+ self.base_lr = optimizer.lr
173
+
174
+ def step(self):
175
+ self.step_count += 1
176
+ step = self.step_count - 1
177
+ if self.warmup_steps and step < self.warmup_steps:
178
+ factor = (step + 1) / self.warmup_steps
179
+ else:
180
+ factor = max(0.1, 1.0 - (step - self.warmup_steps) / max(1, self.total_steps - self.warmup_steps))
181
+ self.optimizer.lr = self.base_lr * factor
182
+
183
+ def state_dict(self):
184
+ return {"step_count": self.step_count}
185
+
186
+ def load_state_dict(self, state):
187
+ self.step_count = int(state.get("step_count", 0))
@@ -0,0 +1,71 @@
1
+ """NumPy MLP for tabular classification and regression."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import List
6
+ import numpy as np
7
+
8
+ from ..engine import Module, Parameter
9
+
10
+
11
+ def _embedding_dim(vocab_size: int) -> int:
12
+ return max(2, min(32, round(1.6 * (vocab_size ** 0.56))))
13
+
14
+
15
+ class TabularMLP(Module):
16
+ def __init__(self, n_numeric, categorical_vocab_sizes: List[int], d_model, layers, dropout, task, n_classes=0):
17
+ self.training = True
18
+ self.task = task
19
+ self.n_numeric = n_numeric
20
+ self.categorical_vocab_sizes = categorical_vocab_sizes
21
+ rng = np.random.default_rng()
22
+ self.embeddings = [Parameter(rng.normal(0, .02, (v, _embedding_dim(v))).astype(np.float32)) for v in categorical_vocab_sizes]
23
+ in_dim = n_numeric + sum(_embedding_dim(v) for v in categorical_vocab_sizes)
24
+ self.weights, self.biases = [], []
25
+ for i in range(layers):
26
+ fan_in = in_dim if i == 0 else d_model
27
+ self.weights.append(Parameter(rng.normal(0, .02, (fan_in, d_model)).astype(np.float32)))
28
+ self.biases.append(Parameter(np.zeros(d_model, dtype=np.float32)))
29
+ out_dim = n_classes if task == "classification" else 1
30
+ self.head_weight = Parameter(rng.normal(0, .02, (d_model if layers else in_dim, out_dim)).astype(np.float32))
31
+ self.head_bias = Parameter(np.zeros(out_dim, dtype=np.float32))
32
+
33
+ def forward(self, numeric, categorical, cache=False):
34
+ parts = [numeric.astype(np.float32)] if self.n_numeric else []
35
+ for i, embedding in enumerate(self.embeddings):
36
+ parts.append(embedding.data[categorical[:, i]])
37
+ x = np.concatenate(parts, axis=1) if parts else numeric.astype(np.float32)
38
+ activations = [x]
39
+ for weight, bias in zip(self.weights, self.biases):
40
+ x = np.maximum(0, x @ weight.data + bias.data)
41
+ activations.append(x)
42
+ out = x @ self.head_weight.data + self.head_bias.data
43
+ if self.task == "regression": out = out[:, 0]
44
+ return (out, (activations, categorical)) if cache else out
45
+
46
+ def loss_and_backward(self, numeric, categorical, target):
47
+ logits, (activations, categorical) = self.forward(numeric, categorical, cache=True)
48
+ if self.task == "regression":
49
+ error = logits - target
50
+ loss = float(np.mean(error * error))
51
+ grad = (2.0 / len(target)) * error[:, None]
52
+ else:
53
+ shifted = logits - logits.max(1, keepdims=True)
54
+ probs = np.exp(shifted); probs /= probs.sum(1, keepdims=True)
55
+ loss = float(-np.log(np.maximum(probs[np.arange(len(target)), target], 1e-12)).mean())
56
+ grad = probs; grad[np.arange(len(target)), target] -= 1; grad /= len(target)
57
+ self.head_weight.grad[...] = activations[-1].T @ grad
58
+ self.head_bias.grad[...] = grad.sum(0)
59
+ dx = grad @ self.head_weight.data.T
60
+ for i in range(len(self.weights) - 1, -1, -1):
61
+ dx = dx * (activations[i + 1] > 0)
62
+ self.weights[i].grad[...] = activations[i].T @ dx
63
+ self.biases[i].grad[...] = dx.sum(0)
64
+ dx = dx @ self.weights[i].data.T
65
+ offset = self.n_numeric
66
+ for i, embedding in enumerate(self.embeddings):
67
+ width = embedding.data.shape[1]
68
+ embedding.grad.fill(0)
69
+ np.add.at(embedding.grad, categorical[:, i], dx[:, offset:offset + width])
70
+ offset += width
71
+ return loss
@@ -11,14 +11,12 @@ from __future__ import annotations
11
11
 
12
12
  from typing import Any, Dict
13
13
 
14
- import torch.nn as nn
15
-
16
14
  from .transformer import TinyTransformer
17
15
  from .mlp import TabularMLP
18
16
  from ..errors import ModelError
19
17
 
20
18
 
21
- def build_model(task: str, model_type: str, cfg: Dict[str, Any], meta: Dict[str, Any]) -> nn.Module:
19
+ def build_model(task: str, model_type: str, cfg: Dict[str, Any], meta: Dict[str, Any]):
22
20
  """Build a fresh, randomly-initialized model.
23
21
 
24
22
  `cfg` is the resolved training config (dict). `meta` carries