libthx 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- libthx-0.1.1/LICENSE +7 -0
- libthx-0.1.1/PKG-INFO +77 -0
- libthx-0.1.1/README.md +121 -0
- libthx-0.1.1/libthx.egg-info/PKG-INFO +77 -0
- libthx-0.1.1/libthx.egg-info/SOURCES.txt +17 -0
- libthx-0.1.1/libthx.egg-info/dependency_links.txt +1 -0
- libthx-0.1.1/libthx.egg-info/entry_points.txt +2 -0
- libthx-0.1.1/libthx.egg-info/requires.txt +80 -0
- libthx-0.1.1/libthx.egg-info/top_level.txt +1 -0
- libthx-0.1.1/pyproject.toml +239 -0
- libthx-0.1.1/setup.cfg +4 -0
- libthx-0.1.1/theseus/__init__.py +1 -0
- libthx-0.1.1/theseus/cli.py +1618 -0
- libthx-0.1.1/theseus/config.py +296 -0
- libthx-0.1.1/theseus/job.py +334 -0
- libthx-0.1.1/theseus/mock.py +200 -0
- libthx-0.1.1/theseus/plot.py +268 -0
- libthx-0.1.1/theseus/quick.py +233 -0
- libthx-0.1.1/theseus/registry.py +80 -0
libthx-0.1.1/LICENSE
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
Copyright (c) 2026 Houjun Liu
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
4
|
+
|
|
5
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
6
|
+
|
|
7
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
libthx-0.1.1/PKG-INFO
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: libthx
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Requires-Python: >=3.11
|
|
5
|
+
License-File: LICENSE
|
|
6
|
+
Requires-Dist: click==8.2.1
|
|
7
|
+
Requires-Dist: flax>=0.12.2
|
|
8
|
+
Requires-Dist: jsonlines>=4.0.0
|
|
9
|
+
Requires-Dist: loguru>=0.7.3
|
|
10
|
+
Requires-Dist: numpy>=2.4.1
|
|
11
|
+
Requires-Dist: omegaconf>=2.3.0
|
|
12
|
+
Requires-Dist: orbax>=0.1.9
|
|
13
|
+
Requires-Dist: pydantic>=2.12.5
|
|
14
|
+
Requires-Dist: python-dotenv>=1.2.1
|
|
15
|
+
Requires-Dist: rich>=14.2.0
|
|
16
|
+
Requires-Dist: seaborn>=0.13.2
|
|
17
|
+
Requires-Dist: tiktoken>=0.12.0
|
|
18
|
+
Requires-Dist: torchax>=0.0.11
|
|
19
|
+
Requires-Dist: wandb>=0.24.1
|
|
20
|
+
Requires-Dist: datasets>=4.5.0
|
|
21
|
+
Provides-Extra: fever
|
|
22
|
+
Requires-Dist: wikipedia>=1.4.0; extra == "fever"
|
|
23
|
+
Provides-Extra: huggingface
|
|
24
|
+
Requires-Dist: tokenizers>=0.22.2; extra == "huggingface"
|
|
25
|
+
Requires-Dist: transformers>=5.1.0; extra == "huggingface"
|
|
26
|
+
Provides-Extra: cuda13
|
|
27
|
+
Requires-Dist: jax[cuda13]>=0.4.23; extra == "cuda13"
|
|
28
|
+
Requires-Dist: torch>=2.9.1; extra == "cuda13"
|
|
29
|
+
Requires-Dist: torchax>=0.0.11; extra == "cuda13"
|
|
30
|
+
Requires-Dist: tokenizers>=0.22.2; extra == "cuda13"
|
|
31
|
+
Requires-Dist: transformers>=5.1.0; extra == "cuda13"
|
|
32
|
+
Provides-Extra: cuda12
|
|
33
|
+
Requires-Dist: jax[cuda12]>=0.4.23; extra == "cuda12"
|
|
34
|
+
Requires-Dist: torch>=2.9.1; extra == "cuda12"
|
|
35
|
+
Requires-Dist: torchax>=0.0.11; extra == "cuda12"
|
|
36
|
+
Requires-Dist: tokenizers>=0.22.2; extra == "cuda12"
|
|
37
|
+
Requires-Dist: transformers>=5.1.0; extra == "cuda12"
|
|
38
|
+
Provides-Extra: tpu
|
|
39
|
+
Requires-Dist: jax[tpu]>=0.4.23; extra == "tpu"
|
|
40
|
+
Requires-Dist: torch>=2.9.1; extra == "tpu"
|
|
41
|
+
Requires-Dist: torchax>=0.0.11; extra == "tpu"
|
|
42
|
+
Requires-Dist: tokenizers>=0.22.2; extra == "tpu"
|
|
43
|
+
Requires-Dist: transformers>=5.1.0; extra == "tpu"
|
|
44
|
+
Provides-Extra: cpu
|
|
45
|
+
Requires-Dist: jax>=0.4.23; extra == "cpu"
|
|
46
|
+
Requires-Dist: torch>=2.9.1; extra == "cpu"
|
|
47
|
+
Requires-Dist: torchax>=0.0.11; extra == "cpu"
|
|
48
|
+
Requires-Dist: tokenizers>=0.22.2; extra == "cpu"
|
|
49
|
+
Requires-Dist: transformers>=5.1.0; extra == "cpu"
|
|
50
|
+
Provides-Extra: web
|
|
51
|
+
Requires-Dist: fastapi>=0.100.0; extra == "web"
|
|
52
|
+
Requires-Dist: uvicorn[standard]>=0.23.0; extra == "web"
|
|
53
|
+
Requires-Dist: jinja2>=3.1.0; extra == "web"
|
|
54
|
+
Requires-Dist: aiofiles>=23.0.0; extra == "web"
|
|
55
|
+
Requires-Dist: sse-starlette>=1.6.0; extra == "web"
|
|
56
|
+
Requires-Dist: pyyaml>=6.0.0; extra == "web"
|
|
57
|
+
Requires-Dist: python-multipart>=0.0.22; extra == "web"
|
|
58
|
+
Requires-Dist: bcrypt>=5.0.0; extra == "web"
|
|
59
|
+
Requires-Dist: itsdangerous>=2.2.0; extra == "web"
|
|
60
|
+
Requires-Dist: watchdog>=4.0.0; extra == "web"
|
|
61
|
+
Provides-Extra: dev
|
|
62
|
+
Requires-Dist: coverage>=7.9.1; extra == "dev"
|
|
63
|
+
Requires-Dist: coveralls>=4.0.1; extra == "dev"
|
|
64
|
+
Requires-Dist: pytest>=8.4.1; extra == "dev"
|
|
65
|
+
Requires-Dist: pytest-cov>=6.2.1; extra == "dev"
|
|
66
|
+
Requires-Dist: ruff>=0.12.1; extra == "dev"
|
|
67
|
+
Requires-Dist: pre-commit>=4.2.0; extra == "dev"
|
|
68
|
+
Requires-Dist: mypy>=1.16.1; extra == "dev"
|
|
69
|
+
Provides-Extra: docs
|
|
70
|
+
Requires-Dist: mkdocs>=1.6.1; extra == "docs"
|
|
71
|
+
Requires-Dist: mkdocs-material>=9.6.20; extra == "docs"
|
|
72
|
+
Requires-Dist: mkdocstrings[python]>=0.30.0; extra == "docs"
|
|
73
|
+
Requires-Dist: pymdown-extensions>=10.15; extra == "docs"
|
|
74
|
+
Requires-Dist: mkdocs-gen-files>=0.5.0; extra == "docs"
|
|
75
|
+
Requires-Dist: mkdocs-literate-nav>=0.6.0; extra == "docs"
|
|
76
|
+
Requires-Dist: mkdocs-section-index>=0.3.0; extra == "docs"
|
|
77
|
+
Dynamic: license-file
|
libthx-0.1.1/README.md
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# theseus
|
|
2
|
+
Have you ever wanted to train a language model from scratch but hate writing boilerplate? Previously the solution to this is to work at a frontier lab with Research Engineers:tm:.
|
|
3
|
+
|
|
4
|
+
Now the solution is to make Jack:tm: (and also a cast of frontier coding models) do your research engineering. It will probably break a lot but what the heck at least I tried.
|
|
5
|
+
|
|
6
|
+
## Download
|
|
7
|
+
|
|
8
|
+
It depends on who gave you computors to make warm:
|
|
9
|
+
|
|
10
|
+
- cuda13: `uv sync --group all --group cuda13`
|
|
11
|
+
- cuda12: `uv sync --group all --group cuda12`
|
|
12
|
+
- you love Google: `uv sync --group all --group tpu`
|
|
13
|
+
- you bought your own computors: `uv sync --group all --group cpu`
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
## Quick Start
|
|
17
|
+
Use the CLI.
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
# List available jobs
|
|
21
|
+
theseus jobs
|
|
22
|
+
|
|
23
|
+
# Generate a config for data tokenization
|
|
24
|
+
theseus configure data/tokenize_variable_dataset tokenize.yaml \
|
|
25
|
+
data.name=fineweb data.max_samples=1000000
|
|
26
|
+
|
|
27
|
+
# Run the tokenization locally
|
|
28
|
+
theseus run tokenize-fineweb tokenize.yaml ./output
|
|
29
|
+
|
|
30
|
+
# Generate a config for pretraining
|
|
31
|
+
theseus configure gpt/train/pretrain train.yaml \
|
|
32
|
+
--chip h100 -n 8
|
|
33
|
+
|
|
34
|
+
# Run training locally
|
|
35
|
+
theseus run my-gpt-run train.yaml ./output
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
### Quick Start, but You Have Infra
|
|
39
|
+
|
|
40
|
+
Set up `~/.theseus.yaml` (see `examples/dispatch.yaml`), then submit jobs to remote clusters:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
theseus submit my-run train.yaml --chip h100 -n 8
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Quickish Start
|
|
47
|
+
|
|
48
|
+
For programmatic configuration and rapid prototyping:
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
from theseus.quick import quick
|
|
52
|
+
from theseus.registry import JOBS
|
|
53
|
+
|
|
54
|
+
with quick("gpt/train/pretrain", "/path/to/output", "my-run") as j:
|
|
55
|
+
j.config.training.per_device_batch_size = 16
|
|
56
|
+
j.config.logging.checkpoint_interval = 4096
|
|
57
|
+
j() # run locally
|
|
58
|
+
|
|
59
|
+
# Or save config for later submission:
|
|
60
|
+
with quick("gpt/train/pretrain", "/path/to/output", "my-run") as j:
|
|
61
|
+
j.config.training.per_device_batch_size = 16
|
|
62
|
+
j.save("config.yaml", chip="h100", n_chips=8)
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Not Quick Start at All
|
|
66
|
+
|
|
67
|
+
When you (or Claude) manage to find some time to chill you can actually extend this package. The package is organized based around `theseus.job.BasicJob`s. They can be extended with checkpointing and recovery tools.
|
|
68
|
+
|
|
69
|
+
The main entrypoint to start hacking:
|
|
70
|
+
|
|
71
|
+
1. take a look at how to compose a model together in `theseus.model.models.base`
|
|
72
|
+
2. bodge together anything you want to change and make a new model in the models folder (be sure to add it to `theseus.model.models.__init__`)
|
|
73
|
+
3. write an experiment, which is a `RestoreableJob`. A very basic one can just inherit the normal trainer, and then that's about it. see `theseus.experiments.gpt` to get started (be sure to add it to `theseus.experiments.__init__`)
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
# theseus/experiments/my_model.py
|
|
77
|
+
from theseus.training.base import BaseTrainer, BaseTrainerConfig
|
|
78
|
+
from theseus.model.models import MyModel
|
|
79
|
+
|
|
80
|
+
class PretrainMyModel(BaseTrainer[BaseTrainerConfig, MyModel]):
|
|
81
|
+
MODEL = MyModel
|
|
82
|
+
CONFIG = BaseTrainerConfig
|
|
83
|
+
|
|
84
|
+
@classmethod
|
|
85
|
+
def schedule(cls):
|
|
86
|
+
return "wsd"
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## JuiceFS Integration
|
|
90
|
+
When you are on many remote computors but bursty you may go "aw schucks I need to copy like 50TB of pretraining data around that's so lame!"
|
|
91
|
+
|
|
92
|
+
Don't worry, we gotchu. If you use the `submit` API, we have a way to ship your root directory around by using a thing called [JuiceFS](https://juicefs.com/en/), which is a distributed filesystem.
|
|
93
|
+
|
|
94
|
+
In your `~/.theseus.yaml`, add the `mount` field to your cluster config:
|
|
95
|
+
|
|
96
|
+
```yaml
|
|
97
|
+
clusters:
|
|
98
|
+
hpc:
|
|
99
|
+
root: /mnt/juicefs/theseus
|
|
100
|
+
work: /scratch/theseus
|
|
101
|
+
mount: redis://:password@redis.example.com:6379/0
|
|
102
|
+
cache_size: 100G
|
|
103
|
+
cache_dir: /scratch/juicefs-cache
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
## (an incomplete list of) Features
|
|
107
|
+
|
|
108
|
+
- **CLI & Programmatic API**: Configure and run jobs via `theseus` CLI or the `quick()` Python API
|
|
109
|
+
- **Remote Dispatch**: Submit jobs to SLURM clusters or plain SSH hosts via `~/.theseus.yaml`
|
|
110
|
+
- **Checkpointing & Recovery**: Jobs are `RestoreableJob`s with built-in checkpoint/restore support
|
|
111
|
+
- **Data Pipelines**: Tokenize datasets (blockwise or streaming) with `data/tokenize_*` jobs
|
|
112
|
+
- **JuiceFS Integration**: Distributed filesystem support for sharing data across clusters
|
|
113
|
+
- **Multi-backend**: CUDA 11/12/13, TPU, and CPU via `uv sync --group`
|
|
114
|
+
- **Extensible**: Add models in `theseus.model.models`, experiments in `theseus.experiments`, and datasets in `theseus.data.datasets`
|
|
115
|
+
- **Dataclass Configs**: Type-safe configuration via dataclasses with OmegaConf, easy configuration with `theseus.config.field` dataclass extension, and Hydra-style cheeky cli overrides (`model.hidden_size=1024`)
|
|
116
|
+
|
|
117
|
+
---
|
|
118
|
+
|
|
119
|
+
<p align="center">
|
|
120
|
+
<img src="https://www.jemoka.com/images/Logo_Transparent.png" width="32">
|
|
121
|
+
</p>
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: libthx
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Requires-Python: >=3.11
|
|
5
|
+
License-File: LICENSE
|
|
6
|
+
Requires-Dist: click==8.2.1
|
|
7
|
+
Requires-Dist: flax>=0.12.2
|
|
8
|
+
Requires-Dist: jsonlines>=4.0.0
|
|
9
|
+
Requires-Dist: loguru>=0.7.3
|
|
10
|
+
Requires-Dist: numpy>=2.4.1
|
|
11
|
+
Requires-Dist: omegaconf>=2.3.0
|
|
12
|
+
Requires-Dist: orbax>=0.1.9
|
|
13
|
+
Requires-Dist: pydantic>=2.12.5
|
|
14
|
+
Requires-Dist: python-dotenv>=1.2.1
|
|
15
|
+
Requires-Dist: rich>=14.2.0
|
|
16
|
+
Requires-Dist: seaborn>=0.13.2
|
|
17
|
+
Requires-Dist: tiktoken>=0.12.0
|
|
18
|
+
Requires-Dist: torchax>=0.0.11
|
|
19
|
+
Requires-Dist: wandb>=0.24.1
|
|
20
|
+
Requires-Dist: datasets>=4.5.0
|
|
21
|
+
Provides-Extra: fever
|
|
22
|
+
Requires-Dist: wikipedia>=1.4.0; extra == "fever"
|
|
23
|
+
Provides-Extra: huggingface
|
|
24
|
+
Requires-Dist: tokenizers>=0.22.2; extra == "huggingface"
|
|
25
|
+
Requires-Dist: transformers>=5.1.0; extra == "huggingface"
|
|
26
|
+
Provides-Extra: cuda13
|
|
27
|
+
Requires-Dist: jax[cuda13]>=0.4.23; extra == "cuda13"
|
|
28
|
+
Requires-Dist: torch>=2.9.1; extra == "cuda13"
|
|
29
|
+
Requires-Dist: torchax>=0.0.11; extra == "cuda13"
|
|
30
|
+
Requires-Dist: tokenizers>=0.22.2; extra == "cuda13"
|
|
31
|
+
Requires-Dist: transformers>=5.1.0; extra == "cuda13"
|
|
32
|
+
Provides-Extra: cuda12
|
|
33
|
+
Requires-Dist: jax[cuda12]>=0.4.23; extra == "cuda12"
|
|
34
|
+
Requires-Dist: torch>=2.9.1; extra == "cuda12"
|
|
35
|
+
Requires-Dist: torchax>=0.0.11; extra == "cuda12"
|
|
36
|
+
Requires-Dist: tokenizers>=0.22.2; extra == "cuda12"
|
|
37
|
+
Requires-Dist: transformers>=5.1.0; extra == "cuda12"
|
|
38
|
+
Provides-Extra: tpu
|
|
39
|
+
Requires-Dist: jax[tpu]>=0.4.23; extra == "tpu"
|
|
40
|
+
Requires-Dist: torch>=2.9.1; extra == "tpu"
|
|
41
|
+
Requires-Dist: torchax>=0.0.11; extra == "tpu"
|
|
42
|
+
Requires-Dist: tokenizers>=0.22.2; extra == "tpu"
|
|
43
|
+
Requires-Dist: transformers>=5.1.0; extra == "tpu"
|
|
44
|
+
Provides-Extra: cpu
|
|
45
|
+
Requires-Dist: jax>=0.4.23; extra == "cpu"
|
|
46
|
+
Requires-Dist: torch>=2.9.1; extra == "cpu"
|
|
47
|
+
Requires-Dist: torchax>=0.0.11; extra == "cpu"
|
|
48
|
+
Requires-Dist: tokenizers>=0.22.2; extra == "cpu"
|
|
49
|
+
Requires-Dist: transformers>=5.1.0; extra == "cpu"
|
|
50
|
+
Provides-Extra: web
|
|
51
|
+
Requires-Dist: fastapi>=0.100.0; extra == "web"
|
|
52
|
+
Requires-Dist: uvicorn[standard]>=0.23.0; extra == "web"
|
|
53
|
+
Requires-Dist: jinja2>=3.1.0; extra == "web"
|
|
54
|
+
Requires-Dist: aiofiles>=23.0.0; extra == "web"
|
|
55
|
+
Requires-Dist: sse-starlette>=1.6.0; extra == "web"
|
|
56
|
+
Requires-Dist: pyyaml>=6.0.0; extra == "web"
|
|
57
|
+
Requires-Dist: python-multipart>=0.0.22; extra == "web"
|
|
58
|
+
Requires-Dist: bcrypt>=5.0.0; extra == "web"
|
|
59
|
+
Requires-Dist: itsdangerous>=2.2.0; extra == "web"
|
|
60
|
+
Requires-Dist: watchdog>=4.0.0; extra == "web"
|
|
61
|
+
Provides-Extra: dev
|
|
62
|
+
Requires-Dist: coverage>=7.9.1; extra == "dev"
|
|
63
|
+
Requires-Dist: coveralls>=4.0.1; extra == "dev"
|
|
64
|
+
Requires-Dist: pytest>=8.4.1; extra == "dev"
|
|
65
|
+
Requires-Dist: pytest-cov>=6.2.1; extra == "dev"
|
|
66
|
+
Requires-Dist: ruff>=0.12.1; extra == "dev"
|
|
67
|
+
Requires-Dist: pre-commit>=4.2.0; extra == "dev"
|
|
68
|
+
Requires-Dist: mypy>=1.16.1; extra == "dev"
|
|
69
|
+
Provides-Extra: docs
|
|
70
|
+
Requires-Dist: mkdocs>=1.6.1; extra == "docs"
|
|
71
|
+
Requires-Dist: mkdocs-material>=9.6.20; extra == "docs"
|
|
72
|
+
Requires-Dist: mkdocstrings[python]>=0.30.0; extra == "docs"
|
|
73
|
+
Requires-Dist: pymdown-extensions>=10.15; extra == "docs"
|
|
74
|
+
Requires-Dist: mkdocs-gen-files>=0.5.0; extra == "docs"
|
|
75
|
+
Requires-Dist: mkdocs-literate-nav>=0.6.0; extra == "docs"
|
|
76
|
+
Requires-Dist: mkdocs-section-index>=0.3.0; extra == "docs"
|
|
77
|
+
Dynamic: license-file
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
libthx.egg-info/PKG-INFO
|
|
5
|
+
libthx.egg-info/SOURCES.txt
|
|
6
|
+
libthx.egg-info/dependency_links.txt
|
|
7
|
+
libthx.egg-info/entry_points.txt
|
|
8
|
+
libthx.egg-info/requires.txt
|
|
9
|
+
libthx.egg-info/top_level.txt
|
|
10
|
+
theseus/__init__.py
|
|
11
|
+
theseus/cli.py
|
|
12
|
+
theseus/config.py
|
|
13
|
+
theseus/job.py
|
|
14
|
+
theseus/mock.py
|
|
15
|
+
theseus/plot.py
|
|
16
|
+
theseus/quick.py
|
|
17
|
+
theseus/registry.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
click==8.2.1
|
|
2
|
+
flax>=0.12.2
|
|
3
|
+
jsonlines>=4.0.0
|
|
4
|
+
loguru>=0.7.3
|
|
5
|
+
numpy>=2.4.1
|
|
6
|
+
omegaconf>=2.3.0
|
|
7
|
+
orbax>=0.1.9
|
|
8
|
+
pydantic>=2.12.5
|
|
9
|
+
python-dotenv>=1.2.1
|
|
10
|
+
rich>=14.2.0
|
|
11
|
+
seaborn>=0.13.2
|
|
12
|
+
tiktoken>=0.12.0
|
|
13
|
+
torchax>=0.0.11
|
|
14
|
+
wandb>=0.24.1
|
|
15
|
+
datasets>=4.5.0
|
|
16
|
+
|
|
17
|
+
[cpu]
|
|
18
|
+
jax>=0.4.23
|
|
19
|
+
torch>=2.9.1
|
|
20
|
+
torchax>=0.0.11
|
|
21
|
+
tokenizers>=0.22.2
|
|
22
|
+
transformers>=5.1.0
|
|
23
|
+
|
|
24
|
+
[cuda12]
|
|
25
|
+
jax[cuda12]>=0.4.23
|
|
26
|
+
torch>=2.9.1
|
|
27
|
+
torchax>=0.0.11
|
|
28
|
+
tokenizers>=0.22.2
|
|
29
|
+
transformers>=5.1.0
|
|
30
|
+
|
|
31
|
+
[cuda13]
|
|
32
|
+
jax[cuda13]>=0.4.23
|
|
33
|
+
torch>=2.9.1
|
|
34
|
+
torchax>=0.0.11
|
|
35
|
+
tokenizers>=0.22.2
|
|
36
|
+
transformers>=5.1.0
|
|
37
|
+
|
|
38
|
+
[dev]
|
|
39
|
+
coverage>=7.9.1
|
|
40
|
+
coveralls>=4.0.1
|
|
41
|
+
pytest>=8.4.1
|
|
42
|
+
pytest-cov>=6.2.1
|
|
43
|
+
ruff>=0.12.1
|
|
44
|
+
pre-commit>=4.2.0
|
|
45
|
+
mypy>=1.16.1
|
|
46
|
+
|
|
47
|
+
[docs]
|
|
48
|
+
mkdocs>=1.6.1
|
|
49
|
+
mkdocs-material>=9.6.20
|
|
50
|
+
mkdocstrings[python]>=0.30.0
|
|
51
|
+
pymdown-extensions>=10.15
|
|
52
|
+
mkdocs-gen-files>=0.5.0
|
|
53
|
+
mkdocs-literate-nav>=0.6.0
|
|
54
|
+
mkdocs-section-index>=0.3.0
|
|
55
|
+
|
|
56
|
+
[fever]
|
|
57
|
+
wikipedia>=1.4.0
|
|
58
|
+
|
|
59
|
+
[huggingface]
|
|
60
|
+
tokenizers>=0.22.2
|
|
61
|
+
transformers>=5.1.0
|
|
62
|
+
|
|
63
|
+
[tpu]
|
|
64
|
+
jax[tpu]>=0.4.23
|
|
65
|
+
torch>=2.9.1
|
|
66
|
+
torchax>=0.0.11
|
|
67
|
+
tokenizers>=0.22.2
|
|
68
|
+
transformers>=5.1.0
|
|
69
|
+
|
|
70
|
+
[web]
|
|
71
|
+
fastapi>=0.100.0
|
|
72
|
+
uvicorn[standard]>=0.23.0
|
|
73
|
+
jinja2>=3.1.0
|
|
74
|
+
aiofiles>=23.0.0
|
|
75
|
+
sse-starlette>=1.6.0
|
|
76
|
+
pyyaml>=6.0.0
|
|
77
|
+
python-multipart>=0.0.22
|
|
78
|
+
bcrypt>=5.0.0
|
|
79
|
+
itsdangerous>=2.2.0
|
|
80
|
+
watchdog>=4.0.0
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
theseus
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "libthx"
|
|
3
|
+
version = "0.1.1"
|
|
4
|
+
requires-python = ">=3.11"
|
|
5
|
+
dependencies = [
|
|
6
|
+
"click==8.2.1", # mkdocs breaks otherwise...
|
|
7
|
+
"flax>=0.12.2",
|
|
8
|
+
"jsonlines>=4.0.0",
|
|
9
|
+
"loguru>=0.7.3",
|
|
10
|
+
"numpy>=2.4.1",
|
|
11
|
+
"omegaconf>=2.3.0",
|
|
12
|
+
"orbax>=0.1.9",
|
|
13
|
+
"pydantic>=2.12.5",
|
|
14
|
+
"python-dotenv>=1.2.1",
|
|
15
|
+
"rich>=14.2.0",
|
|
16
|
+
"seaborn>=0.13.2",
|
|
17
|
+
"tiktoken>=0.12.0",
|
|
18
|
+
"torchax>=0.0.11",
|
|
19
|
+
"wandb>=0.24.1",
|
|
20
|
+
"datasets>=4.5.0"
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
[project.scripts]
|
|
25
|
+
theseus = "theseus.cli:theseus"
|
|
26
|
+
|
|
27
|
+
[project.optional-dependencies]
|
|
28
|
+
fever = ["wikipedia>=1.4.0"]
|
|
29
|
+
huggingface = ["tokenizers>=0.22.2", "transformers>=5.1.0"]
|
|
30
|
+
cuda13 = [
|
|
31
|
+
"jax[cuda13]>=0.4.23",
|
|
32
|
+
"torch>=2.9.1",
|
|
33
|
+
"torchax>=0.0.11",
|
|
34
|
+
"tokenizers>=0.22.2",
|
|
35
|
+
"transformers>=5.1.0",
|
|
36
|
+
]
|
|
37
|
+
cuda12 = [
|
|
38
|
+
"jax[cuda12]>=0.4.23",
|
|
39
|
+
"torch>=2.9.1",
|
|
40
|
+
"torchax>=0.0.11",
|
|
41
|
+
"tokenizers>=0.22.2",
|
|
42
|
+
"transformers>=5.1.0",
|
|
43
|
+
]
|
|
44
|
+
tpu = [
|
|
45
|
+
"jax[tpu]>=0.4.23",
|
|
46
|
+
"torch>=2.9.1",
|
|
47
|
+
"torchax>=0.0.11",
|
|
48
|
+
"tokenizers>=0.22.2",
|
|
49
|
+
"transformers>=5.1.0",
|
|
50
|
+
]
|
|
51
|
+
cpu = [
|
|
52
|
+
"jax>=0.4.23",
|
|
53
|
+
"torch>=2.9.1",
|
|
54
|
+
"torchax>=0.0.11",
|
|
55
|
+
"tokenizers>=0.22.2",
|
|
56
|
+
"transformers>=5.1.0",
|
|
57
|
+
]
|
|
58
|
+
web = [
|
|
59
|
+
"fastapi>=0.100.0",
|
|
60
|
+
"uvicorn[standard]>=0.23.0",
|
|
61
|
+
"jinja2>=3.1.0",
|
|
62
|
+
"aiofiles>=23.0.0",
|
|
63
|
+
"sse-starlette>=1.6.0",
|
|
64
|
+
"pyyaml>=6.0.0",
|
|
65
|
+
"python-multipart>=0.0.22",
|
|
66
|
+
"bcrypt>=5.0.0",
|
|
67
|
+
"itsdangerous>=2.2.0",
|
|
68
|
+
"watchdog>=4.0.0",
|
|
69
|
+
]
|
|
70
|
+
dev = [
|
|
71
|
+
"coverage>=7.9.1",
|
|
72
|
+
"coveralls>=4.0.1",
|
|
73
|
+
"pytest>=8.4.1",
|
|
74
|
+
"pytest-cov>=6.2.1",
|
|
75
|
+
"ruff>=0.12.1",
|
|
76
|
+
"pre-commit>=4.2.0",
|
|
77
|
+
"mypy>=1.16.1",
|
|
78
|
+
]
|
|
79
|
+
docs = [
|
|
80
|
+
"mkdocs>=1.6.1",
|
|
81
|
+
"mkdocs-material>=9.6.20",
|
|
82
|
+
"mkdocstrings[python]>=0.30.0",
|
|
83
|
+
"pymdown-extensions>=10.15",
|
|
84
|
+
"mkdocs-gen-files>=0.5.0",
|
|
85
|
+
"mkdocs-literate-nav>=0.6.0",
|
|
86
|
+
"mkdocs-section-index>=0.3.0",
|
|
87
|
+
]
|
|
88
|
+
|
|
89
|
+
[dependency-groups]
|
|
90
|
+
# job-specific dependencies
|
|
91
|
+
fever = [
|
|
92
|
+
"wikipedia>=1.4.0",
|
|
93
|
+
]
|
|
94
|
+
|
|
95
|
+
# ML dependencies
|
|
96
|
+
# sorry for the cursèd imports; we need
|
|
97
|
+
# BOTH Jax and torch b/c Jax is used as
|
|
98
|
+
# our main infrastructure and torch is
|
|
99
|
+
# sometimes invoked as a compatibility layer
|
|
100
|
+
# in particular to load HF weights etc.
|
|
101
|
+
#
|
|
102
|
+
# also we use torchax for best-effort tracing
|
|
103
|
+
# of models that we didn't reimplement
|
|
104
|
+
huggingface = [
|
|
105
|
+
"tokenizers>=0.22.2",
|
|
106
|
+
"transformers>=5.1.0"
|
|
107
|
+
]
|
|
108
|
+
cuda13 = [
|
|
109
|
+
"jax[cuda13]>=0.4.23",
|
|
110
|
+
"torch>=2.9.1",
|
|
111
|
+
"torchax>=0.0.11",
|
|
112
|
+
{include-group = "huggingface"}
|
|
113
|
+
]
|
|
114
|
+
cuda12 = [
|
|
115
|
+
"jax[cuda12]>=0.4.23",
|
|
116
|
+
"torch>=2.9.1",
|
|
117
|
+
"torchax>=0.0.11",
|
|
118
|
+
{include-group = "huggingface"}
|
|
119
|
+
]
|
|
120
|
+
tpu = [
|
|
121
|
+
"jax[tpu]>=0.4.23",
|
|
122
|
+
"torch>=2.9.1",
|
|
123
|
+
"torchax>=0.0.11",
|
|
124
|
+
{include-group = "huggingface"}
|
|
125
|
+
]
|
|
126
|
+
cpu = [
|
|
127
|
+
"jax>=0.4.23",
|
|
128
|
+
"torch>=2.9.1",
|
|
129
|
+
"torchax>=0.0.11",
|
|
130
|
+
{include-group = "huggingface"}
|
|
131
|
+
]
|
|
132
|
+
|
|
133
|
+
# web dependencies
|
|
134
|
+
web = [
|
|
135
|
+
"fastapi>=0.100.0",
|
|
136
|
+
"uvicorn[standard]>=0.23.0",
|
|
137
|
+
"jinja2>=3.1.0",
|
|
138
|
+
"aiofiles>=23.0.0",
|
|
139
|
+
"sse-starlette>=1.6.0",
|
|
140
|
+
"pyyaml>=6.0.0",
|
|
141
|
+
"python-multipart>=0.0.22",
|
|
142
|
+
"bcrypt>=5.0.0",
|
|
143
|
+
"itsdangerous>=2.2.0",
|
|
144
|
+
"watchdog>=4.0.0",
|
|
145
|
+
]
|
|
146
|
+
|
|
147
|
+
# developer helpers
|
|
148
|
+
dev = [
|
|
149
|
+
"coverage>=7.9.1",
|
|
150
|
+
"coveralls>=4.0.1",
|
|
151
|
+
"pytest>=8.4.1",
|
|
152
|
+
"pytest-cov>=6.2.1",
|
|
153
|
+
"ruff>=0.12.1",
|
|
154
|
+
"pre-commit>=4.2.0",
|
|
155
|
+
"mypy>=1.16.1",
|
|
156
|
+
]
|
|
157
|
+
docs = [
|
|
158
|
+
"mkdocs>=1.6.1",
|
|
159
|
+
"mkdocs-material>=9.6.20",
|
|
160
|
+
"mkdocstrings[python]>=0.30.0",
|
|
161
|
+
"pymdown-extensions>=10.15",
|
|
162
|
+
"mkdocs-gen-files>=0.5.0",
|
|
163
|
+
"mkdocs-literate-nav>=0.6.0",
|
|
164
|
+
"mkdocs-section-index>=0.3.0",
|
|
165
|
+
]
|
|
166
|
+
|
|
167
|
+
# utilities / aliases
|
|
168
|
+
all = [
|
|
169
|
+
{include-group = "dev"},
|
|
170
|
+
{include-group = "web"},
|
|
171
|
+
{include-group = "docs"},
|
|
172
|
+
# and all job-specific dependencies
|
|
173
|
+
{include-group = "fever"}
|
|
174
|
+
]
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
[tool.uv]
|
|
178
|
+
default-groups = ["all"]
|
|
179
|
+
package = true
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
conflicts = [
|
|
183
|
+
# Exactly one of these “flavors” may be used at a time.
|
|
184
|
+
[
|
|
185
|
+
{ group = "cpu" },
|
|
186
|
+
{ group = "tpu" },
|
|
187
|
+
{ group = "cuda12" },
|
|
188
|
+
{ group = "cuda13" },
|
|
189
|
+
]
|
|
190
|
+
]
|
|
191
|
+
|
|
192
|
+
[tool.uv.sources]
|
|
193
|
+
torch = [
|
|
194
|
+
{ index = "pytorch-cpu", group = "cpu" },
|
|
195
|
+
{ index = "pytorch-cpu", group = "tpu" },
|
|
196
|
+
{ index = "pytorch-cu128", group = "cuda12" },
|
|
197
|
+
{ index = "pytorch-cu130", group = "cuda13" },
|
|
198
|
+
]
|
|
199
|
+
|
|
200
|
+
[[tool.uv.index]]
|
|
201
|
+
name = "pytorch-cpu"
|
|
202
|
+
url = "https://download.pytorch.org/whl/cpu"
|
|
203
|
+
explicit = true
|
|
204
|
+
|
|
205
|
+
[[tool.uv.index]]
|
|
206
|
+
name = "pytorch-cu128"
|
|
207
|
+
url = "https://download.pytorch.org/whl/cu128"
|
|
208
|
+
explicit = true
|
|
209
|
+
|
|
210
|
+
[[tool.uv.index]]
|
|
211
|
+
name = "pytorch-cu130"
|
|
212
|
+
url = "https://download.pytorch.org/whl/cu130"
|
|
213
|
+
explicit = true
|
|
214
|
+
|
|
215
|
+
[tool.ruff]
|
|
216
|
+
line-length = 88
|
|
217
|
+
indent-width = 4
|
|
218
|
+
target-version = "py312"
|
|
219
|
+
|
|
220
|
+
[tool.ruff.format]
|
|
221
|
+
indent-style = "space"
|
|
222
|
+
docstring-code-format = true
|
|
223
|
+
docstring-code-line-length = 20
|
|
224
|
+
|
|
225
|
+
[tool.mypy]
|
|
226
|
+
plugins = ['pydantic.mypy']
|
|
227
|
+
python_version = "3.12"
|
|
228
|
+
strict = true
|
|
229
|
+
ignore_missing_imports = true
|
|
230
|
+
exclude = ["tests", "examples"]
|
|
231
|
+
|
|
232
|
+
[[tool.mypy.overrides]]
|
|
233
|
+
module = "huggingface_hub.*"
|
|
234
|
+
ignore_missing_imports = true
|
|
235
|
+
|
|
236
|
+
[tool.setuptools.packages.find]
|
|
237
|
+
where = ["."]
|
|
238
|
+
include = ["theseus"]
|
|
239
|
+
exclude = []
|
libthx-0.1.1/setup.cfg
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|