pelutils 3.9.0__tar.gz → 4.0.0a2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pelutils-4.0.0a2/PKG-INFO +333 -0
- pelutils-4.0.0a2/README.md +309 -0
- pelutils-4.0.0a2/pelutils/__init__.py +3 -0
- {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/__version__.py +1 -1
- {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/_c/__init__.py +5 -8
- {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/_entry_points/linecounter.py +6 -8
- pelutils-4.0.0a2/pelutils/job_parser/__init__.py +61 -0
- pelutils-4.0.0a2/pelutils/job_parser/_job_parser.py +364 -0
- pelutils-4.0.0a2/pelutils/job_parser/_structs.py +260 -0
- pelutils-4.0.0a2/pelutils/logging/__init__.py +39 -0
- pelutils-3.9.0/pelutils/logging/__init__.py → pelutils-4.0.0a2/pelutils/logging/_logger.py +24 -54
- pelutils-4.0.0a2/pelutils/logging/_rich_string.py +49 -0
- {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/logging/_rotate.py +0 -2
- {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/logging/_utils.py +16 -16
- pelutils-4.0.0a2/pelutils/misc/__init__.py +51 -0
- pelutils-3.9.0/pelutils/ds/__init__.py → pelutils-4.0.0a2/pelutils/misc/_array.py +38 -30
- pelutils-4.0.0a2/pelutils/misc/_conditional_import.py +31 -0
- pelutils-4.0.0a2/pelutils/misc/_files.py +60 -0
- pelutils-4.0.0a2/pelutils/misc/_git.py +31 -0
- pelutils-4.0.0a2/pelutils/misc/_misc.py +13 -0
- pelutils-4.0.0a2/pelutils/misc/_platform.py +74 -0
- pelutils-4.0.0a2/pelutils/misc/_table.py +109 -0
- pelutils-4.0.0a2/pelutils/plots/__init__.py +50 -0
- pelutils-4.0.0a2/pelutils/plots/_figure.py +116 -0
- pelutils-4.0.0a2/pelutils/plots/_histogram.py +50 -0
- pelutils-4.0.0a2/pelutils/plots/_utils.py +36 -0
- pelutils-4.0.0a2/pelutils/serialization/__init__.py +43 -0
- pelutils-4.0.0a2/pelutils/serialization/_jsonl.py +38 -0
- {pelutils-3.9.0/pelutils/datastorage2 → pelutils-4.0.0a2/pelutils/serialization}/_pretty_json.py +48 -17
- pelutils-4.0.0a2/pelutils/serialization/_universal_json_model.py +53 -0
- pelutils-4.0.0a2/pelutils/stats/__init__.py +53 -0
- pelutils-3.9.0/pelutils/ds/distributions.py → pelutils-4.0.0a2/pelutils/stats/_distributions.py +0 -2
- pelutils-4.0.0a2/pelutils/stats/_stats.py +26 -0
- pelutils-4.0.0a2/pelutils/tests/__init__.py +32 -0
- pelutils-4.0.0a2/pelutils/tests/_tests.py +88 -0
- pelutils-4.0.0a2/pelutils/ticktock/__init__.py +42 -0
- pelutils-4.0.0a2/pelutils/ticktock/_ticktock.py +360 -0
- pelutils-4.0.0a2/pelutils/types/__init__.py +25 -0
- pelutils-4.0.0a2/pelutils/types/_types.py +17 -0
- pelutils-4.0.0a2/pelutils.egg-info/PKG-INFO +333 -0
- pelutils-4.0.0a2/pelutils.egg-info/SOURCES.txt +52 -0
- {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils.egg-info/entry_points.txt +0 -1
- {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils.egg-info/requires.txt +11 -2
- {pelutils-3.9.0 → pelutils-4.0.0a2}/setup.py +12 -10
- pelutils-3.9.0/PKG-INFO +0 -389
- pelutils-3.9.0/README.md +0 -364
- pelutils-3.9.0/pelutils/__init__.py +0 -322
- pelutils-3.9.0/pelutils/_utils.py +0 -46
- pelutils-3.9.0/pelutils/datastorage.py +0 -151
- pelutils-3.9.0/pelutils/datastorage2/__init__.py +0 -74
- pelutils-3.9.0/pelutils/ds/plots.py +0 -262
- pelutils-3.9.0/pelutils/ds/stats.py +0 -70
- pelutils-3.9.0/pelutils/format.py +0 -143
- pelutils-3.9.0/pelutils/jsonl.py +0 -40
- pelutils-3.9.0/pelutils/parser.py +0 -548
- pelutils-3.9.0/pelutils/pretty_json.py +0 -41
- pelutils-3.9.0/pelutils/tests.py +0 -108
- pelutils-3.9.0/pelutils/ticktock.py +0 -439
- pelutils-3.9.0/pelutils/types.py +0 -14
- pelutils-3.9.0/pelutils.egg-info/PKG-INFO +0 -389
- pelutils-3.9.0/pelutils.egg-info/SOURCES.txt +0 -47
- pelutils-3.9.0/tests/test_datastorage.py +0 -84
- pelutils-3.9.0/tests/test_format.py +0 -53
- pelutils-3.9.0/tests/test_init.py +0 -238
- pelutils-3.9.0/tests/test_jsonl.py +0 -44
- pelutils-3.9.0/tests/test_parser.py +0 -444
- pelutils-3.9.0/tests/test_pretty_json.py +0 -302
- pelutils-3.9.0/tests/test_tests.py +0 -23
- pelutils-3.9.0/tests/test_ticktock.py +0 -405
- pelutils-3.9.0/tests/test_types.py +0 -37
- pelutils-3.9.0/tests/test_utils.py +0 -13
- {pelutils-3.9.0 → pelutils-4.0.0a2}/MANIFEST.in +0 -0
- {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/_c/ds.c +0 -0
- {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/_c/hashmap.c/LICENSE +0 -0
- {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/_c/hashmap.c/hashmap.c +0 -0
- {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/_c/hashmap.c/hashmap.h +0 -0
- {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/_entry_points/__init__.py +0 -0
- {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/py.typed +0 -0
- {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils.egg-info/dependency_links.txt +0 -0
- {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils.egg-info/top_level.txt +0 -0
- {pelutils-3.9.0 → pelutils-4.0.0a2}/setup.cfg +0 -0
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: pelutils
|
|
3
|
+
Version: 4.0.0a2
|
|
4
|
+
Summary: The Swiss army knife of Python projects
|
|
5
|
+
Home-page: https://github.com/peleiden/pelutils
|
|
6
|
+
Download-URL: https://pypi.org/project/pelutils/
|
|
7
|
+
Author: Asger Laurits Schultz, Søren Winkel Holm
|
|
8
|
+
Author-email: asger.s@protonmail.com, swholm@protonmail.com
|
|
9
|
+
License: MIT
|
|
10
|
+
Keywords: utility,logger,parser,profiling,plotting
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Provides-Extra: docs
|
|
23
|
+
License-File: pelutils/_c/hashmap.c/LICENSE
|
|
24
|
+
|
|
25
|
+
# pelutils
|
|
26
|
+
|
|
27
|
+
[](https://github.com/astral-sh/ruff)
|
|
28
|
+
[](https://docs.basedpyright.com)
|
|
29
|
+
[](https://github.com/peleiden/pelutils/actions/workflows/checks.yml)
|
|
30
|
+
[](https://coveralls.io/github/peleiden/pelutils?branch=master)
|
|
31
|
+
[](https://pypi.org/project/pelutils/)
|
|
32
|
+
[](https://img.shields.io/pypi/pyversions/pelutils)
|
|
33
|
+
[](https://github.com/peleiden/pelutils/blob/master/LICENSE.txt)
|
|
34
|
+
[](https://pelutils.readthedocs.io/en/latest/)
|
|
35
|
+
|
|
36
|
+
**The Swiss army knife of Python projects**
|
|
37
|
+
|
|
38
|
+
Every project, experiment, or one-off script inevitably ends up reinventing much of
|
|
39
|
+
the same plumbing: some way to time a loop, saving and loading data to and from disk
|
|
40
|
+
in a convenient and human-readable manner, a decent logger, parsing a config file, a readable table.
|
|
41
|
+
`pelutils` bundles the good versions of these so you can get straight to the actual
|
|
42
|
+
work. It has no required dependencies beyond the scientific-Python staples, ships
|
|
43
|
+
type hints (including `py.typed`), and is easy to start using.
|
|
44
|
+
|
|
45
|
+
📖 **Full documentation: [pelutils.readthedocs.io](https://pelutils.readthedocs.io)**
|
|
46
|
+
|
|
47
|
+
## Highlights
|
|
48
|
+
|
|
49
|
+
- **Logger** — easy-to-use, colourful console output, log files with rotation, automatic
|
|
50
|
+
stacktrace capture, and safe logging from multiple processes.
|
|
51
|
+
- **Timer & profiler** — a Matlab-style `tick`/`tock` timer and a near-zero-overhead
|
|
52
|
+
profiler that prints a readable breakdown of where your time goes.
|
|
53
|
+
- **`UniversalJsonModel`** — a `pydantic.BaseModel` that can save *any* attribute to a
|
|
54
|
+
human-readable JSON file (numpy arrays, tensors, and other unserialisable types are
|
|
55
|
+
pickled transparently) and load it straight back.
|
|
56
|
+
- **`JobParser`** — one parser that unifies command-line arguments and config files,
|
|
57
|
+
with support for running many jobs from a single config and auto-documenting them.
|
|
58
|
+
- **`unique`** — a linear-time drop-in for `numpy.unique`, dramatically faster on
|
|
59
|
+
large arrays (backed by a small C extension).
|
|
60
|
+
- **Data-science helpers** — a `matplotlib` `Figure` context manager with improved default
|
|
61
|
+
settings over `matplotlib`, histogram binning, reparametrised scipy distributions,
|
|
62
|
+
`z_score`, LaTeX-ready tables, and numpy type aliases.
|
|
63
|
+
|
|
64
|
+
## Installation
|
|
65
|
+
|
|
66
|
+
```sh
|
|
67
|
+
pip install pelutils
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`pelutils` supports Python 3.11+. A small subset of functionality can additionally
|
|
71
|
+
make use of [`PyTorch`](https://pytorch.org), which must be installed separately.
|
|
72
|
+
|
|
73
|
+
> **Importing:** every feature lives in its own submodule and must be imported from
|
|
74
|
+
> there — e.g. `from pelutils.logging import log`. Only `__version__` is exported at
|
|
75
|
+
> the top level. See the [docs](https://pelutils.readthedocs.io) for the full API.
|
|
76
|
+
|
|
77
|
+
## Logging
|
|
78
|
+
|
|
79
|
+
Python's built-in `logging` is powerful but fiddly to set up, and a bare `print` gives
|
|
80
|
+
you no importance levels, no timestamps, and nothing on disk to look at afterwards. This logger hits
|
|
81
|
+
the sweet spot: one `configure` call and you get colour-coded, timestamped output to both
|
|
82
|
+
the console and a log file, with severity levels, log rotation, one-line exception logging,
|
|
83
|
+
and multiprocessing-safe collection.
|
|
84
|
+
|
|
85
|
+
```py
|
|
86
|
+
from pelutils.logging import log, LogLevels
|
|
87
|
+
|
|
88
|
+
# Set up the logger by giving it the file to write to
|
|
89
|
+
# Omit the path to only print, never write a file
|
|
90
|
+
log.configure("train.log")
|
|
91
|
+
|
|
92
|
+
log.section("Training run") # Highlighted section header
|
|
93
|
+
log(f"Loaded {len(dataset):,} samples") # Logs at INFO level
|
|
94
|
+
for epoch in range(epochs):
|
|
95
|
+
loss = train_one_epoch()
|
|
96
|
+
log.debug(f"Epoch {epoch}: loss {loss:.4f}") # Logs at DEBUG level - by default, only saved to the log file, but not printed to the console
|
|
97
|
+
if loss > 1e3:
|
|
98
|
+
log.warning("Loss is diverging") # Logs at WARNING level
|
|
99
|
+
log.debug("Final weights", model.state_dict().keys())
|
|
100
|
+
|
|
101
|
+
# If an exeption occurs in a piece of code, log it with its full (chained) stacktrace, then re-raise
|
|
102
|
+
with log.log_errors:
|
|
103
|
+
save_checkpoint(model)
|
|
104
|
+
|
|
105
|
+
# Temporarily change or silence the log level
|
|
106
|
+
with log.level(LogLevels.ERROR):
|
|
107
|
+
log.warning("Suppressed")
|
|
108
|
+
|
|
109
|
+
# Rotate the log file by time or size
|
|
110
|
+
log.configure("train.log", rotation="day") # or "1 GB", "hour", ...
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
When using multiprocessing, wrap a worker in `with log.collect:` so its lines are
|
|
114
|
+
written together instead of interleaving with other processes. See the
|
|
115
|
+
[logging docs](https://pelutils.readthedocs.io/en/latest/api/pelutils.logging.html) for input helpers, multiple loggers,
|
|
116
|
+
and more.
|
|
117
|
+
|
|
118
|
+
## Timing and profiling
|
|
119
|
+
|
|
120
|
+
When you want to know where a script spends its time, `cProfile` gives you a wall of
|
|
121
|
+
function-level numbers, and manual `time.perf_counter()` calls quickly turn into
|
|
122
|
+
bookkeeping. `TickTock` sits in between: wrap the sections *you* care about in named,
|
|
123
|
+
nestable context managers and print a readable table of totals, hit counts, averages,
|
|
124
|
+
and each section's share of its parent's time. The per-profile overhead is tiny, so it
|
|
125
|
+
happily lives inside hot loops and long-running jobs.
|
|
126
|
+
|
|
127
|
+
```py
|
|
128
|
+
from pelutils.ticktock import TT
|
|
129
|
+
|
|
130
|
+
# Time a single block, Matlab style
|
|
131
|
+
TT.tick()
|
|
132
|
+
model = train_model(data)
|
|
133
|
+
print(f"Training took {TT.tock():.1f} s")
|
|
134
|
+
|
|
135
|
+
# Profile named sections across a loop, nesting them however you like
|
|
136
|
+
for image in images:
|
|
137
|
+
with TT.profile("Process image"):
|
|
138
|
+
with TT.profile("Load"):
|
|
139
|
+
img = load(image)
|
|
140
|
+
with TT.profile("Resize"):
|
|
141
|
+
img = resize(img, (224, 224))
|
|
142
|
+
with TT.profile("Inference"):
|
|
143
|
+
predict(model, img)
|
|
144
|
+
|
|
145
|
+
# Print a table of hits, total time, and average time for each section
|
|
146
|
+
print(TT)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
`with TT.profile("name", hits=n):` records `n` hits at once, which is handy for very
|
|
150
|
+
tight loops or for a block that processes `n` items in parallel. `TT.do_at_interval(...)`
|
|
151
|
+
turns the same instance into a throttle for periodic tasks. The default `TT` is a
|
|
152
|
+
shared instance; construct your own with `TickTock()` when you need isolation — most
|
|
153
|
+
importantly one per thread, as profiling is not thread-safe.
|
|
154
|
+
|
|
155
|
+
## Serialisation
|
|
156
|
+
|
|
157
|
+
`UniversalJsonModel` extends `pydantic.BaseModel` with `save`/`load` methods and can
|
|
158
|
+
serialise attributes that pydantic cannot — numpy arrays, tensors, and arbitrary
|
|
159
|
+
objects are base64-pickled inline, everything else stays plain, human-readable JSON.
|
|
160
|
+
Long lists are wrapped to the line-length limit instead of one element per line.
|
|
161
|
+
|
|
162
|
+
```py
|
|
163
|
+
import numpy as np
|
|
164
|
+
from pydantic import BaseModel
|
|
165
|
+
from pelutils.serialization import UniversalJsonModel
|
|
166
|
+
from pelutils.types import FloatArray
|
|
167
|
+
|
|
168
|
+
class Nested(BaseModel):
|
|
169
|
+
label: str
|
|
170
|
+
|
|
171
|
+
class Result(UniversalJsonModel):
|
|
172
|
+
accuracy: float
|
|
173
|
+
predictions: FloatArray # numpy arrays are handled automatically
|
|
174
|
+
meta: Nested
|
|
175
|
+
|
|
176
|
+
result = Result(
|
|
177
|
+
accuracy=0.97,
|
|
178
|
+
predictions=np.arange(5, dtype=np.float16),
|
|
179
|
+
meta=Nested(label="run-1"),
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
result.save("results/run-1.json")
|
|
183
|
+
result = Result.load("results/run-1.json")
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Use `to_json_dict()` / `from_json_dict(...)` to convert to and from a plain dict
|
|
187
|
+
without touching the filesystem — useful for nesting inside other structures. The
|
|
188
|
+
`pretty_json` helper function is also available on its own. The `serialization`
|
|
189
|
+
module also includes JSONL read/write helpers (`jsonl_dump`, `jsonl_load`, ...)
|
|
190
|
+
with largely the same interface as is provided by the built-in `json` module.
|
|
191
|
+
|
|
192
|
+
## Config and command-line argument parsing
|
|
193
|
+
|
|
194
|
+
`JobParser` combines typed command-line options with INI config files. CLI values
|
|
195
|
+
override config values, which override defaults. Declare `RequiredArg` for values
|
|
196
|
+
every job must provide, `OptionalArg` for values with defaults, and `Flag` for
|
|
197
|
+
booleans. Names are `--kebab-case` on the command line and `snake_case` attributes on
|
|
198
|
+
the resulting job.
|
|
199
|
+
|
|
200
|
+
```py
|
|
201
|
+
from pathlib import Path
|
|
202
|
+
from pelutils.job_parser import Flag, JobParser, OptionalArg, RequiredArg
|
|
203
|
+
|
|
204
|
+
parser = JobParser(
|
|
205
|
+
RequiredArg("data-path", help="Training data directory"),
|
|
206
|
+
OptionalArg("learning-rate", default=1e-4, type=float, help="Optimizer learning rate"),
|
|
207
|
+
Flag("fp16", help="Use mixed precision"),
|
|
208
|
+
multiple_jobs=True,
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
for job in parser.parse_jobs():
|
|
212
|
+
print(job.name, job.data_path, job.learning_rate, job.fp16)
|
|
213
|
+
job.write_documentation(Path("runs") / job.name / "arguments.ini")
|
|
214
|
+
# ... run your application with the resolved job values
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
A single config file can define several named jobs (with a shared `[DEFAULT]`
|
|
218
|
+
section), and one CLI override applies to all of them:
|
|
219
|
+
|
|
220
|
+
```console
|
|
221
|
+
python main.py --config-file config.ini --learning-rate 5e-5
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
For a single job, drop `multiple_jobs=True` and call `parse_job()` instead. A config
|
|
225
|
+
path can target one section directly, e.g. `--config-file config.ini:low-lr`. See the
|
|
226
|
+
[job parser docs](https://pelutils.readthedocs.io/en/latest/api/pelutils.job_parser.html) for auto-documentation details.
|
|
227
|
+
|
|
228
|
+
## Fast `unique`
|
|
229
|
+
|
|
230
|
+
A linear-time alternative to `numpy.unique`, significantly faster on large arrays. It
|
|
231
|
+
also accepts torch tensors and pandas series. The returned elements are unsorted.
|
|
232
|
+
|
|
233
|
+
```py
|
|
234
|
+
import numpy as np
|
|
235
|
+
from pelutils.misc import unique
|
|
236
|
+
|
|
237
|
+
x = np.random.randint(0, 100, size=10_000_000)
|
|
238
|
+
values = unique(x)
|
|
239
|
+
values, index, inverse, counts = unique(
|
|
240
|
+
x, return_index=True, return_inverse=True, return_counts=True,
|
|
241
|
+
)
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
## Data science
|
|
245
|
+
|
|
246
|
+
### Statistics
|
|
247
|
+
|
|
248
|
+
Common statistical helpers, plus wrappers around scipy distributions reparametrised
|
|
249
|
+
as in Jim Pitman's *Probability* (rather than scipy's `loc`/`scale`, which are
|
|
250
|
+
unintuitive for many distributions).
|
|
251
|
+
|
|
252
|
+
```py
|
|
253
|
+
from pelutils.stats import z_score
|
|
254
|
+
from pelutils.stats import expon
|
|
255
|
+
|
|
256
|
+
# 95 % confidence interval half-width for a standard normal (defaults give ~1.96)
|
|
257
|
+
half_width = std * z_score()
|
|
258
|
+
|
|
259
|
+
# One-sided z value for an Exponential(λ=2) at the 1 % significance level
|
|
260
|
+
zval = z_score(alpha=0.01, two_sided=False, distribution=expon(lambda_=2))
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
### Plotting
|
|
264
|
+
|
|
265
|
+
The `Figure` context manager fixes common `matplotlib` annoyances — sensible default
|
|
266
|
+
figure and font sizes, easy styling — and saves and closes the figure for you while
|
|
267
|
+
restoring `rcParams` afterwards.
|
|
268
|
+
|
|
269
|
+
```py
|
|
270
|
+
import matplotlib.pyplot as plt
|
|
271
|
+
from pelutils.plots import Figure, histogram, normal_binning
|
|
272
|
+
|
|
273
|
+
with Figure("plot.png", figsize=(20, 10), fontsize=20):
|
|
274
|
+
plt.scatter(x, y, label="Data")
|
|
275
|
+
plt.grid()
|
|
276
|
+
plt.title("Very nice plot")
|
|
277
|
+
# Saved to plot.png and closed here
|
|
278
|
+
|
|
279
|
+
# histogram returns x and y coordinates ready for unpacking
|
|
280
|
+
plt.plot(*histogram(data, binning_fn=normal_binning))
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
Three binning functions are provided — `linear_binning`, `log_binning`, and
|
|
284
|
+
`normal_binning` (more resolution near the centre of roughly-normal data) — and custom
|
|
285
|
+
binning functions are supported. See the
|
|
286
|
+
[plotting docs](https://pelutils.readthedocs.io/en/latest/api/pelutils.plots.html).
|
|
287
|
+
|
|
288
|
+
### Numpy type aliases
|
|
289
|
+
|
|
290
|
+
Type aliases so you (and your type checker) do not have to track array dtypes by hand.
|
|
291
|
+
|
|
292
|
+
```py
|
|
293
|
+
from pelutils.types import FloatArray, IntArray, BoolArray
|
|
294
|
+
|
|
295
|
+
def process(features: FloatArray, labels: IntArray, mask: BoolArray): ...
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
## Also included
|
|
299
|
+
|
|
300
|
+
- `pelutils.misc.Table` — build aligned text tables which can also be easily export to LaTeX with `Table.to_latex()`.
|
|
301
|
+
- `pelutils.misc.hardware_info` / `OS` — describe the machine the code runs on.
|
|
302
|
+
- `pelutils.misc.git_repo_info` — the repo and commit the code is executing in.
|
|
303
|
+
- Assorted file and dict helpers (`reverse_line_iterator`, `except_keys`, ...).
|
|
304
|
+
- `pelutils.tests` — pytest helpers: a `UnitTestCollection` base class with a managed temp directory, and a `restore_argv` decorator.
|
|
305
|
+
|
|
306
|
+
## Supported platforms
|
|
307
|
+
|
|
308
|
+
Precompiled wheels are provided for most common platforms (not 32-bit systems). If no
|
|
309
|
+
wheel matches, `pip` builds from source which requires `<Python.h>` — install it with
|
|
310
|
+
`sudo apt install python3-dev` (Ubuntu) or `sudo dnf install python3-devel` (Fedora).
|
|
311
|
+
|
|
312
|
+
## Updating and releasing
|
|
313
|
+
|
|
314
|
+
`pelutils` uses the `master` branch as a stable development branch.
|
|
315
|
+
The `release` branch contains the latest version on PyPI.
|
|
316
|
+
New features should be made in feature branches from `master` that can be merged into `master` once ready.
|
|
317
|
+
When a new release is ready, update `pelutils/__version__.py` and rebase `master` onto `release`.
|
|
318
|
+
Then push a new tag from `release` named `vX.Y.Z`.
|
|
319
|
+
|
|
320
|
+
When new code is merged into `master`, a number of checks are run.
|
|
321
|
+
These can be tested locally with the following commands.
|
|
322
|
+
```sh
|
|
323
|
+
# Linting and formatting
|
|
324
|
+
ruff format pelutils tests
|
|
325
|
+
ruff check pelutils tests
|
|
326
|
+
# Type checking
|
|
327
|
+
basedpyright pelutils
|
|
328
|
+
# Unit tests
|
|
329
|
+
python -m pytest tests --cov pelutils
|
|
330
|
+
# Build docs
|
|
331
|
+
# Once build, open docs/build/html/index.html in your browser to see them
|
|
332
|
+
make -C docs html
|
|
333
|
+
```
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
# pelutils
|
|
2
|
+
|
|
3
|
+
[](https://github.com/astral-sh/ruff)
|
|
4
|
+
[](https://docs.basedpyright.com)
|
|
5
|
+
[](https://github.com/peleiden/pelutils/actions/workflows/checks.yml)
|
|
6
|
+
[](https://coveralls.io/github/peleiden/pelutils?branch=master)
|
|
7
|
+
[](https://pypi.org/project/pelutils/)
|
|
8
|
+
[](https://img.shields.io/pypi/pyversions/pelutils)
|
|
9
|
+
[](https://github.com/peleiden/pelutils/blob/master/LICENSE.txt)
|
|
10
|
+
[](https://pelutils.readthedocs.io/en/latest/)
|
|
11
|
+
|
|
12
|
+
**The Swiss army knife of Python projects**
|
|
13
|
+
|
|
14
|
+
Every project, experiment, or one-off script inevitably ends up reinventing much of
|
|
15
|
+
the same plumbing: some way to time a loop, saving and loading data to and from disk
|
|
16
|
+
in a convenient and human-readable manner, a decent logger, parsing a config file, a readable table.
|
|
17
|
+
`pelutils` bundles the good versions of these so you can get straight to the actual
|
|
18
|
+
work. It has no required dependencies beyond the scientific-Python staples, ships
|
|
19
|
+
type hints (including `py.typed`), and is easy to start using.
|
|
20
|
+
|
|
21
|
+
📖 **Full documentation: [pelutils.readthedocs.io](https://pelutils.readthedocs.io)**
|
|
22
|
+
|
|
23
|
+
## Highlights
|
|
24
|
+
|
|
25
|
+
- **Logger** — easy-to-use, colourful console output, log files with rotation, automatic
|
|
26
|
+
stacktrace capture, and safe logging from multiple processes.
|
|
27
|
+
- **Timer & profiler** — a Matlab-style `tick`/`tock` timer and a near-zero-overhead
|
|
28
|
+
profiler that prints a readable breakdown of where your time goes.
|
|
29
|
+
- **`UniversalJsonModel`** — a `pydantic.BaseModel` that can save *any* attribute to a
|
|
30
|
+
human-readable JSON file (numpy arrays, tensors, and other unserialisable types are
|
|
31
|
+
pickled transparently) and load it straight back.
|
|
32
|
+
- **`JobParser`** — one parser that unifies command-line arguments and config files,
|
|
33
|
+
with support for running many jobs from a single config and auto-documenting them.
|
|
34
|
+
- **`unique`** — a linear-time drop-in for `numpy.unique`, dramatically faster on
|
|
35
|
+
large arrays (backed by a small C extension).
|
|
36
|
+
- **Data-science helpers** — a `matplotlib` `Figure` context manager with improved default
|
|
37
|
+
settings over `matplotlib`, histogram binning, reparametrised scipy distributions,
|
|
38
|
+
`z_score`, LaTeX-ready tables, and numpy type aliases.
|
|
39
|
+
|
|
40
|
+
## Installation
|
|
41
|
+
|
|
42
|
+
```sh
|
|
43
|
+
pip install pelutils
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
`pelutils` supports Python 3.11+. A small subset of functionality can additionally
|
|
47
|
+
make use of [`PyTorch`](https://pytorch.org), which must be installed separately.
|
|
48
|
+
|
|
49
|
+
> **Importing:** every feature lives in its own submodule and must be imported from
|
|
50
|
+
> there — e.g. `from pelutils.logging import log`. Only `__version__` is exported at
|
|
51
|
+
> the top level. See the [docs](https://pelutils.readthedocs.io) for the full API.
|
|
52
|
+
|
|
53
|
+
## Logging
|
|
54
|
+
|
|
55
|
+
Python's built-in `logging` is powerful but fiddly to set up, and a bare `print` gives
|
|
56
|
+
you no importance levels, no timestamps, and nothing on disk to look at afterwards. This logger hits
|
|
57
|
+
the sweet spot: one `configure` call and you get colour-coded, timestamped output to both
|
|
58
|
+
the console and a log file, with severity levels, log rotation, one-line exception logging,
|
|
59
|
+
and multiprocessing-safe collection.
|
|
60
|
+
|
|
61
|
+
```py
|
|
62
|
+
from pelutils.logging import log, LogLevels
|
|
63
|
+
|
|
64
|
+
# Set up the logger by giving it the file to write to
|
|
65
|
+
# Omit the path to only print, never write a file
|
|
66
|
+
log.configure("train.log")
|
|
67
|
+
|
|
68
|
+
log.section("Training run") # Highlighted section header
|
|
69
|
+
log(f"Loaded {len(dataset):,} samples") # Logs at INFO level
|
|
70
|
+
for epoch in range(epochs):
|
|
71
|
+
loss = train_one_epoch()
|
|
72
|
+
log.debug(f"Epoch {epoch}: loss {loss:.4f}") # Logs at DEBUG level - by default, only saved to the log file, but not printed to the console
|
|
73
|
+
if loss > 1e3:
|
|
74
|
+
log.warning("Loss is diverging") # Logs at WARNING level
|
|
75
|
+
log.debug("Final weights", model.state_dict().keys())
|
|
76
|
+
|
|
77
|
+
# If an exeption occurs in a piece of code, log it with its full (chained) stacktrace, then re-raise
|
|
78
|
+
with log.log_errors:
|
|
79
|
+
save_checkpoint(model)
|
|
80
|
+
|
|
81
|
+
# Temporarily change or silence the log level
|
|
82
|
+
with log.level(LogLevels.ERROR):
|
|
83
|
+
log.warning("Suppressed")
|
|
84
|
+
|
|
85
|
+
# Rotate the log file by time or size
|
|
86
|
+
log.configure("train.log", rotation="day") # or "1 GB", "hour", ...
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
When using multiprocessing, wrap a worker in `with log.collect:` so its lines are
|
|
90
|
+
written together instead of interleaving with other processes. See the
|
|
91
|
+
[logging docs](https://pelutils.readthedocs.io/en/latest/api/pelutils.logging.html) for input helpers, multiple loggers,
|
|
92
|
+
and more.
|
|
93
|
+
|
|
94
|
+
## Timing and profiling
|
|
95
|
+
|
|
96
|
+
When you want to know where a script spends its time, `cProfile` gives you a wall of
|
|
97
|
+
function-level numbers, and manual `time.perf_counter()` calls quickly turn into
|
|
98
|
+
bookkeeping. `TickTock` sits in between: wrap the sections *you* care about in named,
|
|
99
|
+
nestable context managers and print a readable table of totals, hit counts, averages,
|
|
100
|
+
and each section's share of its parent's time. The per-profile overhead is tiny, so it
|
|
101
|
+
happily lives inside hot loops and long-running jobs.
|
|
102
|
+
|
|
103
|
+
```py
|
|
104
|
+
from pelutils.ticktock import TT
|
|
105
|
+
|
|
106
|
+
# Time a single block, Matlab style
|
|
107
|
+
TT.tick()
|
|
108
|
+
model = train_model(data)
|
|
109
|
+
print(f"Training took {TT.tock():.1f} s")
|
|
110
|
+
|
|
111
|
+
# Profile named sections across a loop, nesting them however you like
|
|
112
|
+
for image in images:
|
|
113
|
+
with TT.profile("Process image"):
|
|
114
|
+
with TT.profile("Load"):
|
|
115
|
+
img = load(image)
|
|
116
|
+
with TT.profile("Resize"):
|
|
117
|
+
img = resize(img, (224, 224))
|
|
118
|
+
with TT.profile("Inference"):
|
|
119
|
+
predict(model, img)
|
|
120
|
+
|
|
121
|
+
# Print a table of hits, total time, and average time for each section
|
|
122
|
+
print(TT)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
`with TT.profile("name", hits=n):` records `n` hits at once, which is handy for very
|
|
126
|
+
tight loops or for a block that processes `n` items in parallel. `TT.do_at_interval(...)`
|
|
127
|
+
turns the same instance into a throttle for periodic tasks. The default `TT` is a
|
|
128
|
+
shared instance; construct your own with `TickTock()` when you need isolation — most
|
|
129
|
+
importantly one per thread, as profiling is not thread-safe.
|
|
130
|
+
|
|
131
|
+
## Serialisation
|
|
132
|
+
|
|
133
|
+
`UniversalJsonModel` extends `pydantic.BaseModel` with `save`/`load` methods and can
|
|
134
|
+
serialise attributes that pydantic cannot — numpy arrays, tensors, and arbitrary
|
|
135
|
+
objects are base64-pickled inline, everything else stays plain, human-readable JSON.
|
|
136
|
+
Long lists are wrapped to the line-length limit instead of one element per line.
|
|
137
|
+
|
|
138
|
+
```py
|
|
139
|
+
import numpy as np
|
|
140
|
+
from pydantic import BaseModel
|
|
141
|
+
from pelutils.serialization import UniversalJsonModel
|
|
142
|
+
from pelutils.types import FloatArray
|
|
143
|
+
|
|
144
|
+
class Nested(BaseModel):
|
|
145
|
+
label: str
|
|
146
|
+
|
|
147
|
+
class Result(UniversalJsonModel):
|
|
148
|
+
accuracy: float
|
|
149
|
+
predictions: FloatArray # numpy arrays are handled automatically
|
|
150
|
+
meta: Nested
|
|
151
|
+
|
|
152
|
+
result = Result(
|
|
153
|
+
accuracy=0.97,
|
|
154
|
+
predictions=np.arange(5, dtype=np.float16),
|
|
155
|
+
meta=Nested(label="run-1"),
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
result.save("results/run-1.json")
|
|
159
|
+
result = Result.load("results/run-1.json")
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Use `to_json_dict()` / `from_json_dict(...)` to convert to and from a plain dict
|
|
163
|
+
without touching the filesystem — useful for nesting inside other structures. The
|
|
164
|
+
`pretty_json` helper function is also available on its own. The `serialization`
|
|
165
|
+
module also includes JSONL read/write helpers (`jsonl_dump`, `jsonl_load`, ...)
|
|
166
|
+
with largely the same interface as is provided by the built-in `json` module.
|
|
167
|
+
|
|
168
|
+
## Config and command-line argument parsing
|
|
169
|
+
|
|
170
|
+
`JobParser` combines typed command-line options with INI config files. CLI values
|
|
171
|
+
override config values, which override defaults. Declare `RequiredArg` for values
|
|
172
|
+
every job must provide, `OptionalArg` for values with defaults, and `Flag` for
|
|
173
|
+
booleans. Names are `--kebab-case` on the command line and `snake_case` attributes on
|
|
174
|
+
the resulting job.
|
|
175
|
+
|
|
176
|
+
```py
|
|
177
|
+
from pathlib import Path
|
|
178
|
+
from pelutils.job_parser import Flag, JobParser, OptionalArg, RequiredArg
|
|
179
|
+
|
|
180
|
+
parser = JobParser(
|
|
181
|
+
RequiredArg("data-path", help="Training data directory"),
|
|
182
|
+
OptionalArg("learning-rate", default=1e-4, type=float, help="Optimizer learning rate"),
|
|
183
|
+
Flag("fp16", help="Use mixed precision"),
|
|
184
|
+
multiple_jobs=True,
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
for job in parser.parse_jobs():
|
|
188
|
+
print(job.name, job.data_path, job.learning_rate, job.fp16)
|
|
189
|
+
job.write_documentation(Path("runs") / job.name / "arguments.ini")
|
|
190
|
+
# ... run your application with the resolved job values
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
A single config file can define several named jobs (with a shared `[DEFAULT]`
|
|
194
|
+
section), and one CLI override applies to all of them:
|
|
195
|
+
|
|
196
|
+
```console
|
|
197
|
+
python main.py --config-file config.ini --learning-rate 5e-5
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
For a single job, drop `multiple_jobs=True` and call `parse_job()` instead. A config
|
|
201
|
+
path can target one section directly, e.g. `--config-file config.ini:low-lr`. See the
|
|
202
|
+
[job parser docs](https://pelutils.readthedocs.io/en/latest/api/pelutils.job_parser.html) for auto-documentation details.
|
|
203
|
+
|
|
204
|
+
## Fast `unique`
|
|
205
|
+
|
|
206
|
+
A linear-time alternative to `numpy.unique`, significantly faster on large arrays. It
|
|
207
|
+
also accepts torch tensors and pandas series. The returned elements are unsorted.
|
|
208
|
+
|
|
209
|
+
```py
|
|
210
|
+
import numpy as np
|
|
211
|
+
from pelutils.misc import unique
|
|
212
|
+
|
|
213
|
+
x = np.random.randint(0, 100, size=10_000_000)
|
|
214
|
+
values = unique(x)
|
|
215
|
+
values, index, inverse, counts = unique(
|
|
216
|
+
x, return_index=True, return_inverse=True, return_counts=True,
|
|
217
|
+
)
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
## Data science
|
|
221
|
+
|
|
222
|
+
### Statistics
|
|
223
|
+
|
|
224
|
+
Common statistical helpers, plus wrappers around scipy distributions reparametrised
|
|
225
|
+
as in Jim Pitman's *Probability* (rather than scipy's `loc`/`scale`, which are
|
|
226
|
+
unintuitive for many distributions).
|
|
227
|
+
|
|
228
|
+
```py
|
|
229
|
+
from pelutils.stats import z_score
|
|
230
|
+
from pelutils.stats import expon
|
|
231
|
+
|
|
232
|
+
# 95 % confidence interval half-width for a standard normal (defaults give ~1.96)
|
|
233
|
+
half_width = std * z_score()
|
|
234
|
+
|
|
235
|
+
# One-sided z value for an Exponential(λ=2) at the 1 % significance level
|
|
236
|
+
zval = z_score(alpha=0.01, two_sided=False, distribution=expon(lambda_=2))
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
### Plotting
|
|
240
|
+
|
|
241
|
+
The `Figure` context manager fixes common `matplotlib` annoyances — sensible default
|
|
242
|
+
figure and font sizes, easy styling — and saves and closes the figure for you while
|
|
243
|
+
restoring `rcParams` afterwards.
|
|
244
|
+
|
|
245
|
+
```py
|
|
246
|
+
import matplotlib.pyplot as plt
|
|
247
|
+
from pelutils.plots import Figure, histogram, normal_binning
|
|
248
|
+
|
|
249
|
+
with Figure("plot.png", figsize=(20, 10), fontsize=20):
|
|
250
|
+
plt.scatter(x, y, label="Data")
|
|
251
|
+
plt.grid()
|
|
252
|
+
plt.title("Very nice plot")
|
|
253
|
+
# Saved to plot.png and closed here
|
|
254
|
+
|
|
255
|
+
# histogram returns x and y coordinates ready for unpacking
|
|
256
|
+
plt.plot(*histogram(data, binning_fn=normal_binning))
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
Three binning functions are provided — `linear_binning`, `log_binning`, and
|
|
260
|
+
`normal_binning` (more resolution near the centre of roughly-normal data) — and custom
|
|
261
|
+
binning functions are supported. See the
|
|
262
|
+
[plotting docs](https://pelutils.readthedocs.io/en/latest/api/pelutils.plots.html).
|
|
263
|
+
|
|
264
|
+
### Numpy type aliases
|
|
265
|
+
|
|
266
|
+
Type aliases so you (and your type checker) do not have to track array dtypes by hand.
|
|
267
|
+
|
|
268
|
+
```py
|
|
269
|
+
from pelutils.types import FloatArray, IntArray, BoolArray
|
|
270
|
+
|
|
271
|
+
def process(features: FloatArray, labels: IntArray, mask: BoolArray): ...
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
## Also included
|
|
275
|
+
|
|
276
|
+
- `pelutils.misc.Table` — build aligned text tables which can also be easily export to LaTeX with `Table.to_latex()`.
|
|
277
|
+
- `pelutils.misc.hardware_info` / `OS` — describe the machine the code runs on.
|
|
278
|
+
- `pelutils.misc.git_repo_info` — the repo and commit the code is executing in.
|
|
279
|
+
- Assorted file and dict helpers (`reverse_line_iterator`, `except_keys`, ...).
|
|
280
|
+
- `pelutils.tests` — pytest helpers: a `UnitTestCollection` base class with a managed temp directory, and a `restore_argv` decorator.
|
|
281
|
+
|
|
282
|
+
## Supported platforms
|
|
283
|
+
|
|
284
|
+
Precompiled wheels are provided for most common platforms (not 32-bit systems). If no
|
|
285
|
+
wheel matches, `pip` builds from source which requires `<Python.h>` — install it with
|
|
286
|
+
`sudo apt install python3-dev` (Ubuntu) or `sudo dnf install python3-devel` (Fedora).
|
|
287
|
+
|
|
288
|
+
## Updating and releasing
|
|
289
|
+
|
|
290
|
+
`pelutils` uses the `master` branch as a stable development branch.
|
|
291
|
+
The `release` branch contains the latest version on PyPI.
|
|
292
|
+
New features should be made in feature branches from `master` that can be merged into `master` once ready.
|
|
293
|
+
When a new release is ready, update `pelutils/__version__.py` and rebase `master` onto `release`.
|
|
294
|
+
Then push a new tag from `release` named `vX.Y.Z`.
|
|
295
|
+
|
|
296
|
+
When new code is merged into `master`, a number of checks are run.
|
|
297
|
+
These can be tested locally with the following commands.
|
|
298
|
+
```sh
|
|
299
|
+
# Linting and formatting
|
|
300
|
+
ruff format pelutils tests
|
|
301
|
+
ruff check pelutils tests
|
|
302
|
+
# Type checking
|
|
303
|
+
basedpyright pelutils
|
|
304
|
+
# Unit tests
|
|
305
|
+
python -m pytest tests --cov pelutils
|
|
306
|
+
# Build docs
|
|
307
|
+
# Once build, open docs/build/html/index.html in your browser to see them
|
|
308
|
+
make -C docs html
|
|
309
|
+
```
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
# Do not put anything else in this file
|
|
2
|
-
__version__ = "
|
|
2
|
+
__version__ = "4.0.0a2"
|
|
@@ -5,23 +5,20 @@ from __future__ import annotations
|
|
|
5
5
|
from typing import cast
|
|
6
6
|
|
|
7
7
|
import numpy as np
|
|
8
|
+
import numpy.typing as npt
|
|
8
9
|
|
|
10
|
+
from pelutils.misc._conditional_import import import_torch
|
|
9
11
|
from pelutils.types import AnyArray
|
|
10
12
|
|
|
11
|
-
|
|
12
|
-
import torch
|
|
13
|
-
|
|
14
|
-
_has_torch = True
|
|
15
|
-
except ModuleNotFoundError:
|
|
16
|
-
_has_torch = False
|
|
13
|
+
torch = import_torch()
|
|
17
14
|
|
|
18
15
|
|
|
19
16
|
# Data pointer, num dims, dimensions pointer, strides pointer
|
|
20
17
|
ArrayArgs = tuple[int, int, int, int]
|
|
21
18
|
|
|
22
19
|
|
|
23
|
-
def get_array_c_args(arr:
|
|
24
|
-
if
|
|
20
|
+
def get_array_c_args(arr: npt.ArrayLike) -> ArrayArgs:
|
|
21
|
+
if torch is not None and isinstance(arr, torch.Tensor):
|
|
25
22
|
arr = arr.numpy()
|
|
26
23
|
# Tell the type checker that arr for sure is AnyArray
|
|
27
24
|
# Not applied directly to arr.numpy() above as torch is possibly unbound, making the poor checker confused
|