pelutils 3.9.0__tar.gz → 4.0.0a2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. pelutils-4.0.0a2/PKG-INFO +333 -0
  2. pelutils-4.0.0a2/README.md +309 -0
  3. pelutils-4.0.0a2/pelutils/__init__.py +3 -0
  4. {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/__version__.py +1 -1
  5. {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/_c/__init__.py +5 -8
  6. {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/_entry_points/linecounter.py +6 -8
  7. pelutils-4.0.0a2/pelutils/job_parser/__init__.py +61 -0
  8. pelutils-4.0.0a2/pelutils/job_parser/_job_parser.py +364 -0
  9. pelutils-4.0.0a2/pelutils/job_parser/_structs.py +260 -0
  10. pelutils-4.0.0a2/pelutils/logging/__init__.py +39 -0
  11. pelutils-3.9.0/pelutils/logging/__init__.py → pelutils-4.0.0a2/pelutils/logging/_logger.py +24 -54
  12. pelutils-4.0.0a2/pelutils/logging/_rich_string.py +49 -0
  13. {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/logging/_rotate.py +0 -2
  14. {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/logging/_utils.py +16 -16
  15. pelutils-4.0.0a2/pelutils/misc/__init__.py +51 -0
  16. pelutils-3.9.0/pelutils/ds/__init__.py → pelutils-4.0.0a2/pelutils/misc/_array.py +38 -30
  17. pelutils-4.0.0a2/pelutils/misc/_conditional_import.py +31 -0
  18. pelutils-4.0.0a2/pelutils/misc/_files.py +60 -0
  19. pelutils-4.0.0a2/pelutils/misc/_git.py +31 -0
  20. pelutils-4.0.0a2/pelutils/misc/_misc.py +13 -0
  21. pelutils-4.0.0a2/pelutils/misc/_platform.py +74 -0
  22. pelutils-4.0.0a2/pelutils/misc/_table.py +109 -0
  23. pelutils-4.0.0a2/pelutils/plots/__init__.py +50 -0
  24. pelutils-4.0.0a2/pelutils/plots/_figure.py +116 -0
  25. pelutils-4.0.0a2/pelutils/plots/_histogram.py +50 -0
  26. pelutils-4.0.0a2/pelutils/plots/_utils.py +36 -0
  27. pelutils-4.0.0a2/pelutils/serialization/__init__.py +43 -0
  28. pelutils-4.0.0a2/pelutils/serialization/_jsonl.py +38 -0
  29. {pelutils-3.9.0/pelutils/datastorage2 → pelutils-4.0.0a2/pelutils/serialization}/_pretty_json.py +48 -17
  30. pelutils-4.0.0a2/pelutils/serialization/_universal_json_model.py +53 -0
  31. pelutils-4.0.0a2/pelutils/stats/__init__.py +53 -0
  32. pelutils-3.9.0/pelutils/ds/distributions.py → pelutils-4.0.0a2/pelutils/stats/_distributions.py +0 -2
  33. pelutils-4.0.0a2/pelutils/stats/_stats.py +26 -0
  34. pelutils-4.0.0a2/pelutils/tests/__init__.py +32 -0
  35. pelutils-4.0.0a2/pelutils/tests/_tests.py +88 -0
  36. pelutils-4.0.0a2/pelutils/ticktock/__init__.py +42 -0
  37. pelutils-4.0.0a2/pelutils/ticktock/_ticktock.py +360 -0
  38. pelutils-4.0.0a2/pelutils/types/__init__.py +25 -0
  39. pelutils-4.0.0a2/pelutils/types/_types.py +17 -0
  40. pelutils-4.0.0a2/pelutils.egg-info/PKG-INFO +333 -0
  41. pelutils-4.0.0a2/pelutils.egg-info/SOURCES.txt +52 -0
  42. {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils.egg-info/entry_points.txt +0 -1
  43. {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils.egg-info/requires.txt +11 -2
  44. {pelutils-3.9.0 → pelutils-4.0.0a2}/setup.py +12 -10
  45. pelutils-3.9.0/PKG-INFO +0 -389
  46. pelutils-3.9.0/README.md +0 -364
  47. pelutils-3.9.0/pelutils/__init__.py +0 -322
  48. pelutils-3.9.0/pelutils/_utils.py +0 -46
  49. pelutils-3.9.0/pelutils/datastorage.py +0 -151
  50. pelutils-3.9.0/pelutils/datastorage2/__init__.py +0 -74
  51. pelutils-3.9.0/pelutils/ds/plots.py +0 -262
  52. pelutils-3.9.0/pelutils/ds/stats.py +0 -70
  53. pelutils-3.9.0/pelutils/format.py +0 -143
  54. pelutils-3.9.0/pelutils/jsonl.py +0 -40
  55. pelutils-3.9.0/pelutils/parser.py +0 -548
  56. pelutils-3.9.0/pelutils/pretty_json.py +0 -41
  57. pelutils-3.9.0/pelutils/tests.py +0 -108
  58. pelutils-3.9.0/pelutils/ticktock.py +0 -439
  59. pelutils-3.9.0/pelutils/types.py +0 -14
  60. pelutils-3.9.0/pelutils.egg-info/PKG-INFO +0 -389
  61. pelutils-3.9.0/pelutils.egg-info/SOURCES.txt +0 -47
  62. pelutils-3.9.0/tests/test_datastorage.py +0 -84
  63. pelutils-3.9.0/tests/test_format.py +0 -53
  64. pelutils-3.9.0/tests/test_init.py +0 -238
  65. pelutils-3.9.0/tests/test_jsonl.py +0 -44
  66. pelutils-3.9.0/tests/test_parser.py +0 -444
  67. pelutils-3.9.0/tests/test_pretty_json.py +0 -302
  68. pelutils-3.9.0/tests/test_tests.py +0 -23
  69. pelutils-3.9.0/tests/test_ticktock.py +0 -405
  70. pelutils-3.9.0/tests/test_types.py +0 -37
  71. pelutils-3.9.0/tests/test_utils.py +0 -13
  72. {pelutils-3.9.0 → pelutils-4.0.0a2}/MANIFEST.in +0 -0
  73. {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/_c/ds.c +0 -0
  74. {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/_c/hashmap.c/LICENSE +0 -0
  75. {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/_c/hashmap.c/hashmap.c +0 -0
  76. {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/_c/hashmap.c/hashmap.h +0 -0
  77. {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/_entry_points/__init__.py +0 -0
  78. {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils/py.typed +0 -0
  79. {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils.egg-info/dependency_links.txt +0 -0
  80. {pelutils-3.9.0 → pelutils-4.0.0a2}/pelutils.egg-info/top_level.txt +0 -0
  81. {pelutils-3.9.0 → pelutils-4.0.0a2}/setup.cfg +0 -0
@@ -0,0 +1,333 @@
1
+ Metadata-Version: 2.1
2
+ Name: pelutils
3
+ Version: 4.0.0a2
4
+ Summary: The Swiss army knife of Python projects
5
+ Home-page: https://github.com/peleiden/pelutils
6
+ Download-URL: https://pypi.org/project/pelutils/
7
+ Author: Asger Laurits Schultz, Søren Winkel Holm
8
+ Author-email: asger.s@protonmail.com, swholm@protonmail.com
9
+ License: MIT
10
+ Keywords: utility,logger,parser,profiling,plotting
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Operating System :: OS Independent
13
+ Classifier: Programming Language :: Python
14
+ Classifier: Programming Language :: Python :: 3 :: Only
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Programming Language :: Python :: 3.14
19
+ Requires-Python: >=3.11
20
+ Description-Content-Type: text/markdown
21
+ Provides-Extra: dev
22
+ Provides-Extra: docs
23
+ License-File: pelutils/_c/hashmap.c/LICENSE
24
+
25
+ # pelutils
26
+
27
+ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
28
+ [![basedpyright](https://img.shields.io/endpoint?url=https://docs.basedpyright.com/latest/badge.json)](https://docs.basedpyright.com)
29
+ [![checks](https://github.com/peleiden/pelutils/actions/workflows/checks.yml/badge.svg?branch=master)](https://github.com/peleiden/pelutils/actions/workflows/checks.yml)
30
+ [![Coverage Status](https://coveralls.io/repos/github/peleiden/pelutils/badge.svg?branch=master)](https://coveralls.io/github/peleiden/pelutils?branch=master)
31
+ [![PyPi](https://img.shields.io/pypi/v/pelutils.svg)](https://pypi.org/project/pelutils/)
32
+ [![Python versions](https://img.shields.io/pypi/pyversions/pelutils)](https://img.shields.io/pypi/pyversions/pelutils)
33
+ [![image](https://img.shields.io/pypi/l/pelutils.svg)](https://github.com/peleiden/pelutils/blob/master/LICENSE.txt)
34
+ [![readthedocs](https://app.readthedocs.org/projects/pelutils/badge/?version=latest)](https://pelutils.readthedocs.io/en/latest/)
35
+
36
+ **The Swiss army knife of Python projects**
37
+
38
+ Every project, experiment, or one-off script inevitably ends up reinventing much of
39
+ the same plumbing: some way to time a loop, saving and loading data to and from disk
40
+ in a convenient and human-readable manner, a decent logger, parsing a config file, a readable table.
41
+ `pelutils` bundles the good versions of these so you can get straight to the actual
42
+ work. It has no required dependencies beyond the scientific-Python staples, ships
43
+ type hints (including `py.typed`), and is easy to start using.
44
+
45
+ 📖 **Full documentation: [pelutils.readthedocs.io](https://pelutils.readthedocs.io)**
46
+
47
+ ## Highlights
48
+
49
+ - **Logger** — easy-to-use, colourful console output, log files with rotation, automatic
50
+ stacktrace capture, and safe logging from multiple processes.
51
+ - **Timer & profiler** — a Matlab-style `tick`/`tock` timer and a near-zero-overhead
52
+ profiler that prints a readable breakdown of where your time goes.
53
+ - **`UniversalJsonModel`** — a `pydantic.BaseModel` that can save *any* attribute to a
54
+ human-readable JSON file (numpy arrays, tensors, and other unserialisable types are
55
+ pickled transparently) and load it straight back.
56
+ - **`JobParser`** — one parser that unifies command-line arguments and config files,
57
+ with support for running many jobs from a single config and auto-documenting them.
58
+ - **`unique`** — a linear-time drop-in for `numpy.unique`, dramatically faster on
59
+ large arrays (backed by a small C extension).
60
+ - **Data-science helpers** — a `matplotlib` `Figure` context manager with improved default
61
+ settings over `matplotlib`, histogram binning, reparametrised scipy distributions,
62
+ `z_score`, LaTeX-ready tables, and numpy type aliases.
63
+
64
+ ## Installation
65
+
66
+ ```sh
67
+ pip install pelutils
68
+ ```
69
+
70
+ `pelutils` supports Python 3.11+. A small subset of functionality can additionally
71
+ make use of [`PyTorch`](https://pytorch.org), which must be installed separately.
72
+
73
+ > **Importing:** every feature lives in its own submodule and must be imported from
74
+ > there — e.g. `from pelutils.logging import log`. Only `__version__` is exported at
75
+ > the top level. See the [docs](https://pelutils.readthedocs.io) for the full API.
76
+
77
+ ## Logging
78
+
79
+ Python's built-in `logging` is powerful but fiddly to set up, and a bare `print` gives
80
+ you no importance levels, no timestamps, and nothing on disk to look at afterwards. This logger hits
81
+ the sweet spot: one `configure` call and you get colour-coded, timestamped output to both
82
+ the console and a log file, with severity levels, log rotation, one-line exception logging,
83
+ and multiprocessing-safe collection.
84
+
85
+ ```py
86
+ from pelutils.logging import log, LogLevels
87
+
88
+ # Set up the logger by giving it the file to write to
89
+ # Omit the path to only print, never write a file
90
+ log.configure("train.log")
91
+
92
+ log.section("Training run") # Highlighted section header
93
+ log(f"Loaded {len(dataset):,} samples") # Logs at INFO level
94
+ for epoch in range(epochs):
95
+ loss = train_one_epoch()
96
+ log.debug(f"Epoch {epoch}: loss {loss:.4f}") # Logs at DEBUG level - by default, only saved to the log file, but not printed to the console
97
+ if loss > 1e3:
98
+ log.warning("Loss is diverging") # Logs at WARNING level
99
+ log.debug("Final weights", model.state_dict().keys())
100
+
101
+ # If an exeption occurs in a piece of code, log it with its full (chained) stacktrace, then re-raise
102
+ with log.log_errors:
103
+ save_checkpoint(model)
104
+
105
+ # Temporarily change or silence the log level
106
+ with log.level(LogLevels.ERROR):
107
+ log.warning("Suppressed")
108
+
109
+ # Rotate the log file by time or size
110
+ log.configure("train.log", rotation="day") # or "1 GB", "hour", ...
111
+ ```
112
+
113
+ When using multiprocessing, wrap a worker in `with log.collect:` so its lines are
114
+ written together instead of interleaving with other processes. See the
115
+ [logging docs](https://pelutils.readthedocs.io/en/latest/api/pelutils.logging.html) for input helpers, multiple loggers,
116
+ and more.
117
+
118
+ ## Timing and profiling
119
+
120
+ When you want to know where a script spends its time, `cProfile` gives you a wall of
121
+ function-level numbers, and manual `time.perf_counter()` calls quickly turn into
122
+ bookkeeping. `TickTock` sits in between: wrap the sections *you* care about in named,
123
+ nestable context managers and print a readable table of totals, hit counts, averages,
124
+ and each section's share of its parent's time. The per-profile overhead is tiny, so it
125
+ happily lives inside hot loops and long-running jobs.
126
+
127
+ ```py
128
+ from pelutils.ticktock import TT
129
+
130
+ # Time a single block, Matlab style
131
+ TT.tick()
132
+ model = train_model(data)
133
+ print(f"Training took {TT.tock():.1f} s")
134
+
135
+ # Profile named sections across a loop, nesting them however you like
136
+ for image in images:
137
+ with TT.profile("Process image"):
138
+ with TT.profile("Load"):
139
+ img = load(image)
140
+ with TT.profile("Resize"):
141
+ img = resize(img, (224, 224))
142
+ with TT.profile("Inference"):
143
+ predict(model, img)
144
+
145
+ # Print a table of hits, total time, and average time for each section
146
+ print(TT)
147
+ ```
148
+
149
+ `with TT.profile("name", hits=n):` records `n` hits at once, which is handy for very
150
+ tight loops or for a block that processes `n` items in parallel. `TT.do_at_interval(...)`
151
+ turns the same instance into a throttle for periodic tasks. The default `TT` is a
152
+ shared instance; construct your own with `TickTock()` when you need isolation — most
153
+ importantly one per thread, as profiling is not thread-safe.
154
+
155
+ ## Serialisation
156
+
157
+ `UniversalJsonModel` extends `pydantic.BaseModel` with `save`/`load` methods and can
158
+ serialise attributes that pydantic cannot — numpy arrays, tensors, and arbitrary
159
+ objects are base64-pickled inline, everything else stays plain, human-readable JSON.
160
+ Long lists are wrapped to the line-length limit instead of one element per line.
161
+
162
+ ```py
163
+ import numpy as np
164
+ from pydantic import BaseModel
165
+ from pelutils.serialization import UniversalJsonModel
166
+ from pelutils.types import FloatArray
167
+
168
+ class Nested(BaseModel):
169
+ label: str
170
+
171
+ class Result(UniversalJsonModel):
172
+ accuracy: float
173
+ predictions: FloatArray # numpy arrays are handled automatically
174
+ meta: Nested
175
+
176
+ result = Result(
177
+ accuracy=0.97,
178
+ predictions=np.arange(5, dtype=np.float16),
179
+ meta=Nested(label="run-1"),
180
+ )
181
+
182
+ result.save("results/run-1.json")
183
+ result = Result.load("results/run-1.json")
184
+ ```
185
+
186
+ Use `to_json_dict()` / `from_json_dict(...)` to convert to and from a plain dict
187
+ without touching the filesystem — useful for nesting inside other structures. The
188
+ `pretty_json` helper function is also available on its own. The `serialization`
189
+ module also includes JSONL read/write helpers (`jsonl_dump`, `jsonl_load`, ...)
190
+ with largely the same interface as is provided by the built-in `json` module.
191
+
192
+ ## Config and command-line argument parsing
193
+
194
+ `JobParser` combines typed command-line options with INI config files. CLI values
195
+ override config values, which override defaults. Declare `RequiredArg` for values
196
+ every job must provide, `OptionalArg` for values with defaults, and `Flag` for
197
+ booleans. Names are `--kebab-case` on the command line and `snake_case` attributes on
198
+ the resulting job.
199
+
200
+ ```py
201
+ from pathlib import Path
202
+ from pelutils.job_parser import Flag, JobParser, OptionalArg, RequiredArg
203
+
204
+ parser = JobParser(
205
+ RequiredArg("data-path", help="Training data directory"),
206
+ OptionalArg("learning-rate", default=1e-4, type=float, help="Optimizer learning rate"),
207
+ Flag("fp16", help="Use mixed precision"),
208
+ multiple_jobs=True,
209
+ )
210
+
211
+ for job in parser.parse_jobs():
212
+ print(job.name, job.data_path, job.learning_rate, job.fp16)
213
+ job.write_documentation(Path("runs") / job.name / "arguments.ini")
214
+ # ... run your application with the resolved job values
215
+ ```
216
+
217
+ A single config file can define several named jobs (with a shared `[DEFAULT]`
218
+ section), and one CLI override applies to all of them:
219
+
220
+ ```console
221
+ python main.py --config-file config.ini --learning-rate 5e-5
222
+ ```
223
+
224
+ For a single job, drop `multiple_jobs=True` and call `parse_job()` instead. A config
225
+ path can target one section directly, e.g. `--config-file config.ini:low-lr`. See the
226
+ [job parser docs](https://pelutils.readthedocs.io/en/latest/api/pelutils.job_parser.html) for auto-documentation details.
227
+
228
+ ## Fast `unique`
229
+
230
+ A linear-time alternative to `numpy.unique`, significantly faster on large arrays. It
231
+ also accepts torch tensors and pandas series. The returned elements are unsorted.
232
+
233
+ ```py
234
+ import numpy as np
235
+ from pelutils.misc import unique
236
+
237
+ x = np.random.randint(0, 100, size=10_000_000)
238
+ values = unique(x)
239
+ values, index, inverse, counts = unique(
240
+ x, return_index=True, return_inverse=True, return_counts=True,
241
+ )
242
+ ```
243
+
244
+ ## Data science
245
+
246
+ ### Statistics
247
+
248
+ Common statistical helpers, plus wrappers around scipy distributions reparametrised
249
+ as in Jim Pitman's *Probability* (rather than scipy's `loc`/`scale`, which are
250
+ unintuitive for many distributions).
251
+
252
+ ```py
253
+ from pelutils.stats import z_score
254
+ from pelutils.stats import expon
255
+
256
+ # 95 % confidence interval half-width for a standard normal (defaults give ~1.96)
257
+ half_width = std * z_score()
258
+
259
+ # One-sided z value for an Exponential(λ=2) at the 1 % significance level
260
+ zval = z_score(alpha=0.01, two_sided=False, distribution=expon(lambda_=2))
261
+ ```
262
+
263
+ ### Plotting
264
+
265
+ The `Figure` context manager fixes common `matplotlib` annoyances — sensible default
266
+ figure and font sizes, easy styling — and saves and closes the figure for you while
267
+ restoring `rcParams` afterwards.
268
+
269
+ ```py
270
+ import matplotlib.pyplot as plt
271
+ from pelutils.plots import Figure, histogram, normal_binning
272
+
273
+ with Figure("plot.png", figsize=(20, 10), fontsize=20):
274
+ plt.scatter(x, y, label="Data")
275
+ plt.grid()
276
+ plt.title("Very nice plot")
277
+ # Saved to plot.png and closed here
278
+
279
+ # histogram returns x and y coordinates ready for unpacking
280
+ plt.plot(*histogram(data, binning_fn=normal_binning))
281
+ ```
282
+
283
+ Three binning functions are provided — `linear_binning`, `log_binning`, and
284
+ `normal_binning` (more resolution near the centre of roughly-normal data) — and custom
285
+ binning functions are supported. See the
286
+ [plotting docs](https://pelutils.readthedocs.io/en/latest/api/pelutils.plots.html).
287
+
288
+ ### Numpy type aliases
289
+
290
+ Type aliases so you (and your type checker) do not have to track array dtypes by hand.
291
+
292
+ ```py
293
+ from pelutils.types import FloatArray, IntArray, BoolArray
294
+
295
+ def process(features: FloatArray, labels: IntArray, mask: BoolArray): ...
296
+ ```
297
+
298
+ ## Also included
299
+
300
+ - `pelutils.misc.Table` — build aligned text tables which can also be easily export to LaTeX with `Table.to_latex()`.
301
+ - `pelutils.misc.hardware_info` / `OS` — describe the machine the code runs on.
302
+ - `pelutils.misc.git_repo_info` — the repo and commit the code is executing in.
303
+ - Assorted file and dict helpers (`reverse_line_iterator`, `except_keys`, ...).
304
+ - `pelutils.tests` — pytest helpers: a `UnitTestCollection` base class with a managed temp directory, and a `restore_argv` decorator.
305
+
306
+ ## Supported platforms
307
+
308
+ Precompiled wheels are provided for most common platforms (not 32-bit systems). If no
309
+ wheel matches, `pip` builds from source which requires `<Python.h>` — install it with
310
+ `sudo apt install python3-dev` (Ubuntu) or `sudo dnf install python3-devel` (Fedora).
311
+
312
+ ## Updating and releasing
313
+
314
+ `pelutils` uses the `master` branch as a stable development branch.
315
+ The `release` branch contains the latest version on PyPI.
316
+ New features should be made in feature branches from `master` that can be merged into `master` once ready.
317
+ When a new release is ready, update `pelutils/__version__.py` and rebase `master` onto `release`.
318
+ Then push a new tag from `release` named `vX.Y.Z`.
319
+
320
+ When new code is merged into `master`, a number of checks are run.
321
+ These can be tested locally with the following commands.
322
+ ```sh
323
+ # Linting and formatting
324
+ ruff format pelutils tests
325
+ ruff check pelutils tests
326
+ # Type checking
327
+ basedpyright pelutils
328
+ # Unit tests
329
+ python -m pytest tests --cov pelutils
330
+ # Build docs
331
+ # Once build, open docs/build/html/index.html in your browser to see them
332
+ make -C docs html
333
+ ```
@@ -0,0 +1,309 @@
1
+ # pelutils
2
+
3
+ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
4
+ [![basedpyright](https://img.shields.io/endpoint?url=https://docs.basedpyright.com/latest/badge.json)](https://docs.basedpyright.com)
5
+ [![checks](https://github.com/peleiden/pelutils/actions/workflows/checks.yml/badge.svg?branch=master)](https://github.com/peleiden/pelutils/actions/workflows/checks.yml)
6
+ [![Coverage Status](https://coveralls.io/repos/github/peleiden/pelutils/badge.svg?branch=master)](https://coveralls.io/github/peleiden/pelutils?branch=master)
7
+ [![PyPi](https://img.shields.io/pypi/v/pelutils.svg)](https://pypi.org/project/pelutils/)
8
+ [![Python versions](https://img.shields.io/pypi/pyversions/pelutils)](https://img.shields.io/pypi/pyversions/pelutils)
9
+ [![image](https://img.shields.io/pypi/l/pelutils.svg)](https://github.com/peleiden/pelutils/blob/master/LICENSE.txt)
10
+ [![readthedocs](https://app.readthedocs.org/projects/pelutils/badge/?version=latest)](https://pelutils.readthedocs.io/en/latest/)
11
+
12
+ **The Swiss army knife of Python projects**
13
+
14
+ Every project, experiment, or one-off script inevitably ends up reinventing much of
15
+ the same plumbing: some way to time a loop, saving and loading data to and from disk
16
+ in a convenient and human-readable manner, a decent logger, parsing a config file, a readable table.
17
+ `pelutils` bundles the good versions of these so you can get straight to the actual
18
+ work. It has no required dependencies beyond the scientific-Python staples, ships
19
+ type hints (including `py.typed`), and is easy to start using.
20
+
21
+ 📖 **Full documentation: [pelutils.readthedocs.io](https://pelutils.readthedocs.io)**
22
+
23
+ ## Highlights
24
+
25
+ - **Logger** — easy-to-use, colourful console output, log files with rotation, automatic
26
+ stacktrace capture, and safe logging from multiple processes.
27
+ - **Timer & profiler** — a Matlab-style `tick`/`tock` timer and a near-zero-overhead
28
+ profiler that prints a readable breakdown of where your time goes.
29
+ - **`UniversalJsonModel`** — a `pydantic.BaseModel` that can save *any* attribute to a
30
+ human-readable JSON file (numpy arrays, tensors, and other unserialisable types are
31
+ pickled transparently) and load it straight back.
32
+ - **`JobParser`** — one parser that unifies command-line arguments and config files,
33
+ with support for running many jobs from a single config and auto-documenting them.
34
+ - **`unique`** — a linear-time drop-in for `numpy.unique`, dramatically faster on
35
+ large arrays (backed by a small C extension).
36
+ - **Data-science helpers** — a `matplotlib` `Figure` context manager with improved default
37
+ settings over `matplotlib`, histogram binning, reparametrised scipy distributions,
38
+ `z_score`, LaTeX-ready tables, and numpy type aliases.
39
+
40
+ ## Installation
41
+
42
+ ```sh
43
+ pip install pelutils
44
+ ```
45
+
46
+ `pelutils` supports Python 3.11+. A small subset of functionality can additionally
47
+ make use of [`PyTorch`](https://pytorch.org), which must be installed separately.
48
+
49
+ > **Importing:** every feature lives in its own submodule and must be imported from
50
+ > there — e.g. `from pelutils.logging import log`. Only `__version__` is exported at
51
+ > the top level. See the [docs](https://pelutils.readthedocs.io) for the full API.
52
+
53
+ ## Logging
54
+
55
+ Python's built-in `logging` is powerful but fiddly to set up, and a bare `print` gives
56
+ you no importance levels, no timestamps, and nothing on disk to look at afterwards. This logger hits
57
+ the sweet spot: one `configure` call and you get colour-coded, timestamped output to both
58
+ the console and a log file, with severity levels, log rotation, one-line exception logging,
59
+ and multiprocessing-safe collection.
60
+
61
+ ```py
62
+ from pelutils.logging import log, LogLevels
63
+
64
+ # Set up the logger by giving it the file to write to
65
+ # Omit the path to only print, never write a file
66
+ log.configure("train.log")
67
+
68
+ log.section("Training run") # Highlighted section header
69
+ log(f"Loaded {len(dataset):,} samples") # Logs at INFO level
70
+ for epoch in range(epochs):
71
+ loss = train_one_epoch()
72
+ log.debug(f"Epoch {epoch}: loss {loss:.4f}") # Logs at DEBUG level - by default, only saved to the log file, but not printed to the console
73
+ if loss > 1e3:
74
+ log.warning("Loss is diverging") # Logs at WARNING level
75
+ log.debug("Final weights", model.state_dict().keys())
76
+
77
+ # If an exeption occurs in a piece of code, log it with its full (chained) stacktrace, then re-raise
78
+ with log.log_errors:
79
+ save_checkpoint(model)
80
+
81
+ # Temporarily change or silence the log level
82
+ with log.level(LogLevels.ERROR):
83
+ log.warning("Suppressed")
84
+
85
+ # Rotate the log file by time or size
86
+ log.configure("train.log", rotation="day") # or "1 GB", "hour", ...
87
+ ```
88
+
89
+ When using multiprocessing, wrap a worker in `with log.collect:` so its lines are
90
+ written together instead of interleaving with other processes. See the
91
+ [logging docs](https://pelutils.readthedocs.io/en/latest/api/pelutils.logging.html) for input helpers, multiple loggers,
92
+ and more.
93
+
94
+ ## Timing and profiling
95
+
96
+ When you want to know where a script spends its time, `cProfile` gives you a wall of
97
+ function-level numbers, and manual `time.perf_counter()` calls quickly turn into
98
+ bookkeeping. `TickTock` sits in between: wrap the sections *you* care about in named,
99
+ nestable context managers and print a readable table of totals, hit counts, averages,
100
+ and each section's share of its parent's time. The per-profile overhead is tiny, so it
101
+ happily lives inside hot loops and long-running jobs.
102
+
103
+ ```py
104
+ from pelutils.ticktock import TT
105
+
106
+ # Time a single block, Matlab style
107
+ TT.tick()
108
+ model = train_model(data)
109
+ print(f"Training took {TT.tock():.1f} s")
110
+
111
+ # Profile named sections across a loop, nesting them however you like
112
+ for image in images:
113
+ with TT.profile("Process image"):
114
+ with TT.profile("Load"):
115
+ img = load(image)
116
+ with TT.profile("Resize"):
117
+ img = resize(img, (224, 224))
118
+ with TT.profile("Inference"):
119
+ predict(model, img)
120
+
121
+ # Print a table of hits, total time, and average time for each section
122
+ print(TT)
123
+ ```
124
+
125
+ `with TT.profile("name", hits=n):` records `n` hits at once, which is handy for very
126
+ tight loops or for a block that processes `n` items in parallel. `TT.do_at_interval(...)`
127
+ turns the same instance into a throttle for periodic tasks. The default `TT` is a
128
+ shared instance; construct your own with `TickTock()` when you need isolation — most
129
+ importantly one per thread, as profiling is not thread-safe.
130
+
131
+ ## Serialisation
132
+
133
+ `UniversalJsonModel` extends `pydantic.BaseModel` with `save`/`load` methods and can
134
+ serialise attributes that pydantic cannot — numpy arrays, tensors, and arbitrary
135
+ objects are base64-pickled inline, everything else stays plain, human-readable JSON.
136
+ Long lists are wrapped to the line-length limit instead of one element per line.
137
+
138
+ ```py
139
+ import numpy as np
140
+ from pydantic import BaseModel
141
+ from pelutils.serialization import UniversalJsonModel
142
+ from pelutils.types import FloatArray
143
+
144
+ class Nested(BaseModel):
145
+ label: str
146
+
147
+ class Result(UniversalJsonModel):
148
+ accuracy: float
149
+ predictions: FloatArray # numpy arrays are handled automatically
150
+ meta: Nested
151
+
152
+ result = Result(
153
+ accuracy=0.97,
154
+ predictions=np.arange(5, dtype=np.float16),
155
+ meta=Nested(label="run-1"),
156
+ )
157
+
158
+ result.save("results/run-1.json")
159
+ result = Result.load("results/run-1.json")
160
+ ```
161
+
162
+ Use `to_json_dict()` / `from_json_dict(...)` to convert to and from a plain dict
163
+ without touching the filesystem — useful for nesting inside other structures. The
164
+ `pretty_json` helper function is also available on its own. The `serialization`
165
+ module also includes JSONL read/write helpers (`jsonl_dump`, `jsonl_load`, ...)
166
+ with largely the same interface as is provided by the built-in `json` module.
167
+
168
+ ## Config and command-line argument parsing
169
+
170
+ `JobParser` combines typed command-line options with INI config files. CLI values
171
+ override config values, which override defaults. Declare `RequiredArg` for values
172
+ every job must provide, `OptionalArg` for values with defaults, and `Flag` for
173
+ booleans. Names are `--kebab-case` on the command line and `snake_case` attributes on
174
+ the resulting job.
175
+
176
+ ```py
177
+ from pathlib import Path
178
+ from pelutils.job_parser import Flag, JobParser, OptionalArg, RequiredArg
179
+
180
+ parser = JobParser(
181
+ RequiredArg("data-path", help="Training data directory"),
182
+ OptionalArg("learning-rate", default=1e-4, type=float, help="Optimizer learning rate"),
183
+ Flag("fp16", help="Use mixed precision"),
184
+ multiple_jobs=True,
185
+ )
186
+
187
+ for job in parser.parse_jobs():
188
+ print(job.name, job.data_path, job.learning_rate, job.fp16)
189
+ job.write_documentation(Path("runs") / job.name / "arguments.ini")
190
+ # ... run your application with the resolved job values
191
+ ```
192
+
193
+ A single config file can define several named jobs (with a shared `[DEFAULT]`
194
+ section), and one CLI override applies to all of them:
195
+
196
+ ```console
197
+ python main.py --config-file config.ini --learning-rate 5e-5
198
+ ```
199
+
200
+ For a single job, drop `multiple_jobs=True` and call `parse_job()` instead. A config
201
+ path can target one section directly, e.g. `--config-file config.ini:low-lr`. See the
202
+ [job parser docs](https://pelutils.readthedocs.io/en/latest/api/pelutils.job_parser.html) for auto-documentation details.
203
+
204
+ ## Fast `unique`
205
+
206
+ A linear-time alternative to `numpy.unique`, significantly faster on large arrays. It
207
+ also accepts torch tensors and pandas series. The returned elements are unsorted.
208
+
209
+ ```py
210
+ import numpy as np
211
+ from pelutils.misc import unique
212
+
213
+ x = np.random.randint(0, 100, size=10_000_000)
214
+ values = unique(x)
215
+ values, index, inverse, counts = unique(
216
+ x, return_index=True, return_inverse=True, return_counts=True,
217
+ )
218
+ ```
219
+
220
+ ## Data science
221
+
222
+ ### Statistics
223
+
224
+ Common statistical helpers, plus wrappers around scipy distributions reparametrised
225
+ as in Jim Pitman's *Probability* (rather than scipy's `loc`/`scale`, which are
226
+ unintuitive for many distributions).
227
+
228
+ ```py
229
+ from pelutils.stats import z_score
230
+ from pelutils.stats import expon
231
+
232
+ # 95 % confidence interval half-width for a standard normal (defaults give ~1.96)
233
+ half_width = std * z_score()
234
+
235
+ # One-sided z value for an Exponential(λ=2) at the 1 % significance level
236
+ zval = z_score(alpha=0.01, two_sided=False, distribution=expon(lambda_=2))
237
+ ```
238
+
239
+ ### Plotting
240
+
241
+ The `Figure` context manager fixes common `matplotlib` annoyances — sensible default
242
+ figure and font sizes, easy styling — and saves and closes the figure for you while
243
+ restoring `rcParams` afterwards.
244
+
245
+ ```py
246
+ import matplotlib.pyplot as plt
247
+ from pelutils.plots import Figure, histogram, normal_binning
248
+
249
+ with Figure("plot.png", figsize=(20, 10), fontsize=20):
250
+ plt.scatter(x, y, label="Data")
251
+ plt.grid()
252
+ plt.title("Very nice plot")
253
+ # Saved to plot.png and closed here
254
+
255
+ # histogram returns x and y coordinates ready for unpacking
256
+ plt.plot(*histogram(data, binning_fn=normal_binning))
257
+ ```
258
+
259
+ Three binning functions are provided — `linear_binning`, `log_binning`, and
260
+ `normal_binning` (more resolution near the centre of roughly-normal data) — and custom
261
+ binning functions are supported. See the
262
+ [plotting docs](https://pelutils.readthedocs.io/en/latest/api/pelutils.plots.html).
263
+
264
+ ### Numpy type aliases
265
+
266
+ Type aliases so you (and your type checker) do not have to track array dtypes by hand.
267
+
268
+ ```py
269
+ from pelutils.types import FloatArray, IntArray, BoolArray
270
+
271
+ def process(features: FloatArray, labels: IntArray, mask: BoolArray): ...
272
+ ```
273
+
274
+ ## Also included
275
+
276
+ - `pelutils.misc.Table` — build aligned text tables which can also be easily export to LaTeX with `Table.to_latex()`.
277
+ - `pelutils.misc.hardware_info` / `OS` — describe the machine the code runs on.
278
+ - `pelutils.misc.git_repo_info` — the repo and commit the code is executing in.
279
+ - Assorted file and dict helpers (`reverse_line_iterator`, `except_keys`, ...).
280
+ - `pelutils.tests` — pytest helpers: a `UnitTestCollection` base class with a managed temp directory, and a `restore_argv` decorator.
281
+
282
+ ## Supported platforms
283
+
284
+ Precompiled wheels are provided for most common platforms (not 32-bit systems). If no
285
+ wheel matches, `pip` builds from source which requires `<Python.h>` — install it with
286
+ `sudo apt install python3-dev` (Ubuntu) or `sudo dnf install python3-devel` (Fedora).
287
+
288
+ ## Updating and releasing
289
+
290
+ `pelutils` uses the `master` branch as a stable development branch.
291
+ The `release` branch contains the latest version on PyPI.
292
+ New features should be made in feature branches from `master` that can be merged into `master` once ready.
293
+ When a new release is ready, update `pelutils/__version__.py` and rebase `master` onto `release`.
294
+ Then push a new tag from `release` named `vX.Y.Z`.
295
+
296
+ When new code is merged into `master`, a number of checks are run.
297
+ These can be tested locally with the following commands.
298
+ ```sh
299
+ # Linting and formatting
300
+ ruff format pelutils tests
301
+ ruff check pelutils tests
302
+ # Type checking
303
+ basedpyright pelutils
304
+ # Unit tests
305
+ python -m pytest tests --cov pelutils
306
+ # Build docs
307
+ # Once build, open docs/build/html/index.html in your browser to see them
308
+ make -C docs html
309
+ ```
@@ -0,0 +1,3 @@
1
+ from .__version__ import __version__
2
+
3
+ __all__ = ("__version__",)
@@ -1,2 +1,2 @@
1
1
  # Do not put anything else in this file
2
- __version__ = "3.9.0"
2
+ __version__ = "4.0.0a2"
@@ -5,23 +5,20 @@ from __future__ import annotations
5
5
  from typing import cast
6
6
 
7
7
  import numpy as np
8
+ import numpy.typing as npt
8
9
 
10
+ from pelutils.misc._conditional_import import import_torch
9
11
  from pelutils.types import AnyArray
10
12
 
11
- try:
12
- import torch
13
-
14
- _has_torch = True
15
- except ModuleNotFoundError:
16
- _has_torch = False
13
+ torch = import_torch()
17
14
 
18
15
 
19
16
  # Data pointer, num dims, dimensions pointer, strides pointer
20
17
  ArrayArgs = tuple[int, int, int, int]
21
18
 
22
19
 
23
- def get_array_c_args(arr: AnyArray | torch.Tensor) -> ArrayArgs:
24
- if _has_torch and isinstance(arr, torch.Tensor): # pyright: ignore[reportPossiblyUnboundVariable]
20
+ def get_array_c_args(arr: npt.ArrayLike) -> ArrayArgs:
21
+ if torch is not None and isinstance(arr, torch.Tensor):
25
22
  arr = arr.numpy()
26
23
  # Tell the type checker that arr for sure is AnyArray
27
24
  # Not applied directly to arr.numpy() above as torch is possibly unbound, making the poor checker confused