pelutils 3.8.0__tar.gz → 3.8.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pelutils-3.8.0 → pelutils-3.8.2}/PKG-INFO +9 -9
- {pelutils-3.8.0 → pelutils-3.8.2}/README.md +8 -8
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/__version__.py +1 -1
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/datastorage2/__init__.py +5 -3
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/logging/__init__.py +14 -9
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils.egg-info/PKG-INFO +9 -9
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils.egg-info/SOURCES.txt +2 -15
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils.egg-info/top_level.txt +0 -1
- {pelutils-3.8.0 → pelutils-3.8.2}/setup.py +5 -2
- pelutils-3.8.0/tests/_c/__init__.py +0 -0
- pelutils-3.8.0/tests/_c/test_c.py +0 -26
- pelutils-3.8.0/tests/datastorage2/__init__.py +0 -0
- pelutils-3.8.0/tests/datastorage2/test_init.py +0 -81
- pelutils-3.8.0/tests/datastorage2/test_pretty_json.py +0 -92
- pelutils-3.8.0/tests/ds/__init__.py +0 -0
- pelutils-3.8.0/tests/ds/test_distributions.py +0 -147
- pelutils-3.8.0/tests/ds/test_ds.py +0 -110
- pelutils-3.8.0/tests/ds/test_plots.py +0 -235
- pelutils-3.8.0/tests/ds/test_stats.py +0 -45
- pelutils-3.8.0/tests/logging/__init__.py +0 -0
- pelutils-3.8.0/tests/logging/test_logging.py +0 -330
- pelutils-3.8.0/tests/logging/test_rotate.py +0 -184
- {pelutils-3.8.0 → pelutils-3.8.2}/MANIFEST.in +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/__init__.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/_c/__init__.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/_c/ds.c +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/_c/hashmap.c/LICENSE +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/_c/hashmap.c/hashmap.c +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/_c/hashmap.c/hashmap.h +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/_entry_points/__init__.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/_entry_points/linecounter.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/datastorage.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/datastorage2/_pretty_json.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/ds/__init__.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/ds/distributions.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/ds/plots.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/ds/stats.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/format.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/jsonl.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/logging/_rotate.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/logging/_utils.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/parser.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/pretty_json.py +0 -0
- /pelutils-3.8.0/tests/__init__.py → /pelutils-3.8.2/pelutils/py.typed +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/tests.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/ticktock.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/types.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils.egg-info/dependency_links.txt +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils.egg-info/entry_points.txt +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/pelutils.egg-info/requires.txt +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/setup.cfg +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_datastorage.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_format.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_init.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_jsonl.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_parser.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_pretty_json.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_tests.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_ticktock.py +0 -0
- {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_types.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: pelutils
|
|
3
|
-
Version: 3.8.
|
|
3
|
+
Version: 3.8.2
|
|
4
4
|
Summary: The Swiss army knife of Python projects.
|
|
5
5
|
Home-page: https://github.com/peleiden/pelutils
|
|
6
6
|
Download-URL: https://pypi.org/project/pelutils/
|
|
@@ -36,14 +36,14 @@ License-File: pelutils/_c/hashmap.c/LICENSE
|
|
|
36
36
|
The Swiss army knife of Python projects.
|
|
37
37
|
|
|
38
38
|
- A simple and powerful logger with colourful printing, stacktraces, and log file rotation.
|
|
39
|
-
-
|
|
40
|
-
- A timer inspired by Matlab's `tic` and `toc`.
|
|
41
|
-
-
|
|
42
|
-
- An extension to
|
|
39
|
+
- Argument parser which supports both command line arguments and config files and has baked-in auto-documentation.
|
|
40
|
+
- A simple timer inspired by Matlab's `tic` and `toc`.
|
|
41
|
+
- Easy-to-use, near-zero cost performance profiler.
|
|
42
|
+
- An extension to `pydantic.BaseModel` with support for saving any data structure to, and loading from, a human-readable JSON file.
|
|
43
43
|
- Table formatting with built-in LaTeX support.
|
|
44
44
|
- Miscellaneous standalone functions - see `pelutils/__init__.py`.
|
|
45
45
|
- Data-science submodule with extra utilities for statistics, plotting with `matplotlib`, and machine learning using `PyTorch`.
|
|
46
|
-
- `unique` function in the style of `numpy.unique` which runs in linear time, making it significantly.
|
|
46
|
+
- `unique` function in the style of `numpy.unique` which runs in linear time, making it significantly faster for large arrays.
|
|
47
47
|
|
|
48
48
|
`pelutils` supports Python 3.9+.
|
|
49
49
|
|
|
@@ -121,8 +121,6 @@ based = BasedClass(
|
|
|
121
121
|
baseder=BasederClass(based_string="Hello there")
|
|
122
122
|
)
|
|
123
123
|
# Save it to a file
|
|
124
|
-
based.save(".")
|
|
125
|
-
# Save it to a file
|
|
126
124
|
based.save("directory/to/save/in")
|
|
127
125
|
|
|
128
126
|
# Load it again
|
|
@@ -380,7 +378,9 @@ Examples of all the plotting utilities are shown in the `examples` directory.
|
|
|
380
378
|
|
|
381
379
|
Precompiled wheels are provided for most common platforms.
|
|
382
380
|
Notably, they are not provided for 32-bit systems.
|
|
383
|
-
If no wheel is provided, `pip` should attempt a source install.
|
|
381
|
+
If no wheel is provided, `pip` should attempt a source install - this requires `<Python.h>` to be available.
|
|
382
|
+
On Ubuntu, this can be installed with `sudo apt install python3-dev`, and on Fedora, it is installed with `sudo dnf install python3-devel`.
|
|
383
|
+
|
|
384
384
|
If all else fails, it is possible to install from source by pointing `pip` to Github directly:
|
|
385
385
|
```
|
|
386
386
|
pip install git+https://github.com/peleiden/pelutils.git@release#egg=pelutils
|
|
@@ -11,14 +11,14 @@
|
|
|
11
11
|
The Swiss army knife of Python projects.
|
|
12
12
|
|
|
13
13
|
- A simple and powerful logger with colourful printing, stacktraces, and log file rotation.
|
|
14
|
-
-
|
|
15
|
-
- A timer inspired by Matlab's `tic` and `toc`.
|
|
16
|
-
-
|
|
17
|
-
- An extension to
|
|
14
|
+
- Argument parser which supports both command line arguments and config files and has baked-in auto-documentation.
|
|
15
|
+
- A simple timer inspired by Matlab's `tic` and `toc`.
|
|
16
|
+
- Easy-to-use, near-zero cost performance profiler.
|
|
17
|
+
- An extension to `pydantic.BaseModel` with support for saving any data structure to, and loading from, a human-readable JSON file.
|
|
18
18
|
- Table formatting with built-in LaTeX support.
|
|
19
19
|
- Miscellaneous standalone functions - see `pelutils/__init__.py`.
|
|
20
20
|
- Data-science submodule with extra utilities for statistics, plotting with `matplotlib`, and machine learning using `PyTorch`.
|
|
21
|
-
- `unique` function in the style of `numpy.unique` which runs in linear time, making it significantly.
|
|
21
|
+
- `unique` function in the style of `numpy.unique` which runs in linear time, making it significantly faster for large arrays.
|
|
22
22
|
|
|
23
23
|
`pelutils` supports Python 3.9+.
|
|
24
24
|
|
|
@@ -96,8 +96,6 @@ based = BasedClass(
|
|
|
96
96
|
baseder=BasederClass(based_string="Hello there")
|
|
97
97
|
)
|
|
98
98
|
# Save it to a file
|
|
99
|
-
based.save(".")
|
|
100
|
-
# Save it to a file
|
|
101
99
|
based.save("directory/to/save/in")
|
|
102
100
|
|
|
103
101
|
# Load it again
|
|
@@ -355,7 +353,9 @@ Examples of all the plotting utilities are shown in the `examples` directory.
|
|
|
355
353
|
|
|
356
354
|
Precompiled wheels are provided for most common platforms.
|
|
357
355
|
Notably, they are not provided for 32-bit systems.
|
|
358
|
-
If no wheel is provided, `pip` should attempt a source install.
|
|
356
|
+
If no wheel is provided, `pip` should attempt a source install - this requires `<Python.h>` to be available.
|
|
357
|
+
On Ubuntu, this can be installed with `sudo apt install python3-dev`, and on Fedora, it is installed with `sudo dnf install python3-devel`.
|
|
358
|
+
|
|
359
359
|
If all else fails, it is possible to install from source by pointing `pip` to Github directly:
|
|
360
360
|
```
|
|
361
361
|
pip install git+https://github.com/peleiden/pelutils.git@release#egg=pelutils
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
# Do not put anything else in this file
|
|
2
|
-
__version__ = "3.8.
|
|
2
|
+
__version__ = "3.8.2"
|
|
@@ -6,13 +6,13 @@ from typing import TypeVar
|
|
|
6
6
|
|
|
7
7
|
from pydantic import BaseModel, ConfigDict
|
|
8
8
|
|
|
9
|
-
from pelutils.datastorage2._pretty_json import _make_json_unsafe, _pretty_json # pyright: ignore[reportPrivateUsage]
|
|
9
|
+
from pelutils.datastorage2._pretty_json import _make_json_unsafe, _pickle_encode, _pretty_json # pyright: ignore[reportPrivateUsage]
|
|
10
10
|
|
|
11
11
|
_T = TypeVar("_T", bound="DataStorage2")
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
class DataStorage2(BaseModel):
|
|
15
|
-
"""Better version of DataStorage based on pydantic.
|
|
15
|
+
"""Better version of DataStorage based on pydantic. It makes any class fully JSON serialisable.
|
|
16
16
|
|
|
17
17
|
Usage is very similar to DataStorage, but it has a few key advantages:
|
|
18
18
|
- Allows saving and loading of nested objects.
|
|
@@ -38,12 +38,14 @@ class DataStorage2(BaseModel):
|
|
|
38
38
|
If save_name is None, the path is <directory>/<class name>.json, otherwise it is <directory>/<save_name>.json.
|
|
39
39
|
"""
|
|
40
40
|
savepath = self._resolve_save_file(directory, filename)
|
|
41
|
-
self_dict = self.model_dump()
|
|
41
|
+
self_dict = self.model_dump(mode="json", fallback=_pickle_encode)
|
|
42
42
|
savepath.write_text(
|
|
43
43
|
_pretty_json(
|
|
44
44
|
self_dict,
|
|
45
45
|
max_line_length=max_line_length,
|
|
46
46
|
indent=indent,
|
|
47
|
+
# Safe should not be needed here, as the fallback function in model_dump should ensure no issues
|
|
48
|
+
# However, it is kept for good measure in case there unforeseen edge cases
|
|
47
49
|
safe=True,
|
|
48
50
|
),
|
|
49
51
|
encoding=encoding,
|
|
@@ -4,7 +4,7 @@ import os
|
|
|
4
4
|
import traceback as tb
|
|
5
5
|
from collections.abc import Generator, Iterable
|
|
6
6
|
from pathlib import Path
|
|
7
|
-
from typing import Optional
|
|
7
|
+
from typing import Any, Optional
|
|
8
8
|
|
|
9
9
|
from pelutils import OS, UnsupportedOS, get_repo, get_timestamp
|
|
10
10
|
from pelutils.format import RichString
|
|
@@ -135,7 +135,7 @@ class Logger:
|
|
|
135
135
|
|
|
136
136
|
def _log(
|
|
137
137
|
self,
|
|
138
|
-
*tolog:
|
|
138
|
+
*tolog: Any, # pyright: ignore[reportExplicitAny]
|
|
139
139
|
level: LogLevels = LogLevels.INFO,
|
|
140
140
|
with_info: bool = True,
|
|
141
141
|
sep: str | None = None,
|
|
@@ -276,34 +276,39 @@ class Logger:
|
|
|
276
276
|
raise UnsupportedOS("Log collecting is not supported on windows")
|
|
277
277
|
return _CollectLogs(self)
|
|
278
278
|
|
|
279
|
-
def section(self, *tolog:
|
|
279
|
+
def section(self, *tolog: Any, with_info: bool = True, sep: str | None = None, with_print: bool | None = None, newline: bool = True): # pyright: ignore[reportExplicitAny]
|
|
280
280
|
"""Log at SECTION level. See .log method for argument descriptions."""
|
|
281
281
|
if newline:
|
|
282
282
|
self._log("")
|
|
283
283
|
self._log(*tolog, with_info=with_info, sep=sep, with_print=with_print, level=LogLevels.SECTION)
|
|
284
284
|
|
|
285
|
-
def critical(self, *tolog:
|
|
285
|
+
def critical(self, *tolog: Any, with_info: bool = True, sep: str | None = None, with_print: bool | None = None): # pyright: ignore[reportExplicitAny]
|
|
286
286
|
"""Log at CRITICAL level. See .log method for argument descriptions."""
|
|
287
287
|
self._log(*tolog, with_info=with_info, sep=sep, with_print=with_print, level=LogLevels.CRITICAL)
|
|
288
288
|
|
|
289
|
-
def error(self, *tolog:
|
|
289
|
+
def error(self, *tolog: Any, with_info: bool = True, sep: str | None = None, with_print: bool | None = None): # pyright: ignore[reportExplicitAny]
|
|
290
290
|
"""Log at ERROR level. See .log method for argument descriptions."""
|
|
291
291
|
self._log(*tolog, with_info=with_info, sep=sep, with_print=with_print, level=LogLevels.ERROR)
|
|
292
292
|
|
|
293
|
-
def warning(self, *tolog:
|
|
293
|
+
def warning(self, *tolog: Any, with_info: bool = True, sep: str | None = None, with_print: bool | None = None): # pyright: ignore[reportExplicitAny]
|
|
294
294
|
"""Log at WARNING level. See .log method for argument descriptions."""
|
|
295
295
|
self._log(*tolog, with_info=with_info, sep=sep, with_print=with_print, level=LogLevels.WARNING)
|
|
296
296
|
|
|
297
|
-
def info(self, *tolog:
|
|
297
|
+
def info(self, *tolog: Any, with_info: bool = True, sep: str | None = None, with_print: bool | None = None): # pyright: ignore[reportExplicitAny]
|
|
298
298
|
"""Log at INFO level. See .log method for argument descriptions."""
|
|
299
299
|
self._log(*tolog, with_info=with_info, sep=sep, with_print=with_print, level=LogLevels.INFO)
|
|
300
300
|
|
|
301
|
-
def debug(self, *tolog:
|
|
301
|
+
def debug(self, *tolog: Any, with_info: bool = True, sep: str | None = None, with_print: bool | None = None): # pyright: ignore[reportExplicitAny]
|
|
302
302
|
"""Log at DEBUG level. See .log method for argument descriptions."""
|
|
303
303
|
self._log(*tolog, with_info=with_info, sep=sep, with_print=with_print, level=LogLevels.DEBUG)
|
|
304
304
|
|
|
305
305
|
def __call__(
|
|
306
|
-
self,
|
|
306
|
+
self,
|
|
307
|
+
*tolog: Any, # pyright: ignore[reportExplicitAny]
|
|
308
|
+
level: LogLevels = LogLevels.INFO,
|
|
309
|
+
with_info: bool = True,
|
|
310
|
+
sep: str | None = None,
|
|
311
|
+
with_print: bool | None = None,
|
|
307
312
|
):
|
|
308
313
|
"""Shorthand for specific logging methods where level is specified as an argument."""
|
|
309
314
|
self._log(*tolog, level=level, with_info=with_info, sep=sep, with_print=with_print)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: pelutils
|
|
3
|
-
Version: 3.8.
|
|
3
|
+
Version: 3.8.2
|
|
4
4
|
Summary: The Swiss army knife of Python projects.
|
|
5
5
|
Home-page: https://github.com/peleiden/pelutils
|
|
6
6
|
Download-URL: https://pypi.org/project/pelutils/
|
|
@@ -36,14 +36,14 @@ License-File: pelutils/_c/hashmap.c/LICENSE
|
|
|
36
36
|
The Swiss army knife of Python projects.
|
|
37
37
|
|
|
38
38
|
- A simple and powerful logger with colourful printing, stacktraces, and log file rotation.
|
|
39
|
-
-
|
|
40
|
-
- A timer inspired by Matlab's `tic` and `toc`.
|
|
41
|
-
-
|
|
42
|
-
- An extension to
|
|
39
|
+
- Argument parser which supports both command line arguments and config files and has baked-in auto-documentation.
|
|
40
|
+
- A simple timer inspired by Matlab's `tic` and `toc`.
|
|
41
|
+
- Easy-to-use, near-zero cost performance profiler.
|
|
42
|
+
- An extension to `pydantic.BaseModel` with support for saving any data structure to, and loading from, a human-readable JSON file.
|
|
43
43
|
- Table formatting with built-in LaTeX support.
|
|
44
44
|
- Miscellaneous standalone functions - see `pelutils/__init__.py`.
|
|
45
45
|
- Data-science submodule with extra utilities for statistics, plotting with `matplotlib`, and machine learning using `PyTorch`.
|
|
46
|
-
- `unique` function in the style of `numpy.unique` which runs in linear time, making it significantly.
|
|
46
|
+
- `unique` function in the style of `numpy.unique` which runs in linear time, making it significantly faster for large arrays.
|
|
47
47
|
|
|
48
48
|
`pelutils` supports Python 3.9+.
|
|
49
49
|
|
|
@@ -121,8 +121,6 @@ based = BasedClass(
|
|
|
121
121
|
baseder=BasederClass(based_string="Hello there")
|
|
122
122
|
)
|
|
123
123
|
# Save it to a file
|
|
124
|
-
based.save(".")
|
|
125
|
-
# Save it to a file
|
|
126
124
|
based.save("directory/to/save/in")
|
|
127
125
|
|
|
128
126
|
# Load it again
|
|
@@ -380,7 +378,9 @@ Examples of all the plotting utilities are shown in the `examples` directory.
|
|
|
380
378
|
|
|
381
379
|
Precompiled wheels are provided for most common platforms.
|
|
382
380
|
Notably, they are not provided for 32-bit systems.
|
|
383
|
-
If no wheel is provided, `pip` should attempt a source install.
|
|
381
|
+
If no wheel is provided, `pip` should attempt a source install - this requires `<Python.h>` to be available.
|
|
382
|
+
On Ubuntu, this can be installed with `sudo apt install python3-dev`, and on Fedora, it is installed with `sudo dnf install python3-devel`.
|
|
383
|
+
|
|
384
384
|
If all else fails, it is possible to install from source by pointing `pip` to Github directly:
|
|
385
385
|
```
|
|
386
386
|
pip install git+https://github.com/peleiden/pelutils.git@release#egg=pelutils
|
|
@@ -8,6 +8,7 @@ pelutils/format.py
|
|
|
8
8
|
pelutils/jsonl.py
|
|
9
9
|
pelutils/parser.py
|
|
10
10
|
pelutils/pretty_json.py
|
|
11
|
+
pelutils/py.typed
|
|
11
12
|
pelutils/tests.py
|
|
12
13
|
pelutils/ticktock.py
|
|
13
14
|
pelutils/types.py
|
|
@@ -33,7 +34,6 @@ pelutils/ds/stats.py
|
|
|
33
34
|
pelutils/logging/__init__.py
|
|
34
35
|
pelutils/logging/_rotate.py
|
|
35
36
|
pelutils/logging/_utils.py
|
|
36
|
-
tests/__init__.py
|
|
37
37
|
tests/test_datastorage.py
|
|
38
38
|
tests/test_format.py
|
|
39
39
|
tests/test_init.py
|
|
@@ -42,17 +42,4 @@ tests/test_parser.py
|
|
|
42
42
|
tests/test_pretty_json.py
|
|
43
43
|
tests/test_tests.py
|
|
44
44
|
tests/test_ticktock.py
|
|
45
|
-
tests/test_types.py
|
|
46
|
-
tests/_c/__init__.py
|
|
47
|
-
tests/_c/test_c.py
|
|
48
|
-
tests/datastorage2/__init__.py
|
|
49
|
-
tests/datastorage2/test_init.py
|
|
50
|
-
tests/datastorage2/test_pretty_json.py
|
|
51
|
-
tests/ds/__init__.py
|
|
52
|
-
tests/ds/test_distributions.py
|
|
53
|
-
tests/ds/test_ds.py
|
|
54
|
-
tests/ds/test_plots.py
|
|
55
|
-
tests/ds/test_stats.py
|
|
56
|
-
tests/logging/__init__.py
|
|
57
|
-
tests/logging/test_logging.py
|
|
58
|
-
tests/logging/test_rotate.py
|
|
45
|
+
tests/test_types.py
|
|
@@ -48,7 +48,9 @@ class CExtension(Extension):
|
|
|
48
48
|
"""See this thread for details: https://stackoverflow.com/a/34830639/13196863."""
|
|
49
49
|
|
|
50
50
|
|
|
51
|
-
class build(build_):
|
|
51
|
+
class build(build_):
|
|
52
|
+
"""Extension of normal build class which handles C extensions."""
|
|
53
|
+
|
|
52
54
|
def build_extension(self, ext): # noqa: D102
|
|
53
55
|
self._ctypes = isinstance(ext, CExtension)
|
|
54
56
|
return super().build_extension(ext)
|
|
@@ -71,7 +73,8 @@ setup_args = dict(
|
|
|
71
73
|
long_description_content_type="text/markdown",
|
|
72
74
|
long_description=README,
|
|
73
75
|
license="MIT",
|
|
74
|
-
packages=find_packages(),
|
|
76
|
+
packages=find_packages(exclude=("tests", "tests.*")),
|
|
77
|
+
package_data={"pelutils": ["py.typed"]},
|
|
75
78
|
author="Asger Laurits Schultz, Søren Winkel Holm",
|
|
76
79
|
author_email="asger.s@protonmail.com, swholm@protonmail.com",
|
|
77
80
|
keywords=["utility", "logger", "parser", "profiling", "plotting"],
|
|
File without changes
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import ctypes
|
|
2
|
-
|
|
3
|
-
import numpy as np
|
|
4
|
-
import pytest
|
|
5
|
-
import torch
|
|
6
|
-
|
|
7
|
-
import pelutils._c as c_utils
|
|
8
|
-
from pelutils import OS
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
@pytest.mark.skipif(OS.is_windows, reason="This test does spooky shit that scares Windows")
|
|
12
|
-
def test_get_c_array_args():
|
|
13
|
-
for dtype in int, float, np.float16, np.int32:
|
|
14
|
-
shape = (2, 4, 3)
|
|
15
|
-
np_arr = np.empty(shape, dtype=dtype)
|
|
16
|
-
arr_p, ndim, dims_p, strides_p = c_utils.get_array_c_args(torch.from_numpy(np_arr))
|
|
17
|
-
assert arr_p == np_arr.ctypes.data
|
|
18
|
-
assert ndim == len(shape)
|
|
19
|
-
|
|
20
|
-
itemsize = np.dtype(np.uint).itemsize
|
|
21
|
-
s = np_arr.dtype.itemsize
|
|
22
|
-
for i, d in enumerate(shape[::-1]):
|
|
23
|
-
i = len(shape) - i - 1 # noqa: PLW2901
|
|
24
|
-
assert d == ctypes.c_uint64.from_address(dims_p + i * itemsize).value
|
|
25
|
-
assert s == ctypes.c_uint64.from_address(strides_p + i * itemsize).value
|
|
26
|
-
s *= d
|
|
File without changes
|
|
@@ -1,81 +0,0 @@
|
|
|
1
|
-
import numpy as np
|
|
2
|
-
import pandas as pd
|
|
3
|
-
import pytest
|
|
4
|
-
import torch
|
|
5
|
-
|
|
6
|
-
from pelutils.datastorage2 import DataStorage2
|
|
7
|
-
from pelutils.datastorage2._pretty_json import _PICKLE_PREFIX
|
|
8
|
-
from pelutils.tests import UnitTestCollection
|
|
9
|
-
from pelutils.types import FloatArray
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
class DeepCollection(DataStorage2):
|
|
13
|
-
np_arr: FloatArray
|
|
14
|
-
tensor: torch.Tensor
|
|
15
|
-
df: pd.DataFrame
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
class Collection(DataStorage2):
|
|
19
|
-
string: str
|
|
20
|
-
list_of_floats: list[float]
|
|
21
|
-
tuple_of_floats: tuple[float, ...]
|
|
22
|
-
tuple_of_ints: tuple[int, ...]
|
|
23
|
-
collection: DeepCollection
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
class WhackStorage(DataStorage2):
|
|
27
|
-
"""Whack struct with lots of nested non-native data types."""
|
|
28
|
-
|
|
29
|
-
np_arr: FloatArray
|
|
30
|
-
tensor: torch.Tensor
|
|
31
|
-
df: pd.DataFrame
|
|
32
|
-
collection: Collection
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
data = WhackStorage(
|
|
36
|
-
np_arr=np.arange(5, dtype=np.float16),
|
|
37
|
-
tensor=torch.ones(5),
|
|
38
|
-
df=pd.DataFrame({"col1": [1, 2, 3], "col": np.arange(3)}),
|
|
39
|
-
collection=Collection(
|
|
40
|
-
string="Hello There",
|
|
41
|
-
list_of_floats=[1.0, 2.0, 3.0],
|
|
42
|
-
tuple_of_floats=(0.1, 0.2, 0.3),
|
|
43
|
-
tuple_of_ints=tuple(range(500)),
|
|
44
|
-
collection=DeepCollection(
|
|
45
|
-
np_arr=np.arange(10),
|
|
46
|
-
tensor=torch.zeros(0, dtype=bool),
|
|
47
|
-
df=pd.DataFrame({"col1": ["1", "2", "3"], "col": np.arange(3)}),
|
|
48
|
-
),
|
|
49
|
-
),
|
|
50
|
-
)
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
class TestDataStorage2(UnitTestCollection):
|
|
54
|
-
def test_save_load(self):
|
|
55
|
-
# Test save and load with custom file name
|
|
56
|
-
data.save(self.test_dir, filename="bollocks")
|
|
57
|
-
WhackStorage.load(self.test_dir, filename="bollocks")
|
|
58
|
-
# Set save and load with default file name
|
|
59
|
-
data.save(self.test_dir)
|
|
60
|
-
loaded = WhackStorage.load(self.test_dir)
|
|
61
|
-
assert (data.np_arr == loaded.np_arr).all()
|
|
62
|
-
assert data.np_arr.dtype == loaded.np_arr.dtype
|
|
63
|
-
assert (data.tensor == loaded.tensor).all()
|
|
64
|
-
assert data.tensor.dtype is loaded.tensor.dtype
|
|
65
|
-
assert data.df.equals(loaded.df)
|
|
66
|
-
assert data.collection.string == loaded.collection.string
|
|
67
|
-
assert data.collection.list_of_floats == data.collection.list_of_floats
|
|
68
|
-
assert data.collection.tuple_of_floats == data.collection.tuple_of_floats
|
|
69
|
-
assert data.collection.tuple_of_ints == data.collection.tuple_of_ints
|
|
70
|
-
assert (data.collection.collection.np_arr == loaded.collection.collection.np_arr).all()
|
|
71
|
-
assert data.collection.collection.np_arr.dtype == loaded.collection.collection.np_arr.dtype
|
|
72
|
-
assert (data.collection.collection.tensor == loaded.collection.collection.tensor).all()
|
|
73
|
-
assert data.collection.collection.tensor.dtype is loaded.collection.collection.tensor.dtype
|
|
74
|
-
assert data.collection.collection.df.equals(loaded.collection.collection.df)
|
|
75
|
-
|
|
76
|
-
# Check that corrupted data raises a ValueError
|
|
77
|
-
json_content = data._resolve_save_file(self.test_dir).read_text()
|
|
78
|
-
json_content = json_content.replace(_PICKLE_PREFIX, f"_{_PICKLE_PREFIX}")
|
|
79
|
-
data._resolve_save_file(self.test_dir).write_text(json_content)
|
|
80
|
-
with pytest.raises(ValueError):
|
|
81
|
-
WhackStorage.load(self.test_dir)
|
|
@@ -1,92 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
from dataclasses import dataclass
|
|
3
|
-
from datetime import datetime, timezone
|
|
4
|
-
from decimal import Decimal
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
from typing import Any
|
|
7
|
-
|
|
8
|
-
import pytest
|
|
9
|
-
|
|
10
|
-
from pelutils.datastorage2._pretty_json import _PICKLE_PREFIX, _decode_unpickle, _pretty_json
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
def pretty_json_with_defaults(obj: Any) -> Any:
|
|
14
|
-
return _pretty_json(
|
|
15
|
-
obj,
|
|
16
|
-
max_line_length=140,
|
|
17
|
-
indent=2,
|
|
18
|
-
safe=True,
|
|
19
|
-
)
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
# ─── Pickle / b64 fallback ─────────────────────────────────────────────────
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
@dataclass
|
|
26
|
-
class _Point:
|
|
27
|
-
x: float
|
|
28
|
-
y: float
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
class TestPickleFallback:
|
|
32
|
-
def test_datetime_is_pickled(self) -> None:
|
|
33
|
-
dt = datetime(2025, 1, 1, tzinfo=timezone.utc)
|
|
34
|
-
result = pretty_json_with_defaults({"ts": dt})
|
|
35
|
-
parsed = json.loads(result)
|
|
36
|
-
assert parsed["ts"].startswith(_PICKLE_PREFIX)
|
|
37
|
-
assert _decode_unpickle(parsed["ts"]) == dt
|
|
38
|
-
|
|
39
|
-
def test_dataclass_is_pickled(self) -> None:
|
|
40
|
-
pt = _Point(1.5, 2.5)
|
|
41
|
-
result = pretty_json_with_defaults({"point": pt})
|
|
42
|
-
parsed = json.loads(result)
|
|
43
|
-
restored = _decode_unpickle(parsed["point"])
|
|
44
|
-
assert restored == pt
|
|
45
|
-
|
|
46
|
-
def test_decimal_is_pickled(self) -> None:
|
|
47
|
-
obj = {"price": Decimal("19.99")}
|
|
48
|
-
result = pretty_json_with_defaults(obj)
|
|
49
|
-
parsed = json.loads(result)
|
|
50
|
-
assert _decode_unpickle(parsed["price"]) == Decimal("19.99")
|
|
51
|
-
|
|
52
|
-
def test_path_is_pickled(self) -> None:
|
|
53
|
-
p = Path("/usr/local/bin")
|
|
54
|
-
result = pretty_json_with_defaults({"path": p})
|
|
55
|
-
parsed = json.loads(result)
|
|
56
|
-
assert _decode_unpickle(parsed["path"]) == p
|
|
57
|
-
|
|
58
|
-
def test_set_inside_list_is_pickled(self) -> None:
|
|
59
|
-
obj = [1, {2, 3, 4}, "hello"]
|
|
60
|
-
result = pretty_json_with_defaults(obj)
|
|
61
|
-
parsed = json.loads(result)
|
|
62
|
-
assert parsed[0] == 1
|
|
63
|
-
assert parsed[2] == "hello"
|
|
64
|
-
assert _decode_unpickle(parsed[1]) == {2, 3, 4}
|
|
65
|
-
|
|
66
|
-
def test_nested_non_serialisable(self) -> None:
|
|
67
|
-
obj = {"outer": {"inner": _Point(0, 0)}}
|
|
68
|
-
result = pretty_json_with_defaults(obj)
|
|
69
|
-
parsed = json.loads(result)
|
|
70
|
-
assert _decode_unpickle(parsed["outer"]["inner"]) == _Point(0, 0)
|
|
71
|
-
|
|
72
|
-
def test_unpickle_rejects_plain_string(self) -> None:
|
|
73
|
-
with pytest.raises(ValueError, match="Not a pickled value"):
|
|
74
|
-
_decode_unpickle("just a string")
|
|
75
|
-
|
|
76
|
-
def test_non_string_dict_keys_coerced(self) -> None:
|
|
77
|
-
obj = {1: "one", 2: "two"} # type: ignore[dict-item]
|
|
78
|
-
result = pretty_json_with_defaults(obj) # type: ignore[arg-type]
|
|
79
|
-
parsed = json.loads(result)
|
|
80
|
-
assert parsed == {"1": "one", "2": "two"}
|
|
81
|
-
|
|
82
|
-
def test_output_is_valid_json_with_mixed_types(self) -> None:
|
|
83
|
-
obj = {
|
|
84
|
-
"name": "test",
|
|
85
|
-
"when": datetime(2025, 6, 1),
|
|
86
|
-
"tags": ["a", "b"],
|
|
87
|
-
"nested": {"d": Decimal("1.1"), "n": None},
|
|
88
|
-
}
|
|
89
|
-
parsed = json.loads(pretty_json_with_defaults(obj))
|
|
90
|
-
assert parsed["name"] == "test"
|
|
91
|
-
assert parsed["tags"] == ["a", "b"]
|
|
92
|
-
assert parsed["nested"]["n"] is None
|
|
File without changes
|
|
@@ -1,147 +0,0 @@
|
|
|
1
|
-
from itertools import product
|
|
2
|
-
|
|
3
|
-
import numpy as np
|
|
4
|
-
import pytest
|
|
5
|
-
|
|
6
|
-
from pelutils.ds.distributions import (
|
|
7
|
-
bernoulli,
|
|
8
|
-
beta,
|
|
9
|
-
binomial,
|
|
10
|
-
chi2,
|
|
11
|
-
expon,
|
|
12
|
-
gamma,
|
|
13
|
-
geom0,
|
|
14
|
-
geom1,
|
|
15
|
-
hypergeom,
|
|
16
|
-
lognorm,
|
|
17
|
-
nbinom,
|
|
18
|
-
norm,
|
|
19
|
-
poisson,
|
|
20
|
-
rayleigh,
|
|
21
|
-
)
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def test_norm():
|
|
25
|
-
mus = np.linspace(-10, 10, 5)
|
|
26
|
-
sigmas = np.linspace(0, 10, 5)[1:]
|
|
27
|
-
for mu, sigma in product(mus, sigmas):
|
|
28
|
-
dist = norm(mu, sigma**2)
|
|
29
|
-
assert np.isclose(dist.mean(), mu)
|
|
30
|
-
assert np.isclose(dist.var(), sigma**2)
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def test_lognorm():
|
|
34
|
-
mus = np.linspace(-10, 10, 5)
|
|
35
|
-
sigmas = np.linspace(0, 10, 5)[1:]
|
|
36
|
-
for mu, sigma in product(mus, sigmas):
|
|
37
|
-
dist = lognorm(mu, sigma**2)
|
|
38
|
-
assert np.isclose(dist.mean(), np.exp(mu + sigma**2 / 2))
|
|
39
|
-
assert np.isclose(dist.var(), (np.exp(sigma**2) - 1) * np.exp(2 * mu + sigma**2))
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
def test_expon():
|
|
43
|
-
lambdas = np.linspace(0, 10, 5)[1:]
|
|
44
|
-
for lambda_ in lambdas:
|
|
45
|
-
dist = expon(lambda_)
|
|
46
|
-
assert np.isclose(dist.mean(), 1 / lambda_)
|
|
47
|
-
assert np.isclose(dist.var(), 1 / lambda_**2)
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
def test_gamma():
|
|
51
|
-
rs = np.linspace(0, 10, 5)[1:]
|
|
52
|
-
lambdas = np.linspace(0, 10)[1:]
|
|
53
|
-
for r, lambda_ in product(rs, lambdas):
|
|
54
|
-
dist = gamma(r, lambda_)
|
|
55
|
-
assert np.isclose(dist.mean(), r / lambda_)
|
|
56
|
-
assert np.isclose(dist.var(), r / lambda_**2)
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
def test_chi2():
|
|
60
|
-
dfs = np.linspace(0, 10, 31)[1:]
|
|
61
|
-
for df in dfs:
|
|
62
|
-
dist = chi2(df)
|
|
63
|
-
assert np.isclose(dist.mean(), df)
|
|
64
|
-
assert np.isclose(dist.var(), 2 * df)
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
def test_rayleigh():
|
|
68
|
-
dist = rayleigh()
|
|
69
|
-
assert np.isclose(dist.mean(), np.sqrt(np.pi / 2))
|
|
70
|
-
assert np.isclose(dist.var(), (4 - np.pi) / 2)
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
def test_beta():
|
|
74
|
-
rs = np.linspace(0, 10, 5)[1:]
|
|
75
|
-
ss = np.linspace(0, 10, 5)[1:]
|
|
76
|
-
for r, s in product(rs, ss):
|
|
77
|
-
dist = beta(r, s)
|
|
78
|
-
assert np.isclose(dist.mean(), r / (r + s))
|
|
79
|
-
assert np.isclose(dist.var(), r * s / ((r + s) ** 2 * (r + s + 1)))
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
def test_bernoulli():
|
|
83
|
-
ps = np.linspace(0, 1, 5)
|
|
84
|
-
for p in ps:
|
|
85
|
-
dist = bernoulli(p)
|
|
86
|
-
assert np.isclose(dist.mean(), p)
|
|
87
|
-
assert np.isclose(dist.var(), p * (1 - p))
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
def test_binomial():
|
|
91
|
-
ns = np.arange(0, 10)
|
|
92
|
-
ps = np.linspace(0, 1, 5)
|
|
93
|
-
for n, p in product(ns, ps):
|
|
94
|
-
dist = binomial(n, p)
|
|
95
|
-
assert np.isclose(dist.mean(), n * p)
|
|
96
|
-
assert np.isclose(dist.var(), n * p * (1 - p))
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
def test_poisson():
|
|
100
|
-
mus = np.linspace(0, 10, 5)
|
|
101
|
-
for mu in mus:
|
|
102
|
-
dist = poisson(mu)
|
|
103
|
-
assert np.isclose(dist.mean(), mu)
|
|
104
|
-
assert np.isclose(dist.var(), mu)
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
@pytest.mark.filterwarnings("ignore:invalid value", "ignore:divide by zero")
|
|
108
|
-
def test_hypergeom():
|
|
109
|
-
ns = np.arange(0, 10)
|
|
110
|
-
Ns = np.arange(0, 10) # noqa: N806
|
|
111
|
-
Gs = np.arange(0, 10) # noqa: N806
|
|
112
|
-
for n, N, G in product(ns, Ns, Gs): # noqa: N806
|
|
113
|
-
if n >= N or G > N or n == 0 or N <= 1 or G == 0:
|
|
114
|
-
with pytest.raises(AssertionError):
|
|
115
|
-
dist = hypergeom(n, N, G)
|
|
116
|
-
continue
|
|
117
|
-
dist = hypergeom(n, N, G)
|
|
118
|
-
assert np.isclose(dist.mean(), n * G / N)
|
|
119
|
-
assert np.isclose(dist.var(), n * G / N * (N - G) / N * (N - n) / (N - 1))
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
@pytest.mark.filterwarnings("ignore:divide by zero")
|
|
123
|
-
def test_geom0():
|
|
124
|
-
ps = np.linspace(0, 1, 5)[1:]
|
|
125
|
-
for p in ps:
|
|
126
|
-
dist = geom0(p)
|
|
127
|
-
assert np.isclose(dist.mean(), (1 - p) / p)
|
|
128
|
-
assert np.isclose(dist.var(), (1 - p) / p**2)
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
@pytest.mark.filterwarnings("ignore:divide by zero")
|
|
132
|
-
def test_geom1():
|
|
133
|
-
ps = np.linspace(0, 1, 5)[1:]
|
|
134
|
-
for p in ps:
|
|
135
|
-
dist = geom1(p)
|
|
136
|
-
assert np.isclose(dist.mean(), 1 / p)
|
|
137
|
-
assert np.isclose(dist.var(), (1 - p) / p**2)
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
@pytest.mark.filterwarnings("ignore:divide by zero")
|
|
141
|
-
def test_nbinom():
|
|
142
|
-
rs = np.arange(0, 10)[1:]
|
|
143
|
-
ps = np.linspace(0, 1, 5)[1:]
|
|
144
|
-
for r, p in product(rs, ps):
|
|
145
|
-
dist = nbinom(r, p)
|
|
146
|
-
assert np.isclose(dist.mean(), r * (1 - p) / p)
|
|
147
|
-
assert np.isclose(dist.var(), r * (1 - p) / p**2)
|