pelutils 3.8.0__tar.gz → 3.8.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. {pelutils-3.8.0 → pelutils-3.8.2}/PKG-INFO +9 -9
  2. {pelutils-3.8.0 → pelutils-3.8.2}/README.md +8 -8
  3. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/__version__.py +1 -1
  4. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/datastorage2/__init__.py +5 -3
  5. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/logging/__init__.py +14 -9
  6. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils.egg-info/PKG-INFO +9 -9
  7. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils.egg-info/SOURCES.txt +2 -15
  8. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils.egg-info/top_level.txt +0 -1
  9. {pelutils-3.8.0 → pelutils-3.8.2}/setup.py +5 -2
  10. pelutils-3.8.0/tests/_c/__init__.py +0 -0
  11. pelutils-3.8.0/tests/_c/test_c.py +0 -26
  12. pelutils-3.8.0/tests/datastorage2/__init__.py +0 -0
  13. pelutils-3.8.0/tests/datastorage2/test_init.py +0 -81
  14. pelutils-3.8.0/tests/datastorage2/test_pretty_json.py +0 -92
  15. pelutils-3.8.0/tests/ds/__init__.py +0 -0
  16. pelutils-3.8.0/tests/ds/test_distributions.py +0 -147
  17. pelutils-3.8.0/tests/ds/test_ds.py +0 -110
  18. pelutils-3.8.0/tests/ds/test_plots.py +0 -235
  19. pelutils-3.8.0/tests/ds/test_stats.py +0 -45
  20. pelutils-3.8.0/tests/logging/__init__.py +0 -0
  21. pelutils-3.8.0/tests/logging/test_logging.py +0 -330
  22. pelutils-3.8.0/tests/logging/test_rotate.py +0 -184
  23. {pelutils-3.8.0 → pelutils-3.8.2}/MANIFEST.in +0 -0
  24. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/__init__.py +0 -0
  25. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/_c/__init__.py +0 -0
  26. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/_c/ds.c +0 -0
  27. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/_c/hashmap.c/LICENSE +0 -0
  28. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/_c/hashmap.c/hashmap.c +0 -0
  29. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/_c/hashmap.c/hashmap.h +0 -0
  30. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/_entry_points/__init__.py +0 -0
  31. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/_entry_points/linecounter.py +0 -0
  32. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/datastorage.py +0 -0
  33. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/datastorage2/_pretty_json.py +0 -0
  34. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/ds/__init__.py +0 -0
  35. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/ds/distributions.py +0 -0
  36. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/ds/plots.py +0 -0
  37. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/ds/stats.py +0 -0
  38. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/format.py +0 -0
  39. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/jsonl.py +0 -0
  40. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/logging/_rotate.py +0 -0
  41. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/logging/_utils.py +0 -0
  42. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/parser.py +0 -0
  43. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/pretty_json.py +0 -0
  44. /pelutils-3.8.0/tests/__init__.py → /pelutils-3.8.2/pelutils/py.typed +0 -0
  45. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/tests.py +0 -0
  46. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/ticktock.py +0 -0
  47. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils/types.py +0 -0
  48. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils.egg-info/dependency_links.txt +0 -0
  49. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils.egg-info/entry_points.txt +0 -0
  50. {pelutils-3.8.0 → pelutils-3.8.2}/pelutils.egg-info/requires.txt +0 -0
  51. {pelutils-3.8.0 → pelutils-3.8.2}/setup.cfg +0 -0
  52. {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_datastorage.py +0 -0
  53. {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_format.py +0 -0
  54. {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_init.py +0 -0
  55. {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_jsonl.py +0 -0
  56. {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_parser.py +0 -0
  57. {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_pretty_json.py +0 -0
  58. {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_tests.py +0 -0
  59. {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_ticktock.py +0 -0
  60. {pelutils-3.8.0 → pelutils-3.8.2}/tests/test_types.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: pelutils
3
- Version: 3.8.0
3
+ Version: 3.8.2
4
4
  Summary: The Swiss army knife of Python projects.
5
5
  Home-page: https://github.com/peleiden/pelutils
6
6
  Download-URL: https://pypi.org/project/pelutils/
@@ -36,14 +36,14 @@ License-File: pelutils/_c/hashmap.c/LICENSE
36
36
  The Swiss army knife of Python projects.
37
37
 
38
38
  - A simple and powerful logger with colourful printing, stacktraces, and log file rotation.
39
- - Parsing for combining config files and command-line arguments - especially useful developing algorithms with many parameters.
40
- - A timer inspired by Matlab's `tic` and `toc`.
41
- - Simple, near-zero cost performance profiler.
42
- - An extension to the built-in `dataclass` for saving and loading data.
39
+ - Argument parser which supports both command line arguments and config files and has baked-in auto-documentation.
40
+ - A simple timer inspired by Matlab's `tic` and `toc`.
41
+ - Easy-to-use, near-zero cost performance profiler.
42
+ - An extension to `pydantic.BaseModel` with support for saving any data structure to, and loading from, a human-readable JSON file.
43
43
  - Table formatting with built-in LaTeX support.
44
44
  - Miscellaneous standalone functions - see `pelutils/__init__.py`.
45
45
  - Data-science submodule with extra utilities for statistics, plotting with `matplotlib`, and machine learning using `PyTorch`.
46
- - `unique` function in the style of `numpy.unique` which runs in linear time, making it significantly.
46
+ - `unique` function in the style of `numpy.unique` which runs in linear time, making it significantly faster for large arrays.
47
47
 
48
48
  `pelutils` supports Python 3.9+.
49
49
 
@@ -121,8 +121,6 @@ based = BasedClass(
121
121
  baseder=BasederClass(based_string="Hello there")
122
122
  )
123
123
  # Save it to a file
124
- based.save(".")
125
- # Save it to a file
126
124
  based.save("directory/to/save/in")
127
125
 
128
126
  # Load it again
@@ -380,7 +378,9 @@ Examples of all the plotting utilities are shown in the `examples` directory.
380
378
 
381
379
  Precompiled wheels are provided for most common platforms.
382
380
  Notably, they are not provided for 32-bit systems.
383
- If no wheel is provided, `pip` should attempt a source install.
381
+ If no wheel is provided, `pip` should attempt a source install - this requires `<Python.h>` to be available.
382
+ On Ubuntu, this can be installed with `sudo apt install python3-dev`, and on Fedora, it is installed with `sudo dnf install python3-devel`.
383
+
384
384
  If all else fails, it is possible to install from source by pointing `pip` to Github directly:
385
385
  ```
386
386
  pip install git+https://github.com/peleiden/pelutils.git@release#egg=pelutils
@@ -11,14 +11,14 @@
11
11
  The Swiss army knife of Python projects.
12
12
 
13
13
  - A simple and powerful logger with colourful printing, stacktraces, and log file rotation.
14
- - Parsing for combining config files and command-line arguments - especially useful developing algorithms with many parameters.
15
- - A timer inspired by Matlab's `tic` and `toc`.
16
- - Simple, near-zero cost performance profiler.
17
- - An extension to the built-in `dataclass` for saving and loading data.
14
+ - Argument parser which supports both command line arguments and config files and has baked-in auto-documentation.
15
+ - A simple timer inspired by Matlab's `tic` and `toc`.
16
+ - Easy-to-use, near-zero cost performance profiler.
17
+ - An extension to `pydantic.BaseModel` with support for saving any data structure to, and loading from, a human-readable JSON file.
18
18
  - Table formatting with built-in LaTeX support.
19
19
  - Miscellaneous standalone functions - see `pelutils/__init__.py`.
20
20
  - Data-science submodule with extra utilities for statistics, plotting with `matplotlib`, and machine learning using `PyTorch`.
21
- - `unique` function in the style of `numpy.unique` which runs in linear time, making it significantly.
21
+ - `unique` function in the style of `numpy.unique` which runs in linear time, making it significantly faster for large arrays.
22
22
 
23
23
  `pelutils` supports Python 3.9+.
24
24
 
@@ -96,8 +96,6 @@ based = BasedClass(
96
96
  baseder=BasederClass(based_string="Hello there")
97
97
  )
98
98
  # Save it to a file
99
- based.save(".")
100
- # Save it to a file
101
99
  based.save("directory/to/save/in")
102
100
 
103
101
  # Load it again
@@ -355,7 +353,9 @@ Examples of all the plotting utilities are shown in the `examples` directory.
355
353
 
356
354
  Precompiled wheels are provided for most common platforms.
357
355
  Notably, they are not provided for 32-bit systems.
358
- If no wheel is provided, `pip` should attempt a source install.
356
+ If no wheel is provided, `pip` should attempt a source install - this requires `<Python.h>` to be available.
357
+ On Ubuntu, this can be installed with `sudo apt install python3-dev`, and on Fedora, it is installed with `sudo dnf install python3-devel`.
358
+
359
359
  If all else fails, it is possible to install from source by pointing `pip` to Github directly:
360
360
  ```
361
361
  pip install git+https://github.com/peleiden/pelutils.git@release#egg=pelutils
@@ -1,2 +1,2 @@
1
1
  # Do not put anything else in this file
2
- __version__ = "3.8.0"
2
+ __version__ = "3.8.2"
@@ -6,13 +6,13 @@ from typing import TypeVar
6
6
 
7
7
  from pydantic import BaseModel, ConfigDict
8
8
 
9
- from pelutils.datastorage2._pretty_json import _make_json_unsafe, _pretty_json # pyright: ignore[reportPrivateUsage]
9
+ from pelutils.datastorage2._pretty_json import _make_json_unsafe, _pickle_encode, _pretty_json # pyright: ignore[reportPrivateUsage]
10
10
 
11
11
  _T = TypeVar("_T", bound="DataStorage2")
12
12
 
13
13
 
14
14
  class DataStorage2(BaseModel):
15
- """Better version of DataStorage based on pydantic.
15
+ """Better version of DataStorage based on pydantic. It makes any class fully JSON serialisable.
16
16
 
17
17
  Usage is very similar to DataStorage, but it has a few key advantages:
18
18
  - Allows saving and loading of nested objects.
@@ -38,12 +38,14 @@ class DataStorage2(BaseModel):
38
38
  If save_name is None, the path is <directory>/<class name>.json, otherwise it is <directory>/<save_name>.json.
39
39
  """
40
40
  savepath = self._resolve_save_file(directory, filename)
41
- self_dict = self.model_dump()
41
+ self_dict = self.model_dump(mode="json", fallback=_pickle_encode)
42
42
  savepath.write_text(
43
43
  _pretty_json(
44
44
  self_dict,
45
45
  max_line_length=max_line_length,
46
46
  indent=indent,
47
+ # Safe should not be needed here, as the fallback function in model_dump should ensure no issues
48
+ # However, it is kept for good measure in case there unforeseen edge cases
47
49
  safe=True,
48
50
  ),
49
51
  encoding=encoding,
@@ -4,7 +4,7 @@ import os
4
4
  import traceback as tb
5
5
  from collections.abc import Generator, Iterable
6
6
  from pathlib import Path
7
- from typing import Optional
7
+ from typing import Any, Optional
8
8
 
9
9
  from pelutils import OS, UnsupportedOS, get_repo, get_timestamp
10
10
  from pelutils.format import RichString
@@ -135,7 +135,7 @@ class Logger:
135
135
 
136
136
  def _log(
137
137
  self,
138
- *tolog: str,
138
+ *tolog: Any, # pyright: ignore[reportExplicitAny]
139
139
  level: LogLevels = LogLevels.INFO,
140
140
  with_info: bool = True,
141
141
  sep: str | None = None,
@@ -276,34 +276,39 @@ class Logger:
276
276
  raise UnsupportedOS("Log collecting is not supported on windows")
277
277
  return _CollectLogs(self)
278
278
 
279
- def section(self, *tolog: str, with_info: bool = True, sep: str | None = None, with_print: bool | None = None, newline: bool = True):
279
+ def section(self, *tolog: Any, with_info: bool = True, sep: str | None = None, with_print: bool | None = None, newline: bool = True): # pyright: ignore[reportExplicitAny]
280
280
  """Log at SECTION level. See .log method for argument descriptions."""
281
281
  if newline:
282
282
  self._log("")
283
283
  self._log(*tolog, with_info=with_info, sep=sep, with_print=with_print, level=LogLevels.SECTION)
284
284
 
285
- def critical(self, *tolog: str, with_info: bool = True, sep: str | None = None, with_print: bool | None = None):
285
+ def critical(self, *tolog: Any, with_info: bool = True, sep: str | None = None, with_print: bool | None = None): # pyright: ignore[reportExplicitAny]
286
286
  """Log at CRITICAL level. See .log method for argument descriptions."""
287
287
  self._log(*tolog, with_info=with_info, sep=sep, with_print=with_print, level=LogLevels.CRITICAL)
288
288
 
289
- def error(self, *tolog: str, with_info: bool = True, sep: str | None = None, with_print: bool | None = None):
289
+ def error(self, *tolog: Any, with_info: bool = True, sep: str | None = None, with_print: bool | None = None): # pyright: ignore[reportExplicitAny]
290
290
  """Log at ERROR level. See .log method for argument descriptions."""
291
291
  self._log(*tolog, with_info=with_info, sep=sep, with_print=with_print, level=LogLevels.ERROR)
292
292
 
293
- def warning(self, *tolog: str, with_info: bool = True, sep: str | None = None, with_print: bool | None = None):
293
+ def warning(self, *tolog: Any, with_info: bool = True, sep: str | None = None, with_print: bool | None = None): # pyright: ignore[reportExplicitAny]
294
294
  """Log at WARNING level. See .log method for argument descriptions."""
295
295
  self._log(*tolog, with_info=with_info, sep=sep, with_print=with_print, level=LogLevels.WARNING)
296
296
 
297
- def info(self, *tolog: str, with_info: bool = True, sep: str | None = None, with_print: bool | None = None):
297
+ def info(self, *tolog: Any, with_info: bool = True, sep: str | None = None, with_print: bool | None = None): # pyright: ignore[reportExplicitAny]
298
298
  """Log at INFO level. See .log method for argument descriptions."""
299
299
  self._log(*tolog, with_info=with_info, sep=sep, with_print=with_print, level=LogLevels.INFO)
300
300
 
301
- def debug(self, *tolog: str, with_info: bool = True, sep: str | None = None, with_print: bool | None = None):
301
+ def debug(self, *tolog: Any, with_info: bool = True, sep: str | None = None, with_print: bool | None = None): # pyright: ignore[reportExplicitAny]
302
302
  """Log at DEBUG level. See .log method for argument descriptions."""
303
303
  self._log(*tolog, with_info=with_info, sep=sep, with_print=with_print, level=LogLevels.DEBUG)
304
304
 
305
305
  def __call__(
306
- self, *tolog: str, level: LogLevels = LogLevels.INFO, with_info: bool = True, sep: str | None = None, with_print: bool | None = None
306
+ self,
307
+ *tolog: Any, # pyright: ignore[reportExplicitAny]
308
+ level: LogLevels = LogLevels.INFO,
309
+ with_info: bool = True,
310
+ sep: str | None = None,
311
+ with_print: bool | None = None,
307
312
  ):
308
313
  """Shorthand for specific logging methods where level is specified as an argument."""
309
314
  self._log(*tolog, level=level, with_info=with_info, sep=sep, with_print=with_print)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: pelutils
3
- Version: 3.8.0
3
+ Version: 3.8.2
4
4
  Summary: The Swiss army knife of Python projects.
5
5
  Home-page: https://github.com/peleiden/pelutils
6
6
  Download-URL: https://pypi.org/project/pelutils/
@@ -36,14 +36,14 @@ License-File: pelutils/_c/hashmap.c/LICENSE
36
36
  The Swiss army knife of Python projects.
37
37
 
38
38
  - A simple and powerful logger with colourful printing, stacktraces, and log file rotation.
39
- - Parsing for combining config files and command-line arguments - especially useful developing algorithms with many parameters.
40
- - A timer inspired by Matlab's `tic` and `toc`.
41
- - Simple, near-zero cost performance profiler.
42
- - An extension to the built-in `dataclass` for saving and loading data.
39
+ - Argument parser which supports both command line arguments and config files and has baked-in auto-documentation.
40
+ - A simple timer inspired by Matlab's `tic` and `toc`.
41
+ - Easy-to-use, near-zero cost performance profiler.
42
+ - An extension to `pydantic.BaseModel` with support for saving any data structure to, and loading from, a human-readable JSON file.
43
43
  - Table formatting with built-in LaTeX support.
44
44
  - Miscellaneous standalone functions - see `pelutils/__init__.py`.
45
45
  - Data-science submodule with extra utilities for statistics, plotting with `matplotlib`, and machine learning using `PyTorch`.
46
- - `unique` function in the style of `numpy.unique` which runs in linear time, making it significantly.
46
+ - `unique` function in the style of `numpy.unique` which runs in linear time, making it significantly faster for large arrays.
47
47
 
48
48
  `pelutils` supports Python 3.9+.
49
49
 
@@ -121,8 +121,6 @@ based = BasedClass(
121
121
  baseder=BasederClass(based_string="Hello there")
122
122
  )
123
123
  # Save it to a file
124
- based.save(".")
125
- # Save it to a file
126
124
  based.save("directory/to/save/in")
127
125
 
128
126
  # Load it again
@@ -380,7 +378,9 @@ Examples of all the plotting utilities are shown in the `examples` directory.
380
378
 
381
379
  Precompiled wheels are provided for most common platforms.
382
380
  Notably, they are not provided for 32-bit systems.
383
- If no wheel is provided, `pip` should attempt a source install.
381
+ If no wheel is provided, `pip` should attempt a source install - this requires `<Python.h>` to be available.
382
+ On Ubuntu, this can be installed with `sudo apt install python3-dev`, and on Fedora, it is installed with `sudo dnf install python3-devel`.
383
+
384
384
  If all else fails, it is possible to install from source by pointing `pip` to Github directly:
385
385
  ```
386
386
  pip install git+https://github.com/peleiden/pelutils.git@release#egg=pelutils
@@ -8,6 +8,7 @@ pelutils/format.py
8
8
  pelutils/jsonl.py
9
9
  pelutils/parser.py
10
10
  pelutils/pretty_json.py
11
+ pelutils/py.typed
11
12
  pelutils/tests.py
12
13
  pelutils/ticktock.py
13
14
  pelutils/types.py
@@ -33,7 +34,6 @@ pelutils/ds/stats.py
33
34
  pelutils/logging/__init__.py
34
35
  pelutils/logging/_rotate.py
35
36
  pelutils/logging/_utils.py
36
- tests/__init__.py
37
37
  tests/test_datastorage.py
38
38
  tests/test_format.py
39
39
  tests/test_init.py
@@ -42,17 +42,4 @@ tests/test_parser.py
42
42
  tests/test_pretty_json.py
43
43
  tests/test_tests.py
44
44
  tests/test_ticktock.py
45
- tests/test_types.py
46
- tests/_c/__init__.py
47
- tests/_c/test_c.py
48
- tests/datastorage2/__init__.py
49
- tests/datastorage2/test_init.py
50
- tests/datastorage2/test_pretty_json.py
51
- tests/ds/__init__.py
52
- tests/ds/test_distributions.py
53
- tests/ds/test_ds.py
54
- tests/ds/test_plots.py
55
- tests/ds/test_stats.py
56
- tests/logging/__init__.py
57
- tests/logging/test_logging.py
58
- tests/logging/test_rotate.py
45
+ tests/test_types.py
@@ -1,3 +1,2 @@
1
1
  _pelutils_c
2
2
  pelutils
3
- tests
@@ -48,7 +48,9 @@ class CExtension(Extension):
48
48
  """See this thread for details: https://stackoverflow.com/a/34830639/13196863."""
49
49
 
50
50
 
51
- class build(build_): # noqa: D101
51
+ class build(build_):
52
+ """Extension of normal build class which handles C extensions."""
53
+
52
54
  def build_extension(self, ext): # noqa: D102
53
55
  self._ctypes = isinstance(ext, CExtension)
54
56
  return super().build_extension(ext)
@@ -71,7 +73,8 @@ setup_args = dict(
71
73
  long_description_content_type="text/markdown",
72
74
  long_description=README,
73
75
  license="MIT",
74
- packages=find_packages(),
76
+ packages=find_packages(exclude=("tests", "tests.*")),
77
+ package_data={"pelutils": ["py.typed"]},
75
78
  author="Asger Laurits Schultz, Søren Winkel Holm",
76
79
  author_email="asger.s@protonmail.com, swholm@protonmail.com",
77
80
  keywords=["utility", "logger", "parser", "profiling", "plotting"],
File without changes
@@ -1,26 +0,0 @@
1
- import ctypes
2
-
3
- import numpy as np
4
- import pytest
5
- import torch
6
-
7
- import pelutils._c as c_utils
8
- from pelutils import OS
9
-
10
-
11
- @pytest.mark.skipif(OS.is_windows, reason="This test does spooky shit that scares Windows")
12
- def test_get_c_array_args():
13
- for dtype in int, float, np.float16, np.int32:
14
- shape = (2, 4, 3)
15
- np_arr = np.empty(shape, dtype=dtype)
16
- arr_p, ndim, dims_p, strides_p = c_utils.get_array_c_args(torch.from_numpy(np_arr))
17
- assert arr_p == np_arr.ctypes.data
18
- assert ndim == len(shape)
19
-
20
- itemsize = np.dtype(np.uint).itemsize
21
- s = np_arr.dtype.itemsize
22
- for i, d in enumerate(shape[::-1]):
23
- i = len(shape) - i - 1 # noqa: PLW2901
24
- assert d == ctypes.c_uint64.from_address(dims_p + i * itemsize).value
25
- assert s == ctypes.c_uint64.from_address(strides_p + i * itemsize).value
26
- s *= d
File without changes
@@ -1,81 +0,0 @@
1
- import numpy as np
2
- import pandas as pd
3
- import pytest
4
- import torch
5
-
6
- from pelutils.datastorage2 import DataStorage2
7
- from pelutils.datastorage2._pretty_json import _PICKLE_PREFIX
8
- from pelutils.tests import UnitTestCollection
9
- from pelutils.types import FloatArray
10
-
11
-
12
- class DeepCollection(DataStorage2):
13
- np_arr: FloatArray
14
- tensor: torch.Tensor
15
- df: pd.DataFrame
16
-
17
-
18
- class Collection(DataStorage2):
19
- string: str
20
- list_of_floats: list[float]
21
- tuple_of_floats: tuple[float, ...]
22
- tuple_of_ints: tuple[int, ...]
23
- collection: DeepCollection
24
-
25
-
26
- class WhackStorage(DataStorage2):
27
- """Whack struct with lots of nested non-native data types."""
28
-
29
- np_arr: FloatArray
30
- tensor: torch.Tensor
31
- df: pd.DataFrame
32
- collection: Collection
33
-
34
-
35
- data = WhackStorage(
36
- np_arr=np.arange(5, dtype=np.float16),
37
- tensor=torch.ones(5),
38
- df=pd.DataFrame({"col1": [1, 2, 3], "col": np.arange(3)}),
39
- collection=Collection(
40
- string="Hello There",
41
- list_of_floats=[1.0, 2.0, 3.0],
42
- tuple_of_floats=(0.1, 0.2, 0.3),
43
- tuple_of_ints=tuple(range(500)),
44
- collection=DeepCollection(
45
- np_arr=np.arange(10),
46
- tensor=torch.zeros(0, dtype=bool),
47
- df=pd.DataFrame({"col1": ["1", "2", "3"], "col": np.arange(3)}),
48
- ),
49
- ),
50
- )
51
-
52
-
53
- class TestDataStorage2(UnitTestCollection):
54
- def test_save_load(self):
55
- # Test save and load with custom file name
56
- data.save(self.test_dir, filename="bollocks")
57
- WhackStorage.load(self.test_dir, filename="bollocks")
58
- # Set save and load with default file name
59
- data.save(self.test_dir)
60
- loaded = WhackStorage.load(self.test_dir)
61
- assert (data.np_arr == loaded.np_arr).all()
62
- assert data.np_arr.dtype == loaded.np_arr.dtype
63
- assert (data.tensor == loaded.tensor).all()
64
- assert data.tensor.dtype is loaded.tensor.dtype
65
- assert data.df.equals(loaded.df)
66
- assert data.collection.string == loaded.collection.string
67
- assert data.collection.list_of_floats == data.collection.list_of_floats
68
- assert data.collection.tuple_of_floats == data.collection.tuple_of_floats
69
- assert data.collection.tuple_of_ints == data.collection.tuple_of_ints
70
- assert (data.collection.collection.np_arr == loaded.collection.collection.np_arr).all()
71
- assert data.collection.collection.np_arr.dtype == loaded.collection.collection.np_arr.dtype
72
- assert (data.collection.collection.tensor == loaded.collection.collection.tensor).all()
73
- assert data.collection.collection.tensor.dtype is loaded.collection.collection.tensor.dtype
74
- assert data.collection.collection.df.equals(loaded.collection.collection.df)
75
-
76
- # Check that corrupted data raises a ValueError
77
- json_content = data._resolve_save_file(self.test_dir).read_text()
78
- json_content = json_content.replace(_PICKLE_PREFIX, f"_{_PICKLE_PREFIX}")
79
- data._resolve_save_file(self.test_dir).write_text(json_content)
80
- with pytest.raises(ValueError):
81
- WhackStorage.load(self.test_dir)
@@ -1,92 +0,0 @@
1
- import json
2
- from dataclasses import dataclass
3
- from datetime import datetime, timezone
4
- from decimal import Decimal
5
- from pathlib import Path
6
- from typing import Any
7
-
8
- import pytest
9
-
10
- from pelutils.datastorage2._pretty_json import _PICKLE_PREFIX, _decode_unpickle, _pretty_json
11
-
12
-
13
- def pretty_json_with_defaults(obj: Any) -> Any:
14
- return _pretty_json(
15
- obj,
16
- max_line_length=140,
17
- indent=2,
18
- safe=True,
19
- )
20
-
21
-
22
- # ─── Pickle / b64 fallback ─────────────────────────────────────────────────
23
-
24
-
25
- @dataclass
26
- class _Point:
27
- x: float
28
- y: float
29
-
30
-
31
- class TestPickleFallback:
32
- def test_datetime_is_pickled(self) -> None:
33
- dt = datetime(2025, 1, 1, tzinfo=timezone.utc)
34
- result = pretty_json_with_defaults({"ts": dt})
35
- parsed = json.loads(result)
36
- assert parsed["ts"].startswith(_PICKLE_PREFIX)
37
- assert _decode_unpickle(parsed["ts"]) == dt
38
-
39
- def test_dataclass_is_pickled(self) -> None:
40
- pt = _Point(1.5, 2.5)
41
- result = pretty_json_with_defaults({"point": pt})
42
- parsed = json.loads(result)
43
- restored = _decode_unpickle(parsed["point"])
44
- assert restored == pt
45
-
46
- def test_decimal_is_pickled(self) -> None:
47
- obj = {"price": Decimal("19.99")}
48
- result = pretty_json_with_defaults(obj)
49
- parsed = json.loads(result)
50
- assert _decode_unpickle(parsed["price"]) == Decimal("19.99")
51
-
52
- def test_path_is_pickled(self) -> None:
53
- p = Path("/usr/local/bin")
54
- result = pretty_json_with_defaults({"path": p})
55
- parsed = json.loads(result)
56
- assert _decode_unpickle(parsed["path"]) == p
57
-
58
- def test_set_inside_list_is_pickled(self) -> None:
59
- obj = [1, {2, 3, 4}, "hello"]
60
- result = pretty_json_with_defaults(obj)
61
- parsed = json.loads(result)
62
- assert parsed[0] == 1
63
- assert parsed[2] == "hello"
64
- assert _decode_unpickle(parsed[1]) == {2, 3, 4}
65
-
66
- def test_nested_non_serialisable(self) -> None:
67
- obj = {"outer": {"inner": _Point(0, 0)}}
68
- result = pretty_json_with_defaults(obj)
69
- parsed = json.loads(result)
70
- assert _decode_unpickle(parsed["outer"]["inner"]) == _Point(0, 0)
71
-
72
- def test_unpickle_rejects_plain_string(self) -> None:
73
- with pytest.raises(ValueError, match="Not a pickled value"):
74
- _decode_unpickle("just a string")
75
-
76
- def test_non_string_dict_keys_coerced(self) -> None:
77
- obj = {1: "one", 2: "two"} # type: ignore[dict-item]
78
- result = pretty_json_with_defaults(obj) # type: ignore[arg-type]
79
- parsed = json.loads(result)
80
- assert parsed == {"1": "one", "2": "two"}
81
-
82
- def test_output_is_valid_json_with_mixed_types(self) -> None:
83
- obj = {
84
- "name": "test",
85
- "when": datetime(2025, 6, 1),
86
- "tags": ["a", "b"],
87
- "nested": {"d": Decimal("1.1"), "n": None},
88
- }
89
- parsed = json.loads(pretty_json_with_defaults(obj))
90
- assert parsed["name"] == "test"
91
- assert parsed["tags"] == ["a", "b"]
92
- assert parsed["nested"]["n"] is None
File without changes
@@ -1,147 +0,0 @@
1
- from itertools import product
2
-
3
- import numpy as np
4
- import pytest
5
-
6
- from pelutils.ds.distributions import (
7
- bernoulli,
8
- beta,
9
- binomial,
10
- chi2,
11
- expon,
12
- gamma,
13
- geom0,
14
- geom1,
15
- hypergeom,
16
- lognorm,
17
- nbinom,
18
- norm,
19
- poisson,
20
- rayleigh,
21
- )
22
-
23
-
24
- def test_norm():
25
- mus = np.linspace(-10, 10, 5)
26
- sigmas = np.linspace(0, 10, 5)[1:]
27
- for mu, sigma in product(mus, sigmas):
28
- dist = norm(mu, sigma**2)
29
- assert np.isclose(dist.mean(), mu)
30
- assert np.isclose(dist.var(), sigma**2)
31
-
32
-
33
- def test_lognorm():
34
- mus = np.linspace(-10, 10, 5)
35
- sigmas = np.linspace(0, 10, 5)[1:]
36
- for mu, sigma in product(mus, sigmas):
37
- dist = lognorm(mu, sigma**2)
38
- assert np.isclose(dist.mean(), np.exp(mu + sigma**2 / 2))
39
- assert np.isclose(dist.var(), (np.exp(sigma**2) - 1) * np.exp(2 * mu + sigma**2))
40
-
41
-
42
- def test_expon():
43
- lambdas = np.linspace(0, 10, 5)[1:]
44
- for lambda_ in lambdas:
45
- dist = expon(lambda_)
46
- assert np.isclose(dist.mean(), 1 / lambda_)
47
- assert np.isclose(dist.var(), 1 / lambda_**2)
48
-
49
-
50
- def test_gamma():
51
- rs = np.linspace(0, 10, 5)[1:]
52
- lambdas = np.linspace(0, 10)[1:]
53
- for r, lambda_ in product(rs, lambdas):
54
- dist = gamma(r, lambda_)
55
- assert np.isclose(dist.mean(), r / lambda_)
56
- assert np.isclose(dist.var(), r / lambda_**2)
57
-
58
-
59
- def test_chi2():
60
- dfs = np.linspace(0, 10, 31)[1:]
61
- for df in dfs:
62
- dist = chi2(df)
63
- assert np.isclose(dist.mean(), df)
64
- assert np.isclose(dist.var(), 2 * df)
65
-
66
-
67
- def test_rayleigh():
68
- dist = rayleigh()
69
- assert np.isclose(dist.mean(), np.sqrt(np.pi / 2))
70
- assert np.isclose(dist.var(), (4 - np.pi) / 2)
71
-
72
-
73
- def test_beta():
74
- rs = np.linspace(0, 10, 5)[1:]
75
- ss = np.linspace(0, 10, 5)[1:]
76
- for r, s in product(rs, ss):
77
- dist = beta(r, s)
78
- assert np.isclose(dist.mean(), r / (r + s))
79
- assert np.isclose(dist.var(), r * s / ((r + s) ** 2 * (r + s + 1)))
80
-
81
-
82
- def test_bernoulli():
83
- ps = np.linspace(0, 1, 5)
84
- for p in ps:
85
- dist = bernoulli(p)
86
- assert np.isclose(dist.mean(), p)
87
- assert np.isclose(dist.var(), p * (1 - p))
88
-
89
-
90
- def test_binomial():
91
- ns = np.arange(0, 10)
92
- ps = np.linspace(0, 1, 5)
93
- for n, p in product(ns, ps):
94
- dist = binomial(n, p)
95
- assert np.isclose(dist.mean(), n * p)
96
- assert np.isclose(dist.var(), n * p * (1 - p))
97
-
98
-
99
- def test_poisson():
100
- mus = np.linspace(0, 10, 5)
101
- for mu in mus:
102
- dist = poisson(mu)
103
- assert np.isclose(dist.mean(), mu)
104
- assert np.isclose(dist.var(), mu)
105
-
106
-
107
- @pytest.mark.filterwarnings("ignore:invalid value", "ignore:divide by zero")
108
- def test_hypergeom():
109
- ns = np.arange(0, 10)
110
- Ns = np.arange(0, 10) # noqa: N806
111
- Gs = np.arange(0, 10) # noqa: N806
112
- for n, N, G in product(ns, Ns, Gs): # noqa: N806
113
- if n >= N or G > N or n == 0 or N <= 1 or G == 0:
114
- with pytest.raises(AssertionError):
115
- dist = hypergeom(n, N, G)
116
- continue
117
- dist = hypergeom(n, N, G)
118
- assert np.isclose(dist.mean(), n * G / N)
119
- assert np.isclose(dist.var(), n * G / N * (N - G) / N * (N - n) / (N - 1))
120
-
121
-
122
- @pytest.mark.filterwarnings("ignore:divide by zero")
123
- def test_geom0():
124
- ps = np.linspace(0, 1, 5)[1:]
125
- for p in ps:
126
- dist = geom0(p)
127
- assert np.isclose(dist.mean(), (1 - p) / p)
128
- assert np.isclose(dist.var(), (1 - p) / p**2)
129
-
130
-
131
- @pytest.mark.filterwarnings("ignore:divide by zero")
132
- def test_geom1():
133
- ps = np.linspace(0, 1, 5)[1:]
134
- for p in ps:
135
- dist = geom1(p)
136
- assert np.isclose(dist.mean(), 1 / p)
137
- assert np.isclose(dist.var(), (1 - p) / p**2)
138
-
139
-
140
- @pytest.mark.filterwarnings("ignore:divide by zero")
141
- def test_nbinom():
142
- rs = np.arange(0, 10)[1:]
143
- ps = np.linspace(0, 1, 5)[1:]
144
- for r, p in product(rs, ps):
145
- dist = nbinom(r, p)
146
- assert np.isclose(dist.mean(), r * (1 - p) / p)
147
- assert np.isclose(dist.var(), r * (1 - p) / p**2)