ialdev-core 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- iad/core/__init__.py +9 -0
- iad/core/array.py +1961 -0
- iad/core/binary.py +377 -0
- iad/core/cache.py +903 -0
- iad/core/codetools.py +203 -0
- iad/core/datatools.py +671 -0
- iad/core/docs/locators.ipynb +754 -0
- iad/core/dotstyle.py +99 -0
- iad/core/env.py +271 -0
- iad/core/events.py +650 -0
- iad/core/filesproc.py +1046 -0
- iad/core/fnctools.py +390 -0
- iad/core/label.py +240 -0
- iad/core/logs.py +182 -0
- iad/core/nptools.py +449 -0
- iad/core/one_dark.puml +881 -0
- iad/core/param/__init__.py +17 -0
- iad/core/param/confargparse.py +55 -0
- iad/core/param/paramaze.py +339 -0
- iad/core/param/tbox.py +277 -0
- iad/core/paths.py +563 -0
- iad/core/pdtools.py +2570 -0
- iad/core/pydantools/__init__.py +5 -0
- iad/core/pydantools/fixed_pydantic_yaml/__init__.py +32 -0
- iad/core/pydantools/fixed_pydantic_yaml/compat/__init__.py +0 -0
- iad/core/pydantools/fixed_pydantic_yaml/compat/hacks.py +76 -0
- iad/core/pydantools/fixed_pydantic_yaml/compat/old_enums.py +37 -0
- iad/core/pydantools/fixed_pydantic_yaml/compat/representers.py +92 -0
- iad/core/pydantools/fixed_pydantic_yaml/compat/types.py +122 -0
- iad/core/pydantools/fixed_pydantic_yaml/compat/yaml_lib.py +104 -0
- iad/core/pydantools/fixed_pydantic_yaml/ext/__init__.py +1 -0
- iad/core/pydantools/fixed_pydantic_yaml/ext/semver.py +152 -0
- iad/core/pydantools/fixed_pydantic_yaml/ext/versioned_model.py +113 -0
- iad/core/pydantools/fixed_pydantic_yaml/main.py +30 -0
- iad/core/pydantools/fixed_pydantic_yaml/mixin.py +281 -0
- iad/core/pydantools/fixed_pydantic_yaml/model.py +20 -0
- iad/core/pydantools/fixed_pydantic_yaml/py.typed +1 -0
- iad/core/pydantools/fixed_pydantic_yaml/version.py +1 -0
- iad/core/pydantools/models.py +560 -0
- iad/core/regexp.py +348 -0
- iad/core/short.py +308 -0
- iad/core/strings.py +635 -0
- iad/core/unc_panda.py +270 -0
- iad/core/units.py +58 -0
- iad/core/wrap.py +420 -0
- ialdev_core-0.1.0.dist-info/METADATA +73 -0
- ialdev_core-0.1.0.dist-info/RECORD +48 -0
- ialdev_core-0.1.0.dist-info/WHEEL +4 -0
iad/core/array.py
ADDED
|
@@ -0,0 +1,1961 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
import warnings
|
|
5
|
+
import dataclasses as dcs
|
|
6
|
+
from enum import Enum
|
|
7
|
+
from typing import (Any, Literal, overload, Collection, Callable, Iterable, Tuple,
|
|
8
|
+
Union, Iterator, get_type_hints, get_args, Container)
|
|
9
|
+
import re
|
|
10
|
+
import numpy as np
|
|
11
|
+
import numba
|
|
12
|
+
from numbers import Number
|
|
13
|
+
|
|
14
|
+
from .codetools import NamedObj
|
|
15
|
+
from . import as_list
|
|
16
|
+
from . import nptools as npt
|
|
17
|
+
from .datatools import rm_keys
|
|
18
|
+
|
|
19
|
+
__all__ = ['DForm', 'Array', 'FormArray', 'form_array', 'Color', '_wC2G']
|
|
20
|
+
|
|
21
|
+
FLOAT = np.float32 # default float for images
|
|
22
|
+
_wC2G = np.array([0.2125, 0.7154, 0.0721], FLOAT).reshape(3, 1) # RGB -> Gray weights
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _cast_int(inp, out, round):
|
|
26
|
+
"""
|
|
27
|
+
Simple logic to handle clipping of inf values where casting float to int, where np.clip returns garbage.
|
|
28
|
+
Also for clipping when doing int -> int casting.
|
|
29
|
+
|
|
30
|
+
:param inp: input array
|
|
31
|
+
:param out: output array
|
|
32
|
+
:param round: If true, use round function of python, else let astype do the rounding (would be np.fix)
|
|
33
|
+
:return: array after casting
|
|
34
|
+
"""
|
|
35
|
+
out_type = out.dtype.type
|
|
36
|
+
inp_type = inp.dtype.type
|
|
37
|
+
|
|
38
|
+
lims = np.iinfo(out.dtype)
|
|
39
|
+
mn, mx = out_type(lims.min), out_type(lims.max)
|
|
40
|
+
inp = inp.reshape(inp.size)
|
|
41
|
+
out = out.reshape(inp.size)
|
|
42
|
+
if inp.dtype.kind == 'f':
|
|
43
|
+
inf, ninf = inp_type(np.inf), inp_type(-np.inf)
|
|
44
|
+
for i in range(inp.size):
|
|
45
|
+
if inp[i] > mx or inp[i] == inf:
|
|
46
|
+
out[i] = mx
|
|
47
|
+
elif inp[i] < mn or inp[i] == ninf:
|
|
48
|
+
out[i] = mn
|
|
49
|
+
elif round:
|
|
50
|
+
out[i] = out_type(np.round(inp[i]))
|
|
51
|
+
else:
|
|
52
|
+
out[i] = out_type(inp[i])
|
|
53
|
+
else:
|
|
54
|
+
for i in numba.prange(inp.size):
|
|
55
|
+
if inp[i] > mx:
|
|
56
|
+
out[i] = mx
|
|
57
|
+
elif inp[i] < mn:
|
|
58
|
+
out[i] = mn
|
|
59
|
+
else:
|
|
60
|
+
out[i] = out_type(inp[i])
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
_cast_parallel = numba.jit(nopython=True, fastmath=True, parallel=True)(_cast_int)
|
|
64
|
+
_cast_no_parallel = numba.jit(nopython=True, fastmath=True)(_cast_int)
|
|
65
|
+
_cast_parallel_size = 50000 # size to start parallel
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def cast(a: np.ndarray, dtype: Union[str, np.dtype],
|
|
69
|
+
*, out=None, copy=True, round=True):
|
|
70
|
+
"""
|
|
71
|
+
Casting function of arrays to better cast in problematic use-cases, as a replace for astype()
|
|
72
|
+
|
|
73
|
+
Casting could change value for different reasons:
|
|
74
|
+
1. range of the out type is narrower: i2(255:u2) = -1
|
|
75
|
+
2. precision of the output type is lower: f4(i4.max) - i4.max = 1
|
|
76
|
+
|
|
77
|
+
The improvement is by:
|
|
78
|
+
1. Clipping better for integers.
|
|
79
|
+
2. Rounding instead of np.fix for integers. # todo update optional
|
|
80
|
+
3. Allowing to include an output array.
|
|
81
|
+
|
|
82
|
+
By default,
|
|
83
|
+
- out of range values for float outputs are *automatically* rendered into +-inf.
|
|
84
|
+
- outputs values in integers and floats are clipped *if needed*.
|
|
85
|
+
- precision of float is decreasing when lowering the size of the float (as astype)
|
|
86
|
+
|
|
87
|
+
:param a: input array to be cast to a new type
|
|
88
|
+
:param dtype: dtype to cast
|
|
89
|
+
:param out: If exists, it is the array to be used as the output. It MUST be in the shape of the input, and in
|
|
90
|
+
the same dtype of the dtype requested, else casting would fail.
|
|
91
|
+
If None, a new array would be created.
|
|
92
|
+
:param round: When True, using the round function to round float numbers to integers.
|
|
93
|
+
If False, it uses the astype default strategy (np.fix)
|
|
94
|
+
Default is True.
|
|
95
|
+
:param copy: Relevant only to casting to same dtype, and when there isn't an "out" parameter:
|
|
96
|
+
If True, there is a new copy of the input. The default.
|
|
97
|
+
If False, the output is a view of the input and not a copy.
|
|
98
|
+
:return: Array that was cast to new dtype
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
dtype = np.dtype(dtype)
|
|
102
|
+
if out is not None and out.dtype != dtype:
|
|
103
|
+
raise TypeError(f"Missmatch of types between out type : {out.dtype} and cast to type {dtype}")
|
|
104
|
+
|
|
105
|
+
if a.dtype == dtype: # special cases where there is no changing in dtype
|
|
106
|
+
if out is not None and (a is out or shared_view(a, out)): # a and out have same view, and no casting
|
|
107
|
+
return out
|
|
108
|
+
elif not copy: # not copying - only if same type
|
|
109
|
+
if out is None:
|
|
110
|
+
return a
|
|
111
|
+
raise ValueError(f"Can't cast without copying when out array is present")
|
|
112
|
+
|
|
113
|
+
if dtype.kind == a.dtype.kind and dtype.itemsize >= a.dtype.itemsize: # only increasing size of same kind
|
|
114
|
+
if out is None:
|
|
115
|
+
return a.astype(dtype, casting='safe')
|
|
116
|
+
else:
|
|
117
|
+
np.copyto(out, a.astype(dtype, casting='safe'))
|
|
118
|
+
return out
|
|
119
|
+
else:
|
|
120
|
+
if dtype.kind == 'f' or np.can_cast(a, dtype,
|
|
121
|
+
casting='safe'): # inf values created automatically by astype
|
|
122
|
+
if out is None:
|
|
123
|
+
return a.astype(dtype, casting='unsafe')
|
|
124
|
+
else:
|
|
125
|
+
np.copyto(out, a.astype(dtype, casting='unsafe'))
|
|
126
|
+
return out
|
|
127
|
+
else: # int out or can't safely cast - int16 -> int8, float16 -> int32
|
|
128
|
+
if out is None: # create out if not included
|
|
129
|
+
out = np.empty_like(a, dtype=dtype)
|
|
130
|
+
cast_int = _cast_parallel if a.size > _cast_parallel_size else _cast_no_parallel
|
|
131
|
+
cast_int(a.astype('f4') if a.dtype == np.dtype('f2') else a, # numba doesn't support float16
|
|
132
|
+
out, round)
|
|
133
|
+
return out
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def shared_view(a: np.ndarray, b: np.ndarray, shape=True) -> bool:
|
|
137
|
+
"""Checks if two arrays share same data in the memory.
|
|
138
|
+
|
|
139
|
+
If shape is ``False`` - they may be of different shape, but size must match -
|
|
140
|
+
it does not deal with partial overlaps, and its main purpose to recognize if
|
|
141
|
+
one array is a full views of each over or both are views of some other array.
|
|
142
|
+
"""
|
|
143
|
+
if shape:
|
|
144
|
+
if a.shape != b.shape:
|
|
145
|
+
return False
|
|
146
|
+
elif a.size != b.size:
|
|
147
|
+
return False
|
|
148
|
+
return a.__array_interface__['data'][0] == b.__array_interface__['data'][0]
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _name_to_dtype(name: str) -> np.dtype | None:
|
|
152
|
+
"""
|
|
153
|
+
Return dtype for given string or None if not possible
|
|
154
|
+
:param name:
|
|
155
|
+
:return: dtype
|
|
156
|
+
"""
|
|
157
|
+
try:
|
|
158
|
+
return np.dtype(name)
|
|
159
|
+
except TypeError:
|
|
160
|
+
return None
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _dtype_str(t: np.dtype):
|
|
164
|
+
return f"{t.kind}{t.itemsize}"
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _collect_result_type(*v: np.ndarray | None):
|
|
168
|
+
"""Result type of multiple vector or scalar objects:
|
|
169
|
+
|
|
170
|
+
>>> _collect_result_type([1, 0,1, np.array([1,2,3])])
|
|
171
|
+
"""
|
|
172
|
+
v = (_ for _ in v if _ is not None)
|
|
173
|
+
return np.result_type(*map(np.min_scalar_type, v))
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
class LinT:
|
|
177
|
+
"""Helper class to implement linear transformation ``k*a+b`` over ndarray ``a``
|
|
178
|
+
with carefully managed type conversions and data copying
|
|
179
|
+
"""
|
|
180
|
+
|
|
181
|
+
def __init__(self, k=1., b=0.):
|
|
182
|
+
"""
|
|
183
|
+
:param k: initial k (a number of vector of ``chn`` size)
|
|
184
|
+
:param b: initial b (a number of vector of ``chn`` size)
|
|
185
|
+
"""
|
|
186
|
+
# self.channels = 1 if chn is None else chn
|
|
187
|
+
self.k, self.b = (np.array(v, dtype=float, ndmin=1) for v in (k, b))
|
|
188
|
+
|
|
189
|
+
@property
|
|
190
|
+
def identity(self):
|
|
191
|
+
"""True if transform does not change the data"""
|
|
192
|
+
return np.allclose(self.k, 1) and np.allclose(self.b, 0)
|
|
193
|
+
|
|
194
|
+
def __repr__(self):
|
|
195
|
+
one_or_all = lambda x: x[0] if np.all(x == x[0]) else x
|
|
196
|
+
k, b = map(one_or_all, (self.k, self.b))
|
|
197
|
+
return f"<{type(self).__name__}> [{k = }, {b = }]"
|
|
198
|
+
# f" chn: {self.channels}"
|
|
199
|
+
|
|
200
|
+
def _chain_mult(self, k, b):
|
|
201
|
+
return self.k * k, k * self.b + b
|
|
202
|
+
|
|
203
|
+
def __mul__(self, kb):
|
|
204
|
+
return type(self)(*self._chain_mult(*kb))
|
|
205
|
+
|
|
206
|
+
def __imul__(self, kb):
|
|
207
|
+
self.k, self.b = self._chain_mult(*kb)
|
|
208
|
+
return self
|
|
209
|
+
|
|
210
|
+
def apply(self, a: np.ndarray, *, out: np.ndarray) -> np.ndarray:
|
|
211
|
+
"""
|
|
212
|
+
To minimize unnecessary operations return input for equivalent transform.
|
|
213
|
+
Also perform multiplication with dtype defined by target dtype and the input.
|
|
214
|
+
|
|
215
|
+
:param a: array to apply the transform to
|
|
216
|
+
:param out: array for output results
|
|
217
|
+
"""
|
|
218
|
+
inp_type, out_type = a.dtype, out.dtype
|
|
219
|
+
|
|
220
|
+
k, b = map(npt.as_min_type, (self.k, self.b))
|
|
221
|
+
k_not_1, b_not_0 = not np.allclose(k, 1), not np.allclose(b, 0)
|
|
222
|
+
float_out = out_type.kind == 'f'
|
|
223
|
+
|
|
224
|
+
if not (k_not_1 or b_not_0): # no transformation - just casting
|
|
225
|
+
return cast(a, out_type, out=out)
|
|
226
|
+
|
|
227
|
+
if float_out: # transformation into float does not require clipping
|
|
228
|
+
tmp = np.multiply(a, self.k) if k_not_1 else a
|
|
229
|
+
tmp = np.add(tmp, self.b) if b_not_0 else tmp
|
|
230
|
+
return cast(tmp, out_type, out=out)
|
|
231
|
+
|
|
232
|
+
# determine dtype for operations
|
|
233
|
+
mlt_type = np.result_type(inp_type, k.dtype) if k_not_1 else inp_type
|
|
234
|
+
add_type = np.result_type(mlt_type, b.dtype) if b_not_0 else mlt_type
|
|
235
|
+
# upgrade dtype for possibly range expansion during operations (partially heuristics)
|
|
236
|
+
if add_type.kind != 'f' and add_type.itemsize <= 2: # todo we don't handle itemsize == 4 and increasing to 8?
|
|
237
|
+
add_type = np.dtype(f'i{add_type.itemsize * 2}')
|
|
238
|
+
tmp = out if add_type is out_type else np.empty(a.shape, add_type)
|
|
239
|
+
np.add(np.multiply(a, k, out=tmp) if k_not_1 else a, b, out=tmp)
|
|
240
|
+
return cast(tmp, out_type, out=out)
|
|
241
|
+
|
|
242
|
+
def __bool__(self):
|
|
243
|
+
return not self.identity
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
class Kind(Enum):
|
|
247
|
+
NA = None
|
|
248
|
+
UND = None
|
|
249
|
+
DISP = 'disp'
|
|
250
|
+
IMG = 'image'
|
|
251
|
+
IMAGE = 'image'
|
|
252
|
+
DEP = 'depth'
|
|
253
|
+
DEPTH = 'depth'
|
|
254
|
+
RGN = 'region'
|
|
255
|
+
REGION = 'region'
|
|
256
|
+
CNF = 'conf'
|
|
257
|
+
CONF = 'conf'
|
|
258
|
+
|
|
259
|
+
def __bool__(self):
|
|
260
|
+
return bool(self.value)
|
|
261
|
+
|
|
262
|
+
@classmethod
|
|
263
|
+
def from_str(cls, s: str):
|
|
264
|
+
"""Return Color instance from name if exists or return None"""
|
|
265
|
+
return cls.__members__.get(s.upper(), None) if isinstance(s, str) else None
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
class Color(Enum):
|
|
269
|
+
"""Enum listing differnt color codings of the channels."""
|
|
270
|
+
BIN = 'BIN'
|
|
271
|
+
GRAY = 'GRAY'
|
|
272
|
+
RGB = 'RGB'
|
|
273
|
+
B = 'BIN'
|
|
274
|
+
G = 'GRAY'
|
|
275
|
+
C = 'RGB'
|
|
276
|
+
|
|
277
|
+
def __bool__(self):
|
|
278
|
+
return bool(self.value)
|
|
279
|
+
|
|
280
|
+
@classmethod
|
|
281
|
+
def from_str(cls, s: str) -> str | None:
|
|
282
|
+
"""Return Color instance from name if exists or return None"""
|
|
283
|
+
return cls.__members__.get(s.upper(), None) if isinstance(s, str) else None
|
|
284
|
+
|
|
285
|
+
@property
|
|
286
|
+
def channels(self):
|
|
287
|
+
"""Number of color channels"""
|
|
288
|
+
if self is self.RGB: return 3
|
|
289
|
+
if self in (self.GRAY, self.BIN): return 1
|
|
290
|
+
raise RuntimeError("All cases must return channels number")
|
|
291
|
+
|
|
292
|
+
def short_code(self):
|
|
293
|
+
"""Return the shortest alias to the color"""
|
|
294
|
+
code = self.value
|
|
295
|
+
for name, v in self.__class__.__members__.items():
|
|
296
|
+
if self is v and len(name) < len(code): code = name
|
|
297
|
+
return code
|
|
298
|
+
|
|
299
|
+
def __str__(self):
|
|
300
|
+
return self.value
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def field(default=None, *, init=True, repr=True,
|
|
304
|
+
hash: bool | Callable[[Any], int] = None,
|
|
305
|
+
compare: bool | Callable[[Any, Any], bool] = True,
|
|
306
|
+
**metadata):
|
|
307
|
+
"""
|
|
308
|
+
Wraps and modifies ``dataclass.field`` interface.
|
|
309
|
+
|
|
310
|
+
Changes:
|
|
311
|
+
1. ``default`` argument is optional and is None by default
|
|
312
|
+
2. ``metadata`` is provided not as a argument but as a keyword arguments
|
|
313
|
+
2. ``hash`` may additionally be a function defining custom hash calculation
|
|
314
|
+
3. ``compare`` may additionally be a function defining custom comparison
|
|
315
|
+
|
|
316
|
+
``metadata`` will contain all the keyword arguments not defined in ``datclass.field``.
|
|
317
|
+
|
|
318
|
+
More, if ``hash`` and ``compare`` are Callables they are passed as ``True``
|
|
319
|
+
into ``dataclass.field``, but additionally stored in ``metadata`` with their
|
|
320
|
+
function values.
|
|
321
|
+
|
|
322
|
+
All that allows to simplify definition of DForm fields with extended functionality,
|
|
323
|
+
not only by supporting custom hashing and comparison, but also to
|
|
324
|
+
define entirely new functionality through metadata.
|
|
325
|
+
|
|
326
|
+
In particular ``calc`` keyword argument defines function to estimate field value from the data.
|
|
327
|
+
|
|
328
|
+
:param default
|
|
329
|
+
:param init: add as ``__init__`` argument
|
|
330
|
+
:param repr: include in ``__repr__``
|
|
331
|
+
:param hash: True to include or Callable to calc hash with
|
|
332
|
+
:param compare: True to include in comparison or Callable for special __eq__
|
|
333
|
+
:metadata: additional field attribute as described below:
|
|
334
|
+
|
|
335
|
+
:key calc: function to calculate field from data
|
|
336
|
+
|
|
337
|
+
:return:
|
|
338
|
+
"""
|
|
339
|
+
if isinstance(hash, Callable):
|
|
340
|
+
metadata['hash'], hash = hash, True
|
|
341
|
+
|
|
342
|
+
if isinstance(compare, Callable):
|
|
343
|
+
metadata['compare'], compare = compare, True
|
|
344
|
+
|
|
345
|
+
if 'default_factory' in metadata: # move from metadata into locals
|
|
346
|
+
default_factory = metadata.pop('default_factory')
|
|
347
|
+
|
|
348
|
+
if 'calc' in metadata and not isinstance(metadata['calc'], Callable):
|
|
349
|
+
raise ValueError("Argument `calc` must be a function")
|
|
350
|
+
|
|
351
|
+
return dcs.field(**locals())
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
FormKind = Union['DForm', np.dtype, str, type]
|
|
355
|
+
hash_array = lambda a: None if a is None else hash(tuple(a))
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
# noinspection PyShadowingBuiltins
|
|
359
|
+
@dcs.dataclass
|
|
360
|
+
class DForm:
|
|
361
|
+
"""
|
|
362
|
+
Provides extended description of *form* and *content* of data arrays.
|
|
363
|
+
|
|
364
|
+
In addition to the standard ``dtype`` field, identical to the ``ndarray.dtype``,
|
|
365
|
+
may optionally include information about
|
|
366
|
+
- statistics (``min, max, avr, std``)
|
|
367
|
+
- color information: (``color``, ``cax``)
|
|
368
|
+
|
|
369
|
+
Provides method to automatically ``transform`` array of different forms.
|
|
370
|
+
|
|
371
|
+
Also supplies convenience properties derived from this information:
|
|
372
|
+
- ``range`` (``max`` - ``min``)
|
|
373
|
+
- ``ndim`` - dimension of data if its describe a color image
|
|
374
|
+
- ``norm`` - ``(avr, std)`` how data was normalized
|
|
375
|
+
|
|
376
|
+
All those return ``None`` if not relevant.
|
|
377
|
+
|
|
378
|
+
Initialization
|
|
379
|
+
==============
|
|
380
|
+
All ``DForm`` arguments are initialized with ``None`` by default:
|
|
381
|
+
::
|
|
382
|
+
dtype: np.dtype or convertable into dtype
|
|
383
|
+
name: str | DForm,
|
|
384
|
+
min: float, max: float
|
|
385
|
+
avr: float | list[float], std: float | list[float]
|
|
386
|
+
color: DForm.Color, cax: int
|
|
387
|
+
|
|
388
|
+
Color
|
|
389
|
+
-----
|
|
390
|
+
Color is described by
|
|
391
|
+
- ``color: DForm.Color`` ``enum`` containing all the supportted color spaces
|
|
392
|
+
|
|
393
|
+
- ``cax`` - index of the color axis in the data dimensions
|
|
394
|
+
|
|
395
|
+
could be ``None`` also for ``color` like ``BIN``, ``GRAY``, wih dimensions ``[H x W]``,
|
|
396
|
+
|
|
397
|
+
unless additional color dimention is used ``[H x W x 1]``
|
|
398
|
+
|
|
399
|
+
Empty Form
|
|
400
|
+
----------
|
|
401
|
+
``dtype`` must be provided, except of the case of *empty* form:
|
|
402
|
+
|
|
403
|
+
>>> bool(DForm()) is False # empty form
|
|
404
|
+
|
|
405
|
+
Name and Registrattion
|
|
406
|
+
----------------------
|
|
407
|
+
If ``name`` string is provided, a resulting instance is automatiocally
|
|
408
|
+
registered under ``name.lower()``.
|
|
409
|
+
|
|
410
|
+
It then may be accessed using either:
|
|
411
|
+
- ``registered()`` query or
|
|
412
|
+
- constructor with name as only argument.
|
|
413
|
+
|
|
414
|
+
>>> dform = DForm('Cu1', dtype='uint8', color=DForm.Color.RGB)
|
|
415
|
+
>>> assert DForm.from_name('Cu1') is dform
|
|
416
|
+
>>> assert DForm('Cu1') == dform
|
|
417
|
+
|
|
418
|
+
Use ``DForm.list_registered()`` to list of all the registered forms.
|
|
419
|
+
Defining a ``DForm`` with same currently replaces the previous one with a warning.
|
|
420
|
+
Avoid that to prevent having different forms with same name.
|
|
421
|
+
|
|
422
|
+
Practical Examples
|
|
423
|
+
==================
|
|
424
|
+
Not any combination of the forms attributes are meaningfull.
|
|
425
|
+
Below are some typical cases.
|
|
426
|
+
|
|
427
|
+
Colored
|
|
428
|
+
------
|
|
429
|
+
Color requires either ``range``, or ``norm`` to be defined.
|
|
430
|
+
|
|
431
|
+
Range of float color is usually ``(0,1)`` or ``(-1,1)``, but not necessary.
|
|
432
|
+
|
|
433
|
+
(Curled brackets means every option is possible independent on the others)
|
|
434
|
+
::
|
|
435
|
+
|
|
436
|
+
color: {RGB, GRAY}
|
|
437
|
+
- dtype: u8 # unsigned 8 bits per channel
|
|
438
|
+
range: (0, 255) # type based range
|
|
439
|
+
|
|
440
|
+
- dtype: u16 # unsigned 16 bits
|
|
441
|
+
range: {(0, 1023), # specific (10 bits for example)
|
|
442
|
+
(0, 65535)} # type based
|
|
443
|
+
|
|
444
|
+
- dtype: i16
|
|
445
|
+
- range: {(-32768, 32767), (-32768, 32767), ...}
|
|
446
|
+
|
|
447
|
+
- dtype: {f16, f32 f64}
|
|
448
|
+
range: {(0, 1), (-1, 1)}
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
Supported basic transformations
|
|
452
|
+
-------------------------------
|
|
453
|
+
As ``DForm`` extends ``dtype`` so this transform extends notion of casting:
|
|
454
|
+
- only obvious and safe data manipulations are applied automatically,
|
|
455
|
+
like range mappings, rescaling, etc,
|
|
456
|
+
- aim to safe type casting keeping data values, except of type range clipping.
|
|
457
|
+
|
|
458
|
+
Those basic kind of transformations can be easily defined separately:
|
|
459
|
+
1. range -> range
|
|
460
|
+
2. normalization (avr, std) -> (avr, std)
|
|
461
|
+
3. color -> color
|
|
462
|
+
4. dtype -> dtype
|
|
463
|
+
|
|
464
|
+
All the transformations (except of dtype) are performed if the corresponding attribute is
|
|
465
|
+
defined in both source and target DataForms.
|
|
466
|
+
|
|
467
|
+
Missing input attribute makes transformation impossible and it fails.
|
|
468
|
+
|
|
469
|
+
Missing output attribute keeps data (in its respect) unchanged, however
|
|
470
|
+
it could be changed because of another attributes.
|
|
471
|
+
|
|
472
|
+
In this example:
|
|
473
|
+
``range, color:RGB -> color:GRAY``
|
|
474
|
+
input range is irrelevant, but the color transformation does occur.
|
|
475
|
+
|
|
476
|
+
Implementation tries to minimize number of calculations and data copy.
|
|
477
|
+
That can be facilitated even farther by providing the ``out`` argument
|
|
478
|
+
with array to write the data into.
|
|
479
|
+
|
|
480
|
+
This array must match both the source dimensions and target form.
|
|
481
|
+
If possible it is used along all the transformation stages to reduce copying.
|
|
482
|
+
|
|
483
|
+
Supported Combinations
|
|
484
|
+
======================
|
|
485
|
+
Only some of the combinations of the attributes in the source and target forms can be allowed.
|
|
486
|
+
|
|
487
|
+
DType
|
|
488
|
+
~~~~~
|
|
489
|
+
``dtype`` is always defined for both source and target forms, and any conversion is possible
|
|
490
|
+
together with allowed transform of any other attributes.
|
|
491
|
+
|
|
492
|
+
Range
|
|
493
|
+
~~~~~
|
|
494
|
+
Mathematically ``range`` is incompatible with ``normalization``, therefore
|
|
495
|
+
- ``range, norm -> range norm`` is not allowed
|
|
496
|
+
- ``range -> std`` is possible however
|
|
497
|
+
- ``std -> range`` is not allowed since the initial range is not known
|
|
498
|
+
|
|
499
|
+
When automatic transform is not possible, a ``TypeError`` is raised.
|
|
500
|
+
|
|
501
|
+
That sometimes can be avoided if desired transform is explicitly enabled or disabled using
|
|
502
|
+
``do_range`` and ``do_norm`` arguments. Then exception is raised if a specific transform(s)
|
|
503
|
+
are explicitly enabled but is not possible.
|
|
504
|
+
|
|
505
|
+
Color
|
|
506
|
+
~~~~~
|
|
507
|
+
``color`` can be used in different combinations:
|
|
508
|
+
::
|
|
509
|
+
|
|
510
|
+
color, range -> color, range
|
|
511
|
+
color, range -> color, norm
|
|
512
|
+
color, norm -> color, norm
|
|
513
|
+
|
|
514
|
+
"""
|
|
515
|
+
|
|
516
|
+
_reg_names = {}
|
|
517
|
+
_reg_hashes = {}
|
|
518
|
+
|
|
519
|
+
_auto_dtype = True # allow guessing dtype when not provided
|
|
520
|
+
_auto_names = True
|
|
521
|
+
_default_dtype = np.dtype('f4')
|
|
522
|
+
_verify_cache = False or sys.gettrace() # for debugging. FIXME: remove after freeze
|
|
523
|
+
|
|
524
|
+
@classmethod
|
|
525
|
+
def auto_naming(cls, state: bool = None):
|
|
526
|
+
"""
|
|
527
|
+
Controls over possibility to auto-generate ``DForms`` using name coding conventions
|
|
528
|
+
|
|
529
|
+
If turned ON (default) than accessing a ``DForm`` by name following this convention, instead of
|
|
530
|
+
raising an error, generates and registers it automatically.
|
|
531
|
+
|
|
532
|
+
:param state:True - AutoNaming On (default),
|
|
533
|
+
False - AutoNaming Off.
|
|
534
|
+
None - for getting the current mode.
|
|
535
|
+
:return: the current mode if `state` is ``None`` or ``None``
|
|
536
|
+
|
|
537
|
+
Examples:
|
|
538
|
+
>>> DForm('CNu1')
|
|
539
|
+
Works
|
|
540
|
+
>>> DForm.auto_naming(False)
|
|
541
|
+
>>> DForm('CNu2')
|
|
542
|
+
KeyError: "No registered DForm named 'CNu2'"
|
|
543
|
+
"""
|
|
544
|
+
if state is None:
|
|
545
|
+
return cls._auto_names
|
|
546
|
+
cls._auto_names = state
|
|
547
|
+
return None
|
|
548
|
+
|
|
549
|
+
@classmethod
|
|
550
|
+
def _field_type(cls, fld, allowed: Container) -> type:
|
|
551
|
+
"""Found in the field defintion first type listed in `allow` or raise ``TypeError``.
|
|
552
|
+
"""
|
|
553
|
+
if not getattr(cls, '_fields_types', None):
|
|
554
|
+
cls._fields_types = get_type_hints(cls)
|
|
555
|
+
|
|
556
|
+
_type = cls._fields_types[fld]
|
|
557
|
+
type_options = [_type] if isinstance(_type, type) else get_args(_type)
|
|
558
|
+
for _type in allowed: # find first!
|
|
559
|
+
if _type in type_options:
|
|
560
|
+
return _type
|
|
561
|
+
raise TypeError(f"Neither {type_options=} for field {fld} are {allowed=}")
|
|
562
|
+
|
|
563
|
+
@classmethod
|
|
564
|
+
def _parse_name(cls, name: str) -> dict | None:
|
|
565
|
+
"""
|
|
566
|
+
Parse through the name, validate and return fields as ``dict``"
|
|
567
|
+
|
|
568
|
+
:param name: The name of dform for parsing
|
|
569
|
+
:return: dict of all attributes of dform
|
|
570
|
+
|
|
571
|
+
**The Pattern**:
|
|
572
|
+
::
|
|
573
|
+
[C|G|B][R[<min>:]<max>][N[<std>][|<avr>]]((u|i|f)<bytes>|(s|b)<bits>)
|
|
574
|
+
|
|
575
|
+
**Conventions**:
|
|
576
|
+
::
|
|
577
|
+
(x|y|...) - selection group
|
|
578
|
+
[] - optional group
|
|
579
|
+
<name> - group containing value of variable `name`
|
|
580
|
+
|
|
581
|
+
C - var:color = 'RGB'
|
|
582
|
+
G - var:color = 'Grey'
|
|
583
|
+
B - var:color = 'Binary'
|
|
584
|
+
|
|
585
|
+
**Group ideyntification symbols starting definition of**:
|
|
586
|
+
::
|
|
587
|
+
R - Range
|
|
588
|
+
N - Normalization
|
|
589
|
+
|
|
590
|
+
**Defaults for missing groups**:
|
|
591
|
+
::
|
|
592
|
+
R<max> = R<0>:<max>
|
|
593
|
+
N = N<1>|<0>
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
The only *mandatory* group is for **type**, which may be given in two forms:
|
|
597
|
+
- as number of bits prefixed by 'b' or 's' (for signed) \n
|
|
598
|
+
only for `int` types, contaner ``dtype`` is deduced from that
|
|
599
|
+
- using ``np.dtype`` shortcut notations (u1 = uint8, f8 = float64, ...)
|
|
600
|
+
|
|
601
|
+
**Limitations**:
|
|
602
|
+
- Parameters of `range` and `normalization` are represented by ``float``,
|
|
603
|
+
which can be parsed only in this form:
|
|
604
|
+
::
|
|
605
|
+
[-][<digits>.]<digits>
|
|
606
|
+
|
|
607
|
+
Though `DForm` supports defining normalization *per channel*,
|
|
608
|
+
that is **not** supported by the name parser.
|
|
609
|
+
|
|
610
|
+
Examples:
|
|
611
|
+
>>> DForm('CNu1') == DForm(color='RGB', std=1, avr=0, dtype=np.uint8)
|
|
612
|
+
True
|
|
613
|
+
>>> DForm('N2.5|-3f2') == DForm(std=2.5, avr=-3, dtype=np.float16)
|
|
614
|
+
True
|
|
615
|
+
>>> DForm('R1N|1.2f4') == DForm(min=0, max=1, avr=1.2, dtype=np.float32)
|
|
616
|
+
True
|
|
617
|
+
>>> DForm('Gb10') == DForm(color=Color.GRAY, min=0, max=1023, dtype=np.uint16)
|
|
618
|
+
True
|
|
619
|
+
>>> DForm('s2') == DForm(min=-1, max=1, dtype='i1')
|
|
620
|
+
True
|
|
621
|
+
"""
|
|
622
|
+
|
|
623
|
+
_codes_rex = re.compile(r"""
|
|
624
|
+
(?P<color>[GCB])?
|
|
625
|
+
(?:R
|
|
626
|
+
(?:
|
|
627
|
+
(?P<min>-?\d*\.?\d+):
|
|
628
|
+
)?
|
|
629
|
+
(?P<max>-?\d*\.?\d+)
|
|
630
|
+
)?
|
|
631
|
+
(?P<norm>N
|
|
632
|
+
(?P<std>\d*\.?\d+)?
|
|
633
|
+
(?:\|
|
|
634
|
+
(?P<avr>-?\d*\.?\d+)
|
|
635
|
+
)?
|
|
636
|
+
)?
|
|
637
|
+
(?:
|
|
638
|
+
(?P<bs>[bs])(?P<bits>[1-9]\d*) |
|
|
639
|
+
(?P<dtype>[uif][1248])
|
|
640
|
+
)
|
|
641
|
+
""", re.VERBOSE)
|
|
642
|
+
|
|
643
|
+
def set_default(fld, val):
|
|
644
|
+
if attr[fld] is None:
|
|
645
|
+
attr[fld] = val
|
|
646
|
+
|
|
647
|
+
def cast_numeric_attr(fld):
|
|
648
|
+
if attr[fld] is not None:
|
|
649
|
+
attr[fld] = cls._field_type(fld, (float, int))(attr[fld])
|
|
650
|
+
return attr[fld]
|
|
651
|
+
|
|
652
|
+
if (m := _codes_rex.fullmatch(name)) and (attr := m.groupdict()):
|
|
653
|
+
# shortcuts
|
|
654
|
+
if attr['max'] is not None: # R10 -> R0:10
|
|
655
|
+
set_default('min', 0)
|
|
656
|
+
if attr['norm']: # N -> N1 -> N1|0
|
|
657
|
+
set_default('std', 1)
|
|
658
|
+
set_default('avr', 0)
|
|
659
|
+
|
|
660
|
+
r_min, r_max, *_ = map(cast_numeric_attr, ['min', 'max', 'std', 'avr', 'bits'])
|
|
661
|
+
|
|
662
|
+
if bits_code := attr.pop('bs'):
|
|
663
|
+
if (bits := attr['bits']) > 64:
|
|
664
|
+
raise ValueError(f"Invalid number of {bits=}")
|
|
665
|
+
if bits_code == 'b': # unsigned of size bits
|
|
666
|
+
t_min, t_max = 0, 2 ** bits - 1
|
|
667
|
+
attr['dtype'] = np.min_scalar_type(t_max)
|
|
668
|
+
else: # s - signed
|
|
669
|
+
t_min, t_max = -2 ** (bits - 1), 2 ** (bits - 1) - 1
|
|
670
|
+
attr['dtype'] = np.min_scalar_type(t_min)
|
|
671
|
+
else: # if not a bits code, then dtype is defined
|
|
672
|
+
dtype = attr['dtype'] = np.dtype(attr['dtype']) # ensure valid dtype
|
|
673
|
+
t_info = (np.finfo if dtype.kind == 'f' else np.iinfo)(dtype)
|
|
674
|
+
t_min, t_max = t_info.min, t_info.max
|
|
675
|
+
|
|
676
|
+
if not ((r_min is None or r_min >= t_min) and (
|
|
677
|
+
r_max is None or r_max <= t_max)):
|
|
678
|
+
raise ValueError(f"Range ({r_min},{r_max}) is inconsistent with {bits_code}{bits}")
|
|
679
|
+
|
|
680
|
+
attr['color'] = Color.from_str(attr['color'])
|
|
681
|
+
|
|
682
|
+
# remove parsed keys which are not valid fields
|
|
683
|
+
return rm_keys(attr, set(attr).difference(cls.fields()))
|
|
684
|
+
|
|
685
|
+
return None
|
|
686
|
+
|
|
687
|
+
@classmethod
|
|
688
|
+
def _register(cls, dform):
|
|
689
|
+
""" Add to the global registry """
|
|
690
|
+
h = hash(dform)
|
|
691
|
+
if found := cls._reg_hashes.get(h, None):
|
|
692
|
+
if dform.name and found.name and found.name != dform.name:
|
|
693
|
+
warnings.warn(f"DForm {dform} is already known under {found.name}")
|
|
694
|
+
|
|
695
|
+
if dform.name:
|
|
696
|
+
key = hash(dform.name.lower())
|
|
697
|
+
if found := cls._reg_names.get(key, None):
|
|
698
|
+
warnings.warn(f"{cls.__name__}('{dform.name}') overrides registered {found.name}")
|
|
699
|
+
cls._reg_names[key] = dform
|
|
700
|
+
|
|
701
|
+
@classmethod
|
|
702
|
+
def from_name(cls, name: str, *, auto: bool = None, reg=True) -> DForm:
|
|
703
|
+
"""
|
|
704
|
+
Check if the name is registered as a dform in the registered list.
|
|
705
|
+
If the name is not registered, AutoNaming is On, and the name have values other than None,
|
|
706
|
+
it includes it in the registered list and return the dform created
|
|
707
|
+
Else we return the dform if it was found, or None if it wasn't found
|
|
708
|
+
:param name: The dform to check
|
|
709
|
+
:param reg: look into the registry
|
|
710
|
+
:param auto: allow automatic creation from name
|
|
711
|
+
:return: dform or None
|
|
712
|
+
"""
|
|
713
|
+
if reg and (found := cls._reg_names.get(hash(name.lower()), None)):
|
|
714
|
+
return found
|
|
715
|
+
if auto is True or auto is None and cls._auto_names:
|
|
716
|
+
if attrs := cls._parse_name(name):
|
|
717
|
+
return DForm(name=name, **attrs)
|
|
718
|
+
|
|
719
|
+
@classmethod
|
|
720
|
+
def list_registered(cls, out=False) -> dict[int, DForm] | None:
|
|
721
|
+
"""
|
|
722
|
+
Print list of the registered DataForms.
|
|
723
|
+
|
|
724
|
+
:param out: if True just silently return registration dict
|
|
725
|
+
:return: None or dict
|
|
726
|
+
"""
|
|
727
|
+
if out:
|
|
728
|
+
return cls._reg_names
|
|
729
|
+
|
|
730
|
+
for i, (k, v) in enumerate(cls._reg_names.items()):
|
|
731
|
+
print(f"{i:3}. {v.name:20}: {v}")
|
|
732
|
+
return None
|
|
733
|
+
|
|
734
|
+
class Shaper:
|
|
735
|
+
def __init__(self, src_shape, src_form: DForm, trg_form: DForm):
|
|
736
|
+
"""
|
|
737
|
+
Summary of possible color options and resulting re-shaping:
|
|
738
|
+
::
|
|
739
|
+
case src trg re-shape
|
|
740
|
+
chn cax chn cax
|
|
741
|
+
0 - - + error
|
|
742
|
+
1 - - - - =
|
|
743
|
+
2 + - - =
|
|
744
|
+
3 1 - 1 - =
|
|
745
|
+
4 1 - 1 + + dim
|
|
746
|
+
5 1 - 3 + + dim
|
|
747
|
+
6 1 + 1 - - dim
|
|
748
|
+
7 3 + 1 - - dim
|
|
749
|
+
8 1 + 1 + ? move
|
|
750
|
+
9 3 + 1 + ? move
|
|
751
|
+
10 1 + 3 + ? move
|
|
752
|
+
|
|
753
|
+
:param src_shape:
|
|
754
|
+
:param src_form:
|
|
755
|
+
:param trg_form:
|
|
756
|
+
"""
|
|
757
|
+
# start with 'no shape changes' configuration:
|
|
758
|
+
self.src_shape = self.trg_shape = src_shape
|
|
759
|
+
self.channels, self.cax = None, None
|
|
760
|
+
# if that is actually the case - exit
|
|
761
|
+
if not (src_form and trg_form and src_form.color and trg_form.color): return
|
|
762
|
+
src_chn, trg_chn = src_form.color.channels, trg_form.color.channels
|
|
763
|
+
if src_chn == trg_chn and src_form.cax == trg_form.cax: return
|
|
764
|
+
|
|
765
|
+
self.channels = trg_chn != src_chn and (src_chn, trg_chn)
|
|
766
|
+
self.cax = trg_form.cax != src_form.cax and (src_form.cax, trg_form.cax)
|
|
767
|
+
|
|
768
|
+
shape = list(src_shape) # evaluate target shape
|
|
769
|
+
if src_form.cax is not None: shape.pop(src_form.cax)
|
|
770
|
+
if trg_form.cax is not None:
|
|
771
|
+
shape.insert(trg_form.cax % (len(shape) + 1), trg_chn)
|
|
772
|
+
self.trg_shape = tuple(shape)
|
|
773
|
+
|
|
774
|
+
def __bool__(self):
|
|
775
|
+
return self.src_shape != self.trg_shape
|
|
776
|
+
|
|
777
|
+
def move_cax(self, a: np.ndarray, src_trg: Literal['src', 'trg'],
|
|
778
|
+
state: bool):
|
|
779
|
+
""" Move cax (if needed) of the array
|
|
780
|
+
|
|
781
|
+
:param a: data array to move axis
|
|
782
|
+
:param src_trg: which data is passed: 'src' or 'trg'
|
|
783
|
+
:param state: ``True`` cax to the end, ``False`` - restore
|
|
784
|
+
:return: array with cax moved
|
|
785
|
+
"""
|
|
786
|
+
if self.cax:
|
|
787
|
+
src_trg = src_trg == 'trg'
|
|
788
|
+
chn = self.channels[src_trg]
|
|
789
|
+
cax = self.cax[src_trg] % chn if self.cax[src_trg] else None
|
|
790
|
+
|
|
791
|
+
if cax and cax != (end := chn - 1):
|
|
792
|
+
from_to = (cax, end) if state else (end, cax)
|
|
793
|
+
return np.moveaxis(a, *from_to)
|
|
794
|
+
return a
|
|
795
|
+
|
|
796
|
+
CALC = NamedObj('CALC')
|
|
797
|
+
|
|
798
|
+
# =========================================================================
|
|
799
|
+
name: FormKind | None = field(hash=False, compare=False)
|
|
800
|
+
dtype: Any = None
|
|
801
|
+
bits: int = None
|
|
802
|
+
min: float = field(calc=np.nanmin)
|
|
803
|
+
max: float = field(calc=np.nanmax)
|
|
804
|
+
avr: float | np.ndarray | Collection[float] = field(hash=hash_array, calc=np.nanmean)
|
|
805
|
+
std: float | np.ndarray | Collection[float] = field(hash=hash_array, calc=np.nanstd)
|
|
806
|
+
color: Color | str = None
|
|
807
|
+
cax: int = None # IDEA: turn into axis: ordered list of axis names
|
|
808
|
+
units: float = None # IDEA: add support for physical units
|
|
809
|
+
kind: Kind = None
|
|
810
|
+
|
|
811
|
+
# ==========================================================================
|
|
812
|
+
|
|
813
|
+
@classmethod
|
|
814
|
+
def _meta(cls, fld: str, key: str, default=None):
|
|
815
|
+
"""
|
|
816
|
+
Return key's value from the metadata of a given field
|
|
817
|
+
:param fld: name of the field
|
|
818
|
+
:param key: meta-key
|
|
819
|
+
:param default: return this if key is not in metadata
|
|
820
|
+
"""
|
|
821
|
+
return cls.__dataclass_fields__[fld].metadata.get(key, default)
|
|
822
|
+
|
|
823
|
+
@classmethod
|
|
824
|
+
def _filter_apply_fields(cls, fld_proc: Callable[[dcs.Field], Tuple[Any, bool]], *,
|
|
825
|
+
include: str | Collection[str] | None = None,
|
|
826
|
+
exclude: str | Collection[str] | None = None):
|
|
827
|
+
"""
|
|
828
|
+
Generator yields first item returned by applying given `fld_proc` function
|
|
829
|
+
on all or some of the dataclass fields.
|
|
830
|
+
|
|
831
|
+
Function `fld_proc` must accept `Field` object and return a tuple `(result, accept)`:
|
|
832
|
+
- `result` to be yielded ONLY if
|
|
833
|
+
- `accept` is True
|
|
834
|
+
|
|
835
|
+
Function is applied either *all* the dataclass fields, unless *one* of `include` or `exclude`
|
|
836
|
+
arguments says differently (``NameError`` raised if invalid field name is given)
|
|
837
|
+
|
|
838
|
+
:param fld_proc:
|
|
839
|
+
:param include: fields to include
|
|
840
|
+
:param exclude:
|
|
841
|
+
"""
|
|
842
|
+
cls_fields = cls.__dataclass_fields__
|
|
843
|
+
|
|
844
|
+
if include:
|
|
845
|
+
if exclude:
|
|
846
|
+
raise ValueError("Both include AND exclude args are provided!")
|
|
847
|
+
|
|
848
|
+
include = as_list(include, collect=set)
|
|
849
|
+
if inv := include.difference(cls_fields):
|
|
850
|
+
raise NameError(f"Invalid fields: {inv}")
|
|
851
|
+
fields = include
|
|
852
|
+
elif exclude:
|
|
853
|
+
exclude = as_list(exclude, collect=set)
|
|
854
|
+
if inv := exclude.difference(cls_fields):
|
|
855
|
+
raise NameError(f"Invalid fields: {inv}")
|
|
856
|
+
fields = set(cls_fields) - exclude
|
|
857
|
+
else:
|
|
858
|
+
fields = cls_fields
|
|
859
|
+
|
|
860
|
+
for name in fields:
|
|
861
|
+
val, use = fld_proc(cls_fields[name])
|
|
862
|
+
if use:
|
|
863
|
+
yield val
|
|
864
|
+
|
|
865
|
+
@classmethod
|
|
866
|
+
def field_calc(cls, fld: str):
|
|
867
|
+
return cls._meta(fld, 'calc')
|
|
868
|
+
|
|
869
|
+
@classmethod
|
|
870
|
+
def from_data(cls, a: np.ndarray, /, name: str = None, *,
|
|
871
|
+
dtype: np.dtype = None, **attrs) -> DForm:
|
|
872
|
+
"""
|
|
873
|
+
Create DForm by estimating some attributes from given ``data``, which
|
|
874
|
+
must be either ``ndarray`` or iterable convertible into it.
|
|
875
|
+
|
|
876
|
+
In this case its ``dtype`` is first defined in this order:
|
|
877
|
+
``dform`` argument, ``dform.dtype``, ``DForm._default_dtype``
|
|
878
|
+
|
|
879
|
+
Attributes values may be inherited from given dform if provided.
|
|
880
|
+
That can be overriden by providing corresponding argument,
|
|
881
|
+
which could have different meanings:
|
|
882
|
+
- ``None``: not provided
|
|
883
|
+
- <value>: use it for the attribute
|
|
884
|
+
- "calculate": estimate from the data using internally defined function
|
|
885
|
+
- `Callable`: estimate from the data using this function
|
|
886
|
+
|
|
887
|
+
If attribute value is not defined by the argument and ``dform`` is ``None``,
|
|
888
|
+
it remains None, except of `dtype` which is then assigned `a.dtype``.
|
|
889
|
+
|
|
890
|
+
:param a: ndarray to use for calculations
|
|
891
|
+
:param dform: Provides base to inherit the attributes
|
|
892
|
+
:param dtype:
|
|
893
|
+
:param name: Optional name for new DForm
|
|
894
|
+
:keyword min: min value of the range
|
|
895
|
+
:keyword max: max value of the range
|
|
896
|
+
:keyword std: std of the data distribution
|
|
897
|
+
:keyword avr: avr of the data distribution
|
|
898
|
+
:keyword color: one of the `DForm.Color`
|
|
899
|
+
:keyword cax: ``int`` - color axes location (if color is defined)
|
|
900
|
+
:return: resulting dform
|
|
901
|
+
"""
|
|
902
|
+
if not isinstance(a, np.ndarray):
|
|
903
|
+
a = np.array(a, dtype=dtype)
|
|
904
|
+
if not dtype:
|
|
905
|
+
dtype = a.dtype
|
|
906
|
+
|
|
907
|
+
def value(fld, v):
|
|
908
|
+
if v == 'calculate':
|
|
909
|
+
if calc := cls._meta(fld, 'calc'):
|
|
910
|
+
return calc(a)
|
|
911
|
+
raise NotImplementedError(f"Calc function not defined for '{fld}'")
|
|
912
|
+
if isinstance(v, Callable):
|
|
913
|
+
return v(a)
|
|
914
|
+
return v
|
|
915
|
+
|
|
916
|
+
attrs = {k: value(k, v) for k, v in attrs.items()}
|
|
917
|
+
|
|
918
|
+
return DForm(name=name, dtype=dtype, **attrs)
|
|
919
|
+
|
|
920
|
+
@classmethod
|
|
921
|
+
def from_merge(cls, dform: DForm | str | type = None, **fields) -> DForm:
|
|
922
|
+
"""Create a new object by replacing one with new fields values.
|
|
923
|
+
|
|
924
|
+
- If ``dform`` is omitted, use fields defaults.
|
|
925
|
+
- If ``fields`` are *not provided either*, return ``None``!
|
|
926
|
+
|
|
927
|
+
:param dform to merge with in any form acceptable by DForm constructor
|
|
928
|
+
:param fields: any subset of valid DForm fields
|
|
929
|
+
"""
|
|
930
|
+
if dform:
|
|
931
|
+
if not isinstance(dform, cls):
|
|
932
|
+
dform = cls(dform)
|
|
933
|
+
if not fields:
|
|
934
|
+
return dform
|
|
935
|
+
fields = dict(dform.items()) | fields
|
|
936
|
+
if fields:
|
|
937
|
+
return DForm(**fields)
|
|
938
|
+
raise ValueError("No arguments in from_merge")
|
|
939
|
+
|
|
940
|
+
def type_info(self) -> str:
|
|
941
|
+
base = f"{self.name or ''}{_dtype_str(self.dtype)}"
|
|
942
|
+
if not self.extra_info():
|
|
943
|
+
return f"☉{base}"
|
|
944
|
+
rng = "" if self.min is None else f"⎣{self.min:g}➔{self.max:g}⌉"
|
|
945
|
+
vals = lambda s: ', '.join(f"{_:.3g}" for _ in (s if hasattr(s, '__len__') else [s]))
|
|
946
|
+
avr = "" if self.avr is None else f"<{vals(self.avr)}>"
|
|
947
|
+
std = f"" if self.std is None else f"σ{vals(self.std)}"
|
|
948
|
+
return f"Ⓕ{base}{rng}{avr}{std}"
|
|
949
|
+
|
|
950
|
+
def __repr__(self):
|
|
951
|
+
if self.name and self.name == self._code_name():
|
|
952
|
+
return 'Ⓕ' + self.name
|
|
953
|
+
return self.type_info()
|
|
954
|
+
|
|
955
|
+
def same(self, other: FormKind, *, name: bool | Literal['case'] = 'case'):
|
|
956
|
+
"""
|
|
957
|
+
Return ``True`` if other is the same as self including name.
|
|
958
|
+
Default comparison of names is case-sensitive, use ``True`` otherwise.
|
|
959
|
+
|
|
960
|
+
:param other: dform to compare with
|
|
961
|
+
:param name: 'case' - case-sensitive when comparing names. The default.
|
|
962
|
+
True - compare names. False - don't.
|
|
963
|
+
:return: bool
|
|
964
|
+
"""
|
|
965
|
+
# Consider: replace that by hash comparison once DForm is frozen
|
|
966
|
+
cls = type(self)
|
|
967
|
+
if not isinstance(other, cls):
|
|
968
|
+
# noinspection PyBroadException
|
|
969
|
+
try:
|
|
970
|
+
other = cls(other) # creates default dform (f4) if dform of 'other' is None
|
|
971
|
+
except:
|
|
972
|
+
return False
|
|
973
|
+
|
|
974
|
+
if name not in (allowed := (True, False, 'case')):
|
|
975
|
+
raise ValueError(f'name argument {name} is not among {allowed=}')
|
|
976
|
+
if name and not (
|
|
977
|
+
self.name == other.name if (name is True or self.name is None or other.name is None) else
|
|
978
|
+
self.name.lower() == other.name.lower()
|
|
979
|
+
): return False
|
|
980
|
+
|
|
981
|
+
def not_equal(f):
|
|
982
|
+
"""Return only yield-able False"""
|
|
983
|
+
|
|
984
|
+
def size(v):
|
|
985
|
+
return 0 if v is None else 1 if not isinstance(v, np.ndarray) else len(v)
|
|
986
|
+
|
|
987
|
+
if f.compare:
|
|
988
|
+
v1, v2 = getattr(self, f.name), getattr(other, f.name)
|
|
989
|
+
size_v1 = size(v1)
|
|
990
|
+
size_v2 = size(v2) # Consider: cmp() is not enough?
|
|
991
|
+
if (cmp := f.metadata.get('compare', None)) and cmp(v1, v2) or \
|
|
992
|
+
(size_v1 == size_v2 and size_v1 > 1 and (v1 == v2).all()) or \
|
|
993
|
+
(size_v1 == size_v2 and size_v1 <= 1 and v1 == v2): # 1D
|
|
994
|
+
return None, False # don't yield (None not to be used)
|
|
995
|
+
else:
|
|
996
|
+
return False, True # yield False
|
|
997
|
+
return None, False # don't yield
|
|
998
|
+
|
|
999
|
+
for _ in self._filter_apply_fields(not_equal): return _
|
|
1000
|
+
return True
|
|
1001
|
+
|
|
1002
|
+
def __eq__(self, other: DForm):
|
|
1003
|
+
return self.same(other, name=False)
|
|
1004
|
+
|
|
1005
|
+
def _calc_hash(self):
|
|
1006
|
+
d = self.__dict__
|
|
1007
|
+
|
|
1008
|
+
def hash_filter(f):
|
|
1009
|
+
if f.hash is False:
|
|
1010
|
+
return None, False
|
|
1011
|
+
|
|
1012
|
+
v = d[f.name]
|
|
1013
|
+
if v is not None and (hash_fnc := f.metadata.get('hash', None)):
|
|
1014
|
+
v = hash_fnc(v)
|
|
1015
|
+
return v, True
|
|
1016
|
+
|
|
1017
|
+
return hash(tuple(self._filter_apply_fields(fld_proc=hash_filter)))
|
|
1018
|
+
|
|
1019
|
+
def __hash__(self):
|
|
1020
|
+
found_hash = getattr(self, '_cached_hash', None)
|
|
1021
|
+
if not found_hash or self._verify_cache: # FIXME: freeze DForm and remove verifications
|
|
1022
|
+
now_hash = self._calc_hash()
|
|
1023
|
+
if found_hash and now_hash != found_hash: # can be true only if self.__verify_cache
|
|
1024
|
+
raise RuntimeError(f"Inconsistent cache for {self}")
|
|
1025
|
+
self._cached_hash = now_hash
|
|
1026
|
+
|
|
1027
|
+
return self._cached_hash
|
|
1028
|
+
|
|
1029
|
+
def extra_info(self) -> dict: # ToDo: replace with hash based after DForm freeze
|
|
1030
|
+
"""Return dict with extra type information (besides name and dtype) which is not None."""
|
|
1031
|
+
return dict(
|
|
1032
|
+
self._filter_apply_fields(
|
|
1033
|
+
exclude=('name', 'dtype'),
|
|
1034
|
+
fld_proc=lambda f: (
|
|
1035
|
+
(f.name, val := self.__dict__[f.name]),
|
|
1036
|
+
val is not None
|
|
1037
|
+
)
|
|
1038
|
+
)
|
|
1039
|
+
)
|
|
1040
|
+
|
|
1041
|
+
@classmethod
|
|
1042
|
+
def fields(cls, hash_only=False, exclude=None, include=None) -> Iterator[str]:
|
|
1043
|
+
"""Return iterator over dataclass fields' names.
|
|
1044
|
+
:param hash_only: if ``True`` - only those used for hash calculation
|
|
1045
|
+
:param exclude: optionally exclude one or more of those
|
|
1046
|
+
:param include: only those, first validate their existence
|
|
1047
|
+
"""
|
|
1048
|
+
return cls._filter_apply_fields(
|
|
1049
|
+
fld_proc=lambda f: (f.name, not (hash_only and f.hash is False)),
|
|
1050
|
+
exclude=exclude, include=include
|
|
1051
|
+
)
|
|
1052
|
+
|
|
1053
|
+
def items(self, hash_only=False) -> Iterator[tuple[str, Any]]:
|
|
1054
|
+
"""Return iterator over fields items (name, value)
|
|
1055
|
+
:param hash_only: if True return only fields used in hash calculations
|
|
1056
|
+
"""
|
|
1057
|
+
d = self.__dict__
|
|
1058
|
+
return ((k, d[k]) for k in self.fields(hash_only=hash_only))
|
|
1059
|
+
|
|
1060
|
+
@property
|
|
1061
|
+
def range(self) -> float | None:
|
|
1062
|
+
"""None or length of the range (max - min)"""
|
|
1063
|
+
return None if self.min is None else self.max - self.min
|
|
1064
|
+
|
|
1065
|
+
@property
|
|
1066
|
+
def norm_dim(self) -> int:
|
|
1067
|
+
"""
|
|
1068
|
+
Number of normalization channels, also can be used to check if normalized:
|
|
1069
|
+
- 0 - no normalization,
|
|
1070
|
+
- 1 - normalize without separation by channel
|
|
1071
|
+
- 1+ - number of separate channels
|
|
1072
|
+
|
|
1073
|
+
Normalization is positive if either avr or std is defined.
|
|
1074
|
+
"""
|
|
1075
|
+
return self._norm_channels
|
|
1076
|
+
|
|
1077
|
+
def _code_name(self):
|
|
1078
|
+
# todo Why is this different name convention from type_info?
|
|
1079
|
+
# [C | G | B][R[ < min >:] < max >]][N[ < std >][ |][ < avr >]](u | i) < bytes > | (s | b) < bits >
|
|
1080
|
+
# Idea: produce signal if _parse_name(_code_name(x)) != x
|
|
1081
|
+
clr = self.color.short_code() if self.color else ''
|
|
1082
|
+
|
|
1083
|
+
if self.range:
|
|
1084
|
+
rng = 'R'
|
|
1085
|
+
if self.min: rng += f"{self.min:g}:"
|
|
1086
|
+
rng += f"{self.max:g}"
|
|
1087
|
+
else:
|
|
1088
|
+
rng = ''
|
|
1089
|
+
|
|
1090
|
+
if self.norm_dim > 1:
|
|
1091
|
+
nrm = 'N...'
|
|
1092
|
+
elif self.norm_dim == 1:
|
|
1093
|
+
std, avr = (x if x is None else x.item() for x in (self.std, self.avr))
|
|
1094
|
+
nrm = 'N'
|
|
1095
|
+
if not (std == 1 and avr == 0):
|
|
1096
|
+
if std is not None:
|
|
1097
|
+
nrm += f"{std:g}"
|
|
1098
|
+
if avr is not None:
|
|
1099
|
+
nrm += f"|{avr:g}"
|
|
1100
|
+
else:
|
|
1101
|
+
nrm = ''
|
|
1102
|
+
|
|
1103
|
+
typ = ''
|
|
1104
|
+
if self.dtype.kind == 'i':
|
|
1105
|
+
if self.range and self.min == -self.max and (b := np.log2(self.max + 1)).is_integer():
|
|
1106
|
+
rng = ''
|
|
1107
|
+
if (b := int(b + 1)) // 8 != self.dtype.itemsize:
|
|
1108
|
+
typ = f's{b}'
|
|
1109
|
+
elif self.dtype.kind == 'u':
|
|
1110
|
+
if self.range and self.min == 0 and (b := np.log2(self.max + 1)).is_integer():
|
|
1111
|
+
rng = ''
|
|
1112
|
+
if (b := int(b)) // 8 != self.dtype.itemsize:
|
|
1113
|
+
typ = f'b{int(b)}'
|
|
1114
|
+
return f"{clr}{rng}{nrm}{typ or _dtype_str(self.dtype)}"
|
|
1115
|
+
|
|
1116
|
+
def _copy_from(self, src: DForm | dict):
|
|
1117
|
+
"""Set fields by copying them from other DForm or dict
|
|
1118
|
+
(if DForm _norm_channels are also copied).
|
|
1119
|
+
|
|
1120
|
+
:return: number of fields set with not `None` values
|
|
1121
|
+
"""
|
|
1122
|
+
if isinstance(src, DForm):
|
|
1123
|
+
fields = src.fields(hash_only=False)
|
|
1124
|
+
self._norm_channels = src._norm_channels
|
|
1125
|
+
src = src.__dict__
|
|
1126
|
+
elif isinstance(src, dict):
|
|
1127
|
+
fields = self.fields(hash_only=False, include=src)
|
|
1128
|
+
else:
|
|
1129
|
+
raise TypeError(f"Invalid {type(src) = }!")
|
|
1130
|
+
|
|
1131
|
+
count_real_values = 0
|
|
1132
|
+
for f in fields:
|
|
1133
|
+
if (val := src[f]) is not None:
|
|
1134
|
+
count_real_values += 1
|
|
1135
|
+
setattr(self, f, val)
|
|
1136
|
+
return count_real_values
|
|
1137
|
+
|
|
1138
|
+
def __post_init__(self):
|
|
1139
|
+
self._norm_channels = 0 # number of normalization channels (0 - not normalized)
|
|
1140
|
+
|
|
1141
|
+
parsed = self._decode_name() # name parsed and return True only if only argument DForm(name)
|
|
1142
|
+
self._init_dtype()
|
|
1143
|
+
self._init_range()
|
|
1144
|
+
self._init_norm()
|
|
1145
|
+
self._init_color()
|
|
1146
|
+
self._init_kind()
|
|
1147
|
+
|
|
1148
|
+
# check name consistency for multi-fields init: DFom(name, color, dtype, range, ...)
|
|
1149
|
+
if (not parsed and self.name
|
|
1150
|
+
and (attrs := self._parse_name(self.name)) # name encodes attributes
|
|
1151
|
+
and self.same(DForm(**attrs), name=False)): # matching provided through arguments
|
|
1152
|
+
raise NameError(f"Name is inconsistent with form's attributes")
|
|
1153
|
+
|
|
1154
|
+
DForm._register(self)
|
|
1155
|
+
|
|
1156
|
+
def _decode_name(self) -> bool | int:
|
|
1157
|
+
""" Unpack attributes encoded in the name.
|
|
1158
|
+
|
|
1159
|
+
The `name` attribute string is parsed ONLY if it was the only argument.
|
|
1160
|
+
|
|
1161
|
+
:return: if state changed return ``True`` or number of fields set to not ``None``
|
|
1162
|
+
"""
|
|
1163
|
+
if not (name := self.name):
|
|
1164
|
+
return False
|
|
1165
|
+
|
|
1166
|
+
name_is_str = isinstance(name, str)
|
|
1167
|
+
extra = self.extra_info() # the current state of extra attributes
|
|
1168
|
+
|
|
1169
|
+
if not (extra or self.dtype): # name was the only argument: DForm(name)
|
|
1170
|
+
if name_is_str: # from a single str argument: DForm('Cu1') or DForm('KnownType')
|
|
1171
|
+
if dform := self.from_name(name, auto=False): # query from DataForms registry
|
|
1172
|
+
return self._copy_from(dform) # copy from registered type
|
|
1173
|
+
elif attrs := self._parse_name(name):
|
|
1174
|
+
return self._copy_from(attrs)
|
|
1175
|
+
else:
|
|
1176
|
+
raise NameError(f"Unknown DForm '{self.name}'")
|
|
1177
|
+
elif isinstance(name, DForm):
|
|
1178
|
+
return self._copy_from(name) # including name
|
|
1179
|
+
elif isinstance(name, type): # first arg is type -> dtype
|
|
1180
|
+
self.name = None
|
|
1181
|
+
self.dtype = np.dtype(name)
|
|
1182
|
+
return True
|
|
1183
|
+
else:
|
|
1184
|
+
raise ValueError(f"Invalid name argument {self.name}")
|
|
1185
|
+
elif not name_is_str: # must be string if other arguments are given
|
|
1186
|
+
raise ValueError(f"Expected string for DForm name {self.name}")
|
|
1187
|
+
elif dtype := _name_to_dtype(name): # name encodes dtype
|
|
1188
|
+
if extra: # this form is more than pure dtype
|
|
1189
|
+
raise NameError(f'Assigning dtype {name=} to custom dform')
|
|
1190
|
+
if self.dtype and np.dtype(self.dtype) != dtype:
|
|
1191
|
+
raise NameError(f"Assigning dform.{name=} for dtype {self.dtype}")
|
|
1192
|
+
|
|
1193
|
+
self.name = None # name argument was dtype name - not to register
|
|
1194
|
+
self.dtype = dtype
|
|
1195
|
+
return True
|
|
1196
|
+
return False
|
|
1197
|
+
|
|
1198
|
+
def _init_color(self):
|
|
1199
|
+
if self.color:
|
|
1200
|
+
self.color = Color(self.color) # convert from possible string
|
|
1201
|
+
if self.cax is None and self.color.channels > 1:
|
|
1202
|
+
self.cax = - 1 # default for multiple color channels
|
|
1203
|
+
# here we assume that if normalization is defined per channels - that color channels
|
|
1204
|
+
if self.norm_dim > 1 and self.norm_dim != self.color.channels: raise ValueError(
|
|
1205
|
+
f"Number of color & normalization channels ({self.color.channels}) != {self.norm_dim}")
|
|
1206
|
+
|
|
1207
|
+
if self.color is Color.BIN:
|
|
1208
|
+
mn, mx = map(self.dtype.type, (0, 1))
|
|
1209
|
+
if self.min is None:
|
|
1210
|
+
self.min, self.max = mn, mx
|
|
1211
|
+
elif (self.min, self.max) != (mn, mx):
|
|
1212
|
+
raise ValueError(f"DForm {self} with BW color must have range [0,1]")
|
|
1213
|
+
|
|
1214
|
+
if self.color is not Color.BIN and not (self.norm_dim or self.range):
|
|
1215
|
+
if self.dtype in (np.uint8, np.uint16):
|
|
1216
|
+
info = np.iinfo(self.dtype)
|
|
1217
|
+
self.min, self.max = info.min, info.max
|
|
1218
|
+
else:
|
|
1219
|
+
raise TypeError(f"Color form of type {self.dtype} requires norm or range")
|
|
1220
|
+
elif self.cax is not None:
|
|
1221
|
+
raise ValueError(f"Color axis is defined {self.cax=} without color")
|
|
1222
|
+
|
|
1223
|
+
def _init_dtype(self) -> bool:
|
|
1224
|
+
"""Process state to determine and set dtype.
|
|
1225
|
+
:return: ``True`` if found and changed
|
|
1226
|
+
"""
|
|
1227
|
+
if self.dtype is None:
|
|
1228
|
+
if not self._auto_dtype: # see if guessing is allowed
|
|
1229
|
+
raise TypeError('Missing dtype in DForm definition')
|
|
1230
|
+
elif self.color and Color(self.color) is Color.RGB:
|
|
1231
|
+
self.dtype = 'uint8'
|
|
1232
|
+
else:
|
|
1233
|
+
self.dtype = self._default_dtype
|
|
1234
|
+
return True
|
|
1235
|
+
|
|
1236
|
+
self.dtype = np.dtype(self.dtype)
|
|
1237
|
+
if self.dtype.kind not in 'biuf':
|
|
1238
|
+
raise TypeError(f"Not a numeric type for dtype!")
|
|
1239
|
+
return False
|
|
1240
|
+
|
|
1241
|
+
def _init_range(self):
|
|
1242
|
+
""":return: ``True`` if range been set"""
|
|
1243
|
+
|
|
1244
|
+
if (self.min, self.max) == (None, None):
|
|
1245
|
+
return False
|
|
1246
|
+
|
|
1247
|
+
if self.min is None:
|
|
1248
|
+
self.min = 0
|
|
1249
|
+
elif self.max is None:
|
|
1250
|
+
raise ValueError("DForm min is defined without max")
|
|
1251
|
+
|
|
1252
|
+
dtype = _collect_result_type(self.min, self.max)
|
|
1253
|
+
if not np.can_cast(dtype, self.dtype):
|
|
1254
|
+
TypeError(f"Can't cast range {dtype=} into {self.dtype}")
|
|
1255
|
+
|
|
1256
|
+
self.min, self.max = map(self.dtype.type, (self.min, self.max))
|
|
1257
|
+
if self.max <= self.min:
|
|
1258
|
+
raise ValueError(f"DForm {self} has invalid range")
|
|
1259
|
+
return True
|
|
1260
|
+
|
|
1261
|
+
def _init_norm(self):
|
|
1262
|
+
"""
|
|
1263
|
+
Calculate, update and return `_norm_channels`.
|
|
1264
|
+
:return: 0 means neither `std` nor `avr` are defined
|
|
1265
|
+
"""
|
|
1266
|
+
if self.std is not None:
|
|
1267
|
+
self.std = np.asarray(self.std).flatten()
|
|
1268
|
+
self._norm_channels = max(self._norm_channels, self.std.size)
|
|
1269
|
+
if self.std.min() < 10 * np.finfo('float32').eps:
|
|
1270
|
+
raise ValueError(f"Expected DForm std = {self.std.min()} > eps")
|
|
1271
|
+
|
|
1272
|
+
if self.avr is not None:
|
|
1273
|
+
self.avr = np.asarray(self.avr).flatten()
|
|
1274
|
+
self._norm_channels = max(self._norm_channels, self.avr.size)
|
|
1275
|
+
|
|
1276
|
+
return self._norm_channels
|
|
1277
|
+
|
|
1278
|
+
def _init_kind(self):
|
|
1279
|
+
"""Ensure that `color` consistent with IMAGE `kind`"""
|
|
1280
|
+
if not self.kind and self.color:
|
|
1281
|
+
self.kind = Kind.IMAGE
|
|
1282
|
+
elif self.kind != Kind.IMAGE and self.color:
|
|
1283
|
+
raise ValueError(f"{self.color} implies {Kind.IMAGE} found {self.kind}")
|
|
1284
|
+
|
|
1285
|
+
def _verify_color(self, a: np.ndarray) -> str | None:
|
|
1286
|
+
"""Verify that array matches the color info.
|
|
1287
|
+
|
|
1288
|
+
Return str message describing why not or None if OK
|
|
1289
|
+
"""
|
|
1290
|
+
if self.color:
|
|
1291
|
+
# 1 chn: MxN or MxNx1 | 3 chn: MxNx3
|
|
1292
|
+
if not (self.color.channels == 1 and a.ndim in (2, 3) or a.ndim == 3):
|
|
1293
|
+
return f"Array's dims {a.ndim} incompatible with color {self.color}"
|
|
1294
|
+
|
|
1295
|
+
if not (self.cax is None or # no color channel
|
|
1296
|
+
a.ndim == 2 and self.color.channels == 1 or # 2D image with no separate color channel
|
|
1297
|
+
a.shape[
|
|
1298
|
+
self.cax] == self.color.channels): # color channel location and size
|
|
1299
|
+
return f"Wrong color channels # {a.shape}[{self.cax}] != {self.color.channels}"
|
|
1300
|
+
return None
|
|
1301
|
+
|
|
1302
|
+
def verify_array(self, a: np.ndarray, exc=TypeError):
|
|
1303
|
+
"""Verify that array matches the data form
|
|
1304
|
+
|
|
1305
|
+
:param a: array to verify
|
|
1306
|
+
:param exc: Exception class to raise on error,
|
|
1307
|
+
if None - return error message instead
|
|
1308
|
+
:raises: If shape does not match color information
|
|
1309
|
+
:return: None if no error, otherwise Error message (if not exc)
|
|
1310
|
+
"""
|
|
1311
|
+
assert isinstance(a, np.ndarray)
|
|
1312
|
+
|
|
1313
|
+
def failure(message):
|
|
1314
|
+
if exc: raise exc(message)
|
|
1315
|
+
return message
|
|
1316
|
+
|
|
1317
|
+
if self.dtype is not a.dtype: failure(f"Array and form dtypes mismatch {a.dtype}!={self.dtype}")
|
|
1318
|
+
if msg := self._verify_color(a):
|
|
1319
|
+
return failure(msg)
|
|
1320
|
+
return None
|
|
1321
|
+
|
|
1322
|
+
@classmethod
|
|
1323
|
+
def guess_from_data(cls, a: np.ndarray, fail=False) -> DForm | None:
|
|
1324
|
+
"""
|
|
1325
|
+
Using arrays shape and dtype try to guess its ``DForm``.
|
|
1326
|
+
If fails return ``None`` or raise ``TypeError`` if ``fail is True``
|
|
1327
|
+
:param a: np.ndarray
|
|
1328
|
+
:param fail: default = False return None
|
|
1329
|
+
:return: guessed dform or None
|
|
1330
|
+
"""
|
|
1331
|
+
if a.dtype == np.uint8: # currently only this case is supported
|
|
1332
|
+
return cls(a.dtype, min=0, max=255,
|
|
1333
|
+
color=Color.RGB if a.ndim == 3 and (a.shape[-1] == 3)
|
|
1334
|
+
else Color.GRAY if a.ndim == 1 else None)
|
|
1335
|
+
if fail: raise TypeError(f"Can't guess {cls.__name__} from array's properties")
|
|
1336
|
+
|
|
1337
|
+
def _range_trans_params(self, src_form) -> LinT:
|
|
1338
|
+
"""
|
|
1339
|
+
Calculate linear parameters (k, b) in y = kx+b for normalization amd range transforms
|
|
1340
|
+
If do_range and do_norm not provided, then searching first for range changing, and
|
|
1341
|
+
if there is not, searching for norm changing.
|
|
1342
|
+
:param src_form: source data format
|
|
1343
|
+
:return: k, b - tuple, (1, 0) if transform is not required
|
|
1344
|
+
"""
|
|
1345
|
+
if not src_form.range: # RANGE -> RANGE
|
|
1346
|
+
raise TypeError(f"Can't convert into range from not range-limited source {src_form}")
|
|
1347
|
+
|
|
1348
|
+
k = (self.max - self.min) / (src_form.max - src_form.min)
|
|
1349
|
+
b = self.min - k * src_form.min
|
|
1350
|
+
return LinT(k, b)
|
|
1351
|
+
|
|
1352
|
+
def _norm_trans_params(self, src_form: DForm) -> LinT:
|
|
1353
|
+
"""
|
|
1354
|
+
Calculate linear parameters (k, b) in y = kx+b for normalization amd range transforms
|
|
1355
|
+
If do_range and do_norm not provided, then search first for range changing, and
|
|
1356
|
+
if there is not, search for norm changing.
|
|
1357
|
+
:param src: the source array to be transformed
|
|
1358
|
+
:param src_form: source data format
|
|
1359
|
+
:return: k, b - tuple, (1, 0) if transform is not required
|
|
1360
|
+
"""
|
|
1361
|
+
src_avr = src_form.avr
|
|
1362
|
+
|
|
1363
|
+
# TODO: how to continue if src_avr is None?
|
|
1364
|
+
if self.std is not None: # output std requires src std or given or calculated
|
|
1365
|
+
src_std = src_form.std
|
|
1366
|
+
if self.avr is None: # output avr not defined - only scaling
|
|
1367
|
+
return LinT(self.std / src_std, 0)
|
|
1368
|
+
return LinT(self.std / src_std, self.avr - src_form.avr * self.std / src_std)
|
|
1369
|
+
self.avr: float
|
|
1370
|
+
return LinT(1, self.avr - src_avr) # no std conversion means k=1
|
|
1371
|
+
|
|
1372
|
+
def _apply_transform(self, src, src_form: DForm, out) -> np.ndarray:
|
|
1373
|
+
if self.range and src_form.range and self.norm_dim:
|
|
1374
|
+
# TODO: need to think of all the possibilities of norm and range combinations
|
|
1375
|
+
if self.min != src_form.min or self.max != src_form.max:
|
|
1376
|
+
raise ValueError("Simultaneous Range mapping AND Normalization is not supported")
|
|
1377
|
+
|
|
1378
|
+
lt = (self.range and self._range_trans_params(src_form) or
|
|
1379
|
+
self.norm_dim and self._norm_trans_params(src_form) or
|
|
1380
|
+
LinT())
|
|
1381
|
+
|
|
1382
|
+
return lt.apply(src, out=out)
|
|
1383
|
+
|
|
1384
|
+
def transform(self, src: np.ndarray | Array,
|
|
1385
|
+
src_form: DForm | Literal['guess', 'cast', 'ignore'] = None, *,
|
|
1386
|
+
round=True, out: np.ndarray = None, copy: bool = None) -> np.ndarray:
|
|
1387
|
+
"""
|
|
1388
|
+
Convert ``src`` array from its DForm ``src_form`` into``self`` form.
|
|
1389
|
+
|
|
1390
|
+
Argument ``src_form`` must be one of those:
|
|
1391
|
+
- a ``DForm`` (or cast-able into) object (overrides with warning possible ``src.dform``)
|
|
1392
|
+
- 'guess' str to attempt guessing DForm from ``src`` (or inherit its ``dform`` if available)
|
|
1393
|
+
- 'ignore' try to ``self.dform`` to src ignoring its possible ``dform`` with minimal changes.
|
|
1394
|
+
- ``None``, ``src`` must have ``.dform`` in this case
|
|
1395
|
+
|
|
1396
|
+
If ``out`` is provided with ``.dform`` it must either match ``self.dform``
|
|
1397
|
+
or be compatible: same ``dtype`` and implied shape (color channels).
|
|
1398
|
+
|
|
1399
|
+
|
|
1400
|
+
|
|
1401
|
+
:param src: ndarray to transform
|
|
1402
|
+
:param src_form: DForm of the source array or 'guess' to request guessing
|
|
1403
|
+
:param out: provide array to fill the transform in
|
|
1404
|
+
:param copy: if ``True`` enforce copy even if data has not been changed
|
|
1405
|
+
:return: ndarray converted into self DForm
|
|
1406
|
+
"""
|
|
1407
|
+
src, src_form = self._prepare_src_form(src, src_form) # None means src_form ignore is requested!
|
|
1408
|
+
if src_form in ('ignore', self): # semantically equal or ignore src_form
|
|
1409
|
+
return self._bypass_transform(src, out, copy)
|
|
1410
|
+
|
|
1411
|
+
shaper = DForm.Shaper(src.shape, src_form, self)
|
|
1412
|
+
if out is None: # allocate output array if not provided
|
|
1413
|
+
out = np.empty(shape=shaper.trg_shape, dtype=self.dtype) # created with cax always last!
|
|
1414
|
+
else: # or verify it if it is provided
|
|
1415
|
+
if out.dtype != self.dtype: raise TypeError(f"Mismatch {out.dtype=} != {self.dtype}")
|
|
1416
|
+
if out.shape != shaper.trg_shape: raise TypeError(f"Mismatch {out.shape=} != {shaper.trg_shape}")
|
|
1417
|
+
out = out.view(np.ndarray) # strip possible dform
|
|
1418
|
+
|
|
1419
|
+
src = shaper.move_cax(src, 'src', True)
|
|
1420
|
+
out = shaper.move_cax(out, 'trg', True)
|
|
1421
|
+
|
|
1422
|
+
src, src_form = self._color_pre_transform(src, src_form, out)
|
|
1423
|
+
self._apply_transform(src, src_form, out)
|
|
1424
|
+
|
|
1425
|
+
return shaper.move_cax(out, 'trg', False)
|
|
1426
|
+
|
|
1427
|
+
@staticmethod
|
|
1428
|
+
def _bypass_transform(src, out, copy):
|
|
1429
|
+
if out is None:
|
|
1430
|
+
out = src.copy() if copy else src
|
|
1431
|
+
else:
|
|
1432
|
+
if out.shape != src.shape or out.dtype != src.dtype:
|
|
1433
|
+
raise TypeError(f"Incompatible out: {out.shape}{out.dtype} with {src.shape}{src.dtype}")
|
|
1434
|
+
if npt.shared_view(out, src) and copy is True:
|
|
1435
|
+
raise ValueError("Can't copy: out shares src data")
|
|
1436
|
+
if copy:
|
|
1437
|
+
np.copyto(out, src, casting='no')
|
|
1438
|
+
return out
|
|
1439
|
+
|
|
1440
|
+
def _color_pre_transform(self, src, src_form: DForm, out):
|
|
1441
|
+
"""
|
|
1442
|
+
Color transformation can be achieved by pre-transforming ``src``, ``src_form``
|
|
1443
|
+
before passed to the ``_apply_transform`` method.
|
|
1444
|
+
|
|
1445
|
+
:return: prepared ``src``, ``src_form``
|
|
1446
|
+
"""
|
|
1447
|
+
if self.color and self.color != src_form.color: # only if target color is defined
|
|
1448
|
+
clr = Color
|
|
1449
|
+
colors = (src_form.color, self.color)
|
|
1450
|
+
|
|
1451
|
+
if not src_form.color: raise TypeError("Color transform from undefined color")
|
|
1452
|
+
# if src_form.std is not None and self.norm: # TODO: check this condition
|
|
1453
|
+
# raise TypeError("Can't transform color from std-ized image")
|
|
1454
|
+
if clr.BIN in colors: raise TypeError("Can't convert between any color and binary")
|
|
1455
|
+
|
|
1456
|
+
if colors == (clr.RGB, clr.GRAY): # MxNx3 -> MxN
|
|
1457
|
+
src = np.dot(src, _wC2G).squeeze()
|
|
1458
|
+
if src_form.norm_dim: # in this case possible per channel norm should be scaled too
|
|
1459
|
+
weighted = lambda _: None if _ is None else np.dot(np.resize(_, 3), _wC2G).squeeze()
|
|
1460
|
+
src_form = DForm(dtype=src.dtype, color=Color.GRAY,
|
|
1461
|
+
min=src_form.min, max=src_form.max,
|
|
1462
|
+
std=weighted(src_form.std),
|
|
1463
|
+
avr=weighted(src_form.avr))
|
|
1464
|
+
elif colors == (clr.GRAY, clr.RGB):
|
|
1465
|
+
src = np.dstack([src] * colors[1].channels)
|
|
1466
|
+
else:
|
|
1467
|
+
raise NotImplementedError(f'Not supported transform {colors=}')
|
|
1468
|
+
return src, src_form
|
|
1469
|
+
|
|
1470
|
+
def _prepare_src_form(self, src: Array | np.ndarray,
|
|
1471
|
+
src_form: Literal['ignore', 'guess'] | DForm | type
|
|
1472
|
+
) -> tuple[np.ndarray, DForm | Literal['ignore']]:
|
|
1473
|
+
"""
|
|
1474
|
+
From various forms of arguments ``src`` and ``src_form`` produce tuple:
|
|
1475
|
+
- ``src`` as ``ndarray`` (stripped from dform) and
|
|
1476
|
+
- ``src_form`` as ``DForm`` (or 'ignore')
|
|
1477
|
+
|
|
1478
|
+
If both define different DForm ``src_form != src.dform`` use ``src_form`` and issue warning.
|
|
1479
|
+
|
|
1480
|
+
Raise error if:
|
|
1481
|
+
1. ``src_form="guess"`` but can't guess ``DForm`` from the data
|
|
1482
|
+
2. ``src_form=None`` and ``src`` does not have DForm information
|
|
1483
|
+
3. ``src_form`` can not be cast into ``DForm``
|
|
1484
|
+
"""
|
|
1485
|
+
|
|
1486
|
+
if not_array := not isinstance(src, np.ndarray):
|
|
1487
|
+
if src_form in (None, 'ignore', 'guess'):
|
|
1488
|
+
raise TypeError(f"Neither src nor {src_form=} contain dform information!")
|
|
1489
|
+
dform_in_src = None
|
|
1490
|
+
elif dform_in_src := getattr(src, 'dform', None): # recover dform from src
|
|
1491
|
+
src = src.view(np.ndarray) # before stripping
|
|
1492
|
+
|
|
1493
|
+
if src_form is None:
|
|
1494
|
+
return src, dform_in_src or DForm(dtype=src.dtype)
|
|
1495
|
+
if src_form == 'ignore': return src, src_form # dissociate any dform from src
|
|
1496
|
+
if src_form == 'guess': return src, dform_in_src or self.guess_from_data(src, fail=True)
|
|
1497
|
+
|
|
1498
|
+
if not isinstance(src_form, DForm): src_form = DForm(src_form)
|
|
1499
|
+
if not_array: src = np.array(src, dtype=src_form.dtype)
|
|
1500
|
+
|
|
1501
|
+
if not dform_in_src: # src_form is provided separately - check validity
|
|
1502
|
+
if msg := src_form.verify_array(src): raise TypeError(msg)
|
|
1503
|
+
elif src_form != dform_in_src: # IDEA: allowed case and correct behaviour?
|
|
1504
|
+
warnings.warn(f"Overriding {dform_in_src} by {src_form}")
|
|
1505
|
+
|
|
1506
|
+
return src, src_form
|
|
1507
|
+
|
|
1508
|
+
|
|
1509
|
+
_format_options = dict(
|
|
1510
|
+
rows=16,
|
|
1511
|
+
cols=12,
|
|
1512
|
+
precision=3,
|
|
1513
|
+
rng=True,
|
|
1514
|
+
norm=False,
|
|
1515
|
+
nans=True,
|
|
1516
|
+
stats=(9, 1e7),
|
|
1517
|
+
info=True
|
|
1518
|
+
)
|
|
1519
|
+
|
|
1520
|
+
|
|
1521
|
+
class FormArray(type):
|
|
1522
|
+
"""
|
|
1523
|
+
Metaclass for creating Array-like classes.
|
|
1524
|
+
"""
|
|
1525
|
+
|
|
1526
|
+
EnableCond = Union[None, bool, Number, tuple[Number, Number]]
|
|
1527
|
+
_UND = type('_U', (), {'__str__': lambda _: 'UND'})()
|
|
1528
|
+
|
|
1529
|
+
_reg: dict[FormArray, FormArray] = {}
|
|
1530
|
+
_np_options = np.get_printoptions()
|
|
1531
|
+
_def_options = _format_options.copy()
|
|
1532
|
+
|
|
1533
|
+
def __new__(mcs, name: str | DForm | type, bases=(), attrs=None, *, dform: str | DForm = None,
|
|
1534
|
+
_base=None) -> Array:
|
|
1535
|
+
if _base: # used to create base classes without dform, not for user!
|
|
1536
|
+
if dform: raise RuntimeError("Base Array class can't define dform")
|
|
1537
|
+
cls = super().__new__(mcs, name, bases, attrs or {})
|
|
1538
|
+
cls.dform = None
|
|
1539
|
+
return cls
|
|
1540
|
+
|
|
1541
|
+
if dform is None: # dfrom may be passed through name argument
|
|
1542
|
+
if ( # FormArray('NCu1') - name is a registered dform or follows coding convention
|
|
1543
|
+
isinstance(name, str) and ( # FixMe: same as DForm.from_name(name)?
|
|
1544
|
+
dform := DForm.from_name(name, auto=True)) and DForm.auto_naming()
|
|
1545
|
+
or # FormArray(dform) - explicitly provided DForm
|
|
1546
|
+
isinstance(name, DForm) and (dform := name) # assign to dform
|
|
1547
|
+
):
|
|
1548
|
+
name = f'Array{dform.name or "_"}'
|
|
1549
|
+
elif not bases and not attrs: # FormArray('uint8') - from dtype
|
|
1550
|
+
dform = DForm(dtype=name)
|
|
1551
|
+
name = f'Array_{_dtype_str(dform.dtype)}'
|
|
1552
|
+
|
|
1553
|
+
if not isinstance(dform, DForm): # 1. FormArray() - default dtype DForm
|
|
1554
|
+
dform = DForm(dform) # 2. FormArray('Name', dform=int)
|
|
1555
|
+
|
|
1556
|
+
if not any(issubclass(b, np.ndarray) for b in bases):
|
|
1557
|
+
bases = (Array, *bases)
|
|
1558
|
+
cls = super().__new__(mcs, name, bases, attrs or {})
|
|
1559
|
+
cls.dform = dform
|
|
1560
|
+
cls._format_options = FormArray._def_options.copy()
|
|
1561
|
+
return FormArray._reg.setdefault(cls, cls)
|
|
1562
|
+
|
|
1563
|
+
@classmethod # fixme: make it self method
|
|
1564
|
+
def from_array(cls, data: np.ndarray | Iterable | Array, *, dform: FormKind = None,
|
|
1565
|
+
copy=True, **dform_attrs) -> Array:
|
|
1566
|
+
"""From given array create instance of Array-based class,
|
|
1567
|
+
with dform derived from the data or optional ``dform`` argument.
|
|
1568
|
+
|
|
1569
|
+
General compatibility requirements between the data and the dform applies.
|
|
1570
|
+
|
|
1571
|
+
In the minimal case with no additional arguments when only ``dtype`` can be deduced from
|
|
1572
|
+
the data, the resulting Array will be based on ``DForm(data.dtype)``
|
|
1573
|
+
|
|
1574
|
+
- If given ``dform`` argument overrides data type in all the cases!
|
|
1575
|
+
- In addition or alternatively DForm attributes may be provided as ``kws``.
|
|
1576
|
+
|
|
1577
|
+
If data is an iterable, then first an ``ndarray`` is created and then
|
|
1578
|
+
``dtype`` is determined.
|
|
1579
|
+
|
|
1580
|
+
Even if data comes with its own ``dform``, it is replaced **without transformation**!
|
|
1581
|
+
|
|
1582
|
+
Also, available as a direct function in this module:
|
|
1583
|
+
::
|
|
1584
|
+
form_array(data, dform, **dform_attrs)
|
|
1585
|
+
|
|
1586
|
+
:param data: ndarray or Array or iterable
|
|
1587
|
+
:param dform: DForm in any of supported form
|
|
1588
|
+
:param copy: if False try using view when possible
|
|
1589
|
+
:param dform_attrs: DForm attributes as key-value pairs
|
|
1590
|
+
"""
|
|
1591
|
+
if not isinstance(data, np.ndarray):
|
|
1592
|
+
data = np.array(data)
|
|
1593
|
+
|
|
1594
|
+
data_form = getattr(data, 'dform', None)
|
|
1595
|
+
data_type = getattr(data, 'dtype', None)
|
|
1596
|
+
dform = DForm.from_merge(dform, **dform_attrs)
|
|
1597
|
+
|
|
1598
|
+
if dform:
|
|
1599
|
+
if not data_type:
|
|
1600
|
+
return FormArray(dform)(np.array(data, dtype=dform.dtype, copy=copy))
|
|
1601
|
+
|
|
1602
|
+
if data_form: # noinspection PyUnresolvedReferences
|
|
1603
|
+
return FormArray(dform)(data, copy=copy)
|
|
1604
|
+
else:
|
|
1605
|
+
return FormArray(dform)(data, copy=copy)
|
|
1606
|
+
else:
|
|
1607
|
+
if data_form:
|
|
1608
|
+
return np.asanyarray(data)
|
|
1609
|
+
else:
|
|
1610
|
+
return FormArray(data.dtype)(data)
|
|
1611
|
+
|
|
1612
|
+
@classmethod
|
|
1613
|
+
def global_print_options(mcs, *, rows=_UND, cols=_UND, precision=_UND, info=_UND,
|
|
1614
|
+
stats: EnableCond = _UND, rng: EnableCond = _UND,
|
|
1615
|
+
norm: EnableCond = _UND, nans: EnableCond = _UND, **options):
|
|
1616
|
+
"""
|
|
1617
|
+
Set or get print options for ALL the FormArray classes.
|
|
1618
|
+
|
|
1619
|
+
Arguments ``stats, rng, norm, nans`` define conditions when to calculate
|
|
1620
|
+
specific statistics, and could be:
|
|
1621
|
+
- set explicitly (True|False)
|
|
1622
|
+
- defined (min, max) limits of array's size
|
|
1623
|
+
- only by the maximum limit
|
|
1624
|
+
- set to None, in which case it follows `stats` condition
|
|
1625
|
+
|
|
1626
|
+
``stats`` is a general condition, if it evaluates to False, no calculations will be performed.
|
|
1627
|
+
Otherwise, other arguments will define their specific condition, as described above.
|
|
1628
|
+
|
|
1629
|
+
:param cols: maximal columns to show content
|
|
1630
|
+
:param rows: maximal rows to show content
|
|
1631
|
+
:param info: bool - show info summary, (None - automatic)
|
|
1632
|
+
:param precision (one of ``np.set_printoptions``)
|
|
1633
|
+
:param stats: condition to calculate all the stats
|
|
1634
|
+
:param rng: condition to calculate data range (min, max)
|
|
1635
|
+
:param norm: condition to calculate avr and std
|
|
1636
|
+
:param nans: condition to count found ``nan``s (only for floating point arrays)
|
|
1637
|
+
:param options: other options for ``np.set_printoptions``
|
|
1638
|
+
:return:
|
|
1639
|
+
"""
|
|
1640
|
+
loc = locals().copy()
|
|
1641
|
+
for k in ['mcs', 'options']: del loc[k]
|
|
1642
|
+
|
|
1643
|
+
if inv := set(options).difference(mcs._np_options):
|
|
1644
|
+
raise KeyError(f"Unknown numpy print options: {inv}")
|
|
1645
|
+
|
|
1646
|
+
defined = {k: v for k, v in loc.items() if v is not mcs._UND}
|
|
1647
|
+
mcs._def_options.update(**defined)
|
|
1648
|
+
mcs._np_options.update(**options)
|
|
1649
|
+
|
|
1650
|
+
if defined or options:
|
|
1651
|
+
for cls in mcs._reg:
|
|
1652
|
+
cls.print_options(**defined, **options)
|
|
1653
|
+
else:
|
|
1654
|
+
return mcs._np_options | mcs._def_options
|
|
1655
|
+
|
|
1656
|
+
def print_options(cls, *, rows=_UND, cols=_UND, precision=_UND, info=_UND,
|
|
1657
|
+
stats: EnableCond = _UND, rng: EnableCond = _UND,
|
|
1658
|
+
norm: EnableCond = _UND, nans: EnableCond = _UND, **options):
|
|
1659
|
+
"""
|
|
1660
|
+
Set or get print options for specific class of Arrays.
|
|
1661
|
+
|
|
1662
|
+
When called without arguments returns a copy of the current options.
|
|
1663
|
+
|
|
1664
|
+
Arguments ``stats, rng, norm, nans`` define conditions when to calculate
|
|
1665
|
+
specific statistics, and could be:
|
|
1666
|
+
- set explicitly (True|False)
|
|
1667
|
+
- defined (min, max) limits of array's size
|
|
1668
|
+
- only by the maximum limit
|
|
1669
|
+
- set to None, in which case it follows `stats` condition
|
|
1670
|
+
|
|
1671
|
+
``stats`` is a general condition, if it evaluates to False, no calculations will be performed.
|
|
1672
|
+
Otherwise, other arguments will define their specific condition, as described above.
|
|
1673
|
+
|
|
1674
|
+
:param cols: maximal columns to show content
|
|
1675
|
+
:param rows: maximal rows to show content
|
|
1676
|
+
:param info: bool - show info summary, (None - automatic)
|
|
1677
|
+
:param precision (one of ``np.set_printoptions``)
|
|
1678
|
+
:param stats: condition to calculate all the stats
|
|
1679
|
+
:param rng: condition to calculate data range (min, max)
|
|
1680
|
+
:param norm: condition to calculate avr and std
|
|
1681
|
+
:param nans: condition to count found ``nan``s (only for floating point arrays)
|
|
1682
|
+
:param options: other options for ``np.set_printoptions``
|
|
1683
|
+
:return:
|
|
1684
|
+
"""
|
|
1685
|
+
loc = locals().copy()
|
|
1686
|
+
for k in ['cls', 'options']: del loc[k]
|
|
1687
|
+
|
|
1688
|
+
if inv := set(options).difference(cls._np_options):
|
|
1689
|
+
raise KeyError(f"Unknown numpy print options: {inv}")
|
|
1690
|
+
|
|
1691
|
+
defined = {k: v for k, v in loc.items() if v is not cls._UND}
|
|
1692
|
+
|
|
1693
|
+
if options or defined:
|
|
1694
|
+
cls._format_options.update(**options, **defined)
|
|
1695
|
+
else:
|
|
1696
|
+
return cls._format_options.copy()
|
|
1697
|
+
|
|
1698
|
+
@overload
|
|
1699
|
+
def __init__(cls, class_name: str, *, dform: str | DForm):
|
|
1700
|
+
...
|
|
1701
|
+
|
|
1702
|
+
@overload
|
|
1703
|
+
def __init__(cls, dform: str | DForm | type):
|
|
1704
|
+
pass
|
|
1705
|
+
|
|
1706
|
+
def __init__(cls, name: str | DForm | type, bases=(), attrs: dict = None, *,
|
|
1707
|
+
dform: str | DForm = None, _base=None):
|
|
1708
|
+
"""
|
|
1709
|
+
Create a new ``Array`` like class with associated ``DForm``.
|
|
1710
|
+
Supported initializations:
|
|
1711
|
+
|
|
1712
|
+
>>> FormArray('MyArray', (Array,), {}, dform=DForm('Cu1') ) # full form
|
|
1713
|
+
>>> FormArray('MyArray', dform='Cu1') # Assuming Cu8 is a registered DForm
|
|
1714
|
+
>>> FormArray('Cu1') == FormArray('ArrayCu8', 'Cu1') == FormArray(DForm('Cu1'))
|
|
1715
|
+
|
|
1716
|
+
:param name: class name or dform (registered name or an instance)
|
|
1717
|
+
:param bases: base classes include at least one subclass of ``np.ndarray``.
|
|
1718
|
+
Otherwise, ``Array`` is automatically.
|
|
1719
|
+
:param attrs: optional additional attributes
|
|
1720
|
+
:param dform: ``DForm``- name or instance. ``None`` equivalent to ``DForm()``
|
|
1721
|
+
"""
|
|
1722
|
+
|
|
1723
|
+
def __init_subclass__(cls, *, dform=None):
|
|
1724
|
+
if dform is None:
|
|
1725
|
+
dform = DForm()
|
|
1726
|
+
elif not isinstance(dform, DForm):
|
|
1727
|
+
raise TypeError(f"dform must be None or DForm")
|
|
1728
|
+
cls.dform = dform
|
|
1729
|
+
|
|
1730
|
+
def __eq__(self, other: FormArray):
|
|
1731
|
+
if isinstance(other, FormArray):
|
|
1732
|
+
return self.dform == other.dform
|
|
1733
|
+
return False
|
|
1734
|
+
|
|
1735
|
+
def __hash__(self):
|
|
1736
|
+
return hash(self.dform) + hash(self.__name__)
|
|
1737
|
+
|
|
1738
|
+
def __repr__(self):
|
|
1739
|
+
return self.__qualname__
|
|
1740
|
+
|
|
1741
|
+
|
|
1742
|
+
# noinspection PyShadowingBuiltins
|
|
1743
|
+
class Array(np.ndarray, metaclass=FormArray, _base=True):
|
|
1744
|
+
dform: DForm
|
|
1745
|
+
|
|
1746
|
+
@property
|
|
1747
|
+
def ndarray(self):
|
|
1748
|
+
return np.asarray(self)
|
|
1749
|
+
|
|
1750
|
+
def __repr__(self):
|
|
1751
|
+
return npt.array_info_func(**self._format_options)(self)
|
|
1752
|
+
|
|
1753
|
+
def __str__(self):
|
|
1754
|
+
return self.__repr__()
|
|
1755
|
+
|
|
1756
|
+
def __new__(cls, obj, dform=None, *, trans=False, round=False, copy=False):
|
|
1757
|
+
"""
|
|
1758
|
+
Creating a new instance of an Array-like class.
|
|
1759
|
+
Notice! This class can create instances of another classes, not only Array.
|
|
1760
|
+
Use cases:
|
|
1761
|
+
|
|
1762
|
+
>>> Array([]) # same as Array([], 'f4').from_array([])
|
|
1763
|
+
>>> Array([], dform='Nf2') # creates instance of ArrayNf2
|
|
1764
|
+
|
|
1765
|
+
>>> ArrayCu1([]) # ok
|
|
1766
|
+
>>> ArrayCu1([], dform='Nf2') # error
|
|
1767
|
+
|
|
1768
|
+
:param obj: data as iterable or ndarray or Array-like with dfrom
|
|
1769
|
+
:param trans: allow data transfromation when casting into new form
|
|
1770
|
+
:param copy: if False try to avoid memory copy (use view)
|
|
1771
|
+
:param dform: only allowed in base class (Array)
|
|
1772
|
+
"""
|
|
1773
|
+
if cls.dform: # ArrayF(a)
|
|
1774
|
+
if dform and dform != cls.dform: # ArrayF1(af, f1) !ArrayF1.dform == f1
|
|
1775
|
+
raise TypeError(f"argument {dform=} clashes with {cls.dform=}")
|
|
1776
|
+
else: # Array(obj, dform='Nu1')
|
|
1777
|
+
return FormArray(dform or FLOAT)(obj, trans=trans, round=round, copy=copy)
|
|
1778
|
+
|
|
1779
|
+
if type(obj) == cls: # Array(obj:Frm, Frm) !From -> Frm
|
|
1780
|
+
return obj.copy() if copy else obj
|
|
1781
|
+
|
|
1782
|
+
if not isinstance(obj, np.ndarray): # ArrayFrm([1,2])
|
|
1783
|
+
obj = np.array(obj) # - create ndarray
|
|
1784
|
+
copy = False
|
|
1785
|
+
if obj.dtype.kind not in 'bifu':
|
|
1786
|
+
raise TypeError('Array data must be of numeric type')
|
|
1787
|
+
|
|
1788
|
+
if trans: # ArrayF(a:ndarray)
|
|
1789
|
+
obj = cls.dform.transform(obj, copy=copy, round=round)
|
|
1790
|
+
else:
|
|
1791
|
+
obj = cast(obj, dtype=cls.dform.dtype, round=round, copy=copy)
|
|
1792
|
+
return obj.view(cls)
|
|
1793
|
+
|
|
1794
|
+
# noinspection PyMethodOverriding
|
|
1795
|
+
def __array_finalize__(self, obj):
|
|
1796
|
+
if obj is None: return
|
|
1797
|
+
if self.dform.dtype and self.dform.dtype != self.dtype:
|
|
1798
|
+
raise TypeError(f"dtype {self.dtype} incompatible with {self.dform}'s {self.dform.dtype}")
|
|
1799
|
+
|
|
1800
|
+
def __array_ufunc__(self, ufunc, method, *inputs, out=None, **kwargs):
|
|
1801
|
+
# Based on: https://numpy.org/doc/stable/user/basics.subclassing.html
|
|
1802
|
+
# Required to make sure ndarray are provided where expected in Universal functions (Ufunc).
|
|
1803
|
+
# Also avoids automatic conversion of scalar results into Array
|
|
1804
|
+
|
|
1805
|
+
# Unless out arrays are provided, all the resulting arrays are returned as Array (empty DForm)
|
|
1806
|
+
# leaving to the user reassignment to the proper DForm if needed.
|
|
1807
|
+
# That because an automatic propagation of the DForm would be too tricky in most the cases.
|
|
1808
|
+
# TODO: should we implement those few clear cases?
|
|
1809
|
+
|
|
1810
|
+
args = tuple(input_.view(np.ndarray) if isinstance(type(input_), FormArray) else
|
|
1811
|
+
input_ for input_ in inputs) # view inputs as regular ndarrays
|
|
1812
|
+
|
|
1813
|
+
if out: # also view as ndarrays optional outputs
|
|
1814
|
+
outputs = out
|
|
1815
|
+
kwargs['out'] = tuple(output.view(np.ndarray) if isinstance(type(output), FormArray) else
|
|
1816
|
+
output for output in outputs)
|
|
1817
|
+
else:
|
|
1818
|
+
outputs = (None,) * ufunc.nout
|
|
1819
|
+
|
|
1820
|
+
results = super().__array_ufunc__(ufunc, method, *args, **kwargs)
|
|
1821
|
+
if results is NotImplemented: return NotImplemented
|
|
1822
|
+
if method == 'at': return
|
|
1823
|
+
|
|
1824
|
+
def select(r, o):
|
|
1825
|
+
if o is not None: return o
|
|
1826
|
+
if not isinstance(r, np.ndarray): return r
|
|
1827
|
+
return np.asarray(r).view(FormArray(r.dtype))
|
|
1828
|
+
|
|
1829
|
+
if ufunc.nout == 1: return select(results, outputs[0])
|
|
1830
|
+
return tuple(select(result, output) for result, output in zip(results, outputs))
|
|
1831
|
+
|
|
1832
|
+
# def __init__(self, obj, dform=None, *, trans=False, copy=False):
|
|
1833
|
+
# """
|
|
1834
|
+
# Something
|
|
1835
|
+
#
|
|
1836
|
+
# :param obj:
|
|
1837
|
+
# :param dform:
|
|
1838
|
+
# :param trans:
|
|
1839
|
+
# :param copy:
|
|
1840
|
+
# """
|
|
1841
|
+
# ...
|
|
1842
|
+
|
|
1843
|
+
def asform(self, dform: str | DForm = None, **kws):
|
|
1844
|
+
"""
|
|
1845
|
+
Return present array as a different `DForm`, based on the:
|
|
1846
|
+
1. empty `DForm()` if ``dform = None (and no kws)``
|
|
1847
|
+
2. provided as DForm instance
|
|
1848
|
+
3. provided as a name to registered DForm instance
|
|
1849
|
+
|
|
1850
|
+
Additional keyword arguments may be provided to override specific ``DForm`` attributes:
|
|
1851
|
+
``name, color, cax, std, avr, min, max``
|
|
1852
|
+
|
|
1853
|
+
Returned array is of a different type (FormArray sub-class).
|
|
1854
|
+
|
|
1855
|
+
Use this method when specificity of the form of an existing data must be updated.
|
|
1856
|
+
Initial transform is ignored
|
|
1857
|
+
|
|
1858
|
+
Notice:
|
|
1859
|
+
- only attributes are replaced
|
|
1860
|
+
- ``dtype`` can not be changed
|
|
1861
|
+
- resulting array is another view on same data
|
|
1862
|
+
|
|
1863
|
+
:return: An instance of a new Array subclass with different ``DForm`` but same data.
|
|
1864
|
+
"""
|
|
1865
|
+
if isinstance(dform, str):
|
|
1866
|
+
dform = DForm(dform)
|
|
1867
|
+
elif dform is None:
|
|
1868
|
+
if not kws: # neither dform nor kws defined - strip dform
|
|
1869
|
+
return FormArray(self.dtype)(self.ndarray)
|
|
1870
|
+
dform = self.dform # inherit from self.dform with updates from kws
|
|
1871
|
+
elif not isinstance(dform, DForm):
|
|
1872
|
+
raise ValueError(f"Invalid argument {dform=}")
|
|
1873
|
+
|
|
1874
|
+
if kws:
|
|
1875
|
+
attrs = dcs.fields(DForm)
|
|
1876
|
+
if unknown := set(kws).difference(attrs):
|
|
1877
|
+
raise KeyError(f"DForm has no attribute: {unknown}")
|
|
1878
|
+
|
|
1879
|
+
# inherit undefined attributes from the dform
|
|
1880
|
+
if 'dtype' in kws: raise TypeError("dtype can not be overriden")
|
|
1881
|
+
kws = {f.name: kws.get(f.name, getattr(dform, f.name))
|
|
1882
|
+
for f in attrs if f.init}
|
|
1883
|
+
return FormArray(DForm(**kws))(self.ndarray)
|
|
1884
|
+
elif dform == self.dform:
|
|
1885
|
+
return np.asanyarray(self)
|
|
1886
|
+
else:
|
|
1887
|
+
return dform.transform(self, src_form='ignore')
|
|
1888
|
+
|
|
1889
|
+
def transform(self, dform: DForm, *, guess=True, copy=True, out=None):
|
|
1890
|
+
"""Creates a NEW object of required form.
|
|
1891
|
+
If ``dform`` is same as ``self.dform`` data is not transformed.
|
|
1892
|
+
|
|
1893
|
+
If ``self.dform`` is not defined, attempt to guess it before failing.
|
|
1894
|
+
Undefined ``dform`` always fails.
|
|
1895
|
+
|
|
1896
|
+
:param dform: target DForm of resulting array
|
|
1897
|
+
:param guess: if ``self.dform`` is not defined, guess from the data
|
|
1898
|
+
:return: new array instance with requested ``dform``.
|
|
1899
|
+
"""
|
|
1900
|
+
# todo implement copy and out (with and without dform). If out neglect copy
|
|
1901
|
+
if isinstance(dform, str): dform = DForm(dform)
|
|
1902
|
+
if dform == self.dform: return type(self)(self)
|
|
1903
|
+
src_form = 'guess' if (self.dform is None and guess) else self.dform
|
|
1904
|
+
return FormArray(dform)(dform.transform(self, src_form=src_form))
|
|
1905
|
+
|
|
1906
|
+
|
|
1907
|
+
# Consider auto-generation based on the naming convention:
|
|
1908
|
+
#
|
|
1909
|
+
# [C|G[<bits>b]][R[<min>[:]]<max>|N[<std>[:<avr>]]
|
|
1910
|
+
# C - color (RGB)
|
|
1911
|
+
# G - Gray scale
|
|
1912
|
+
# R - Range
|
|
1913
|
+
# N - Normalized (by default std=1, avr=0)
|
|
1914
|
+
#
|
|
1915
|
+
# In R<min>:<max> ':' optional if min and max single digit
|
|
1916
|
+
# That makes Rxy ambiguous: R0:xy or Rx:y. Second is assumed
|
|
1917
|
+
|
|
1918
|
+
|
|
1919
|
+
# Some standard data forms
|
|
1920
|
+
# DForm('R01f4', FLOAT, min=0, max=1) # any dim
|
|
1921
|
+
# DForm('R-11f4', dtype=FLOAT, min=-1, max=1) # any dim
|
|
1922
|
+
#
|
|
1923
|
+
# DForm('CR01f4', dtype=FLOAT, min=0, max=1, color=DForm.Color.RGB) # HxWx3
|
|
1924
|
+
# DForm('CNf4', FLOAT, color=DForm.Color.RGB, avr=0, std=1)
|
|
1925
|
+
# DForm('Nf4', dtype=FLOAT, avr=0, std=1)
|
|
1926
|
+
#
|
|
1927
|
+
# DForm('Cu1', dtype='u1', color=DForm.Color.RGB)
|
|
1928
|
+
# DForm('Gb10', dtype='u2', min=0, max=1023, color=DForm.Color.GRAY)
|
|
1929
|
+
|
|
1930
|
+
@overload
|
|
1931
|
+
def form_array(data: Iterable | np.ndarray, *, dform: FormKind = None, copy=False) -> Array: ...
|
|
1932
|
+
|
|
1933
|
+
|
|
1934
|
+
@overload
|
|
1935
|
+
def form_array(data: Iterable | np.ndarray, *, copy=False, **dform_attrs) -> Array: ...
|
|
1936
|
+
|
|
1937
|
+
|
|
1938
|
+
def form_array(data: Iterable | np.ndarray | Array, *,
|
|
1939
|
+
dform: FormArray | FormKind = None, copy=False,
|
|
1940
|
+
**dform_attrs) -> Array:
|
|
1941
|
+
"""
|
|
1942
|
+
From given array create instance of Array-based class,
|
|
1943
|
+
with dform derived from the data or optional ``dform`` argument.
|
|
1944
|
+
|
|
1945
|
+
In basic case resulting Array will be based on ``DForm(data.dtype)``
|
|
1946
|
+
|
|
1947
|
+
- If given ``dform`` argument overrides data type in all the cases!
|
|
1948
|
+
- In addition or alternatively DForm attributes may be provided as ``kws``.
|
|
1949
|
+
|
|
1950
|
+
If data is an iterable, then first an ``ndarray`` is created and then
|
|
1951
|
+
``dtype`` is determined.
|
|
1952
|
+
|
|
1953
|
+
Equivalent to:
|
|
1954
|
+
::
|
|
1955
|
+
FormArray.from_array(data, dform=dform, **dform_attrs)
|
|
1956
|
+
|
|
1957
|
+
:param data: ndarray or Array or iterable
|
|
1958
|
+
:param dform: Array-like class or DForm in any of supported form
|
|
1959
|
+
:param dform_attrs: DForm attributes as key-value pairs
|
|
1960
|
+
"""
|
|
1961
|
+
return FormArray.from_array(data, dform=dform, copy=copy, **dform_attrs)
|