ialdev-core 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. iad/core/__init__.py +9 -0
  2. iad/core/array.py +1961 -0
  3. iad/core/binary.py +377 -0
  4. iad/core/cache.py +903 -0
  5. iad/core/codetools.py +203 -0
  6. iad/core/datatools.py +671 -0
  7. iad/core/docs/locators.ipynb +754 -0
  8. iad/core/dotstyle.py +99 -0
  9. iad/core/env.py +271 -0
  10. iad/core/events.py +650 -0
  11. iad/core/filesproc.py +1046 -0
  12. iad/core/fnctools.py +390 -0
  13. iad/core/label.py +240 -0
  14. iad/core/logs.py +182 -0
  15. iad/core/nptools.py +449 -0
  16. iad/core/one_dark.puml +881 -0
  17. iad/core/param/__init__.py +17 -0
  18. iad/core/param/confargparse.py +55 -0
  19. iad/core/param/paramaze.py +339 -0
  20. iad/core/param/tbox.py +277 -0
  21. iad/core/paths.py +563 -0
  22. iad/core/pdtools.py +2570 -0
  23. iad/core/pydantools/__init__.py +5 -0
  24. iad/core/pydantools/fixed_pydantic_yaml/__init__.py +32 -0
  25. iad/core/pydantools/fixed_pydantic_yaml/compat/__init__.py +0 -0
  26. iad/core/pydantools/fixed_pydantic_yaml/compat/hacks.py +76 -0
  27. iad/core/pydantools/fixed_pydantic_yaml/compat/old_enums.py +37 -0
  28. iad/core/pydantools/fixed_pydantic_yaml/compat/representers.py +92 -0
  29. iad/core/pydantools/fixed_pydantic_yaml/compat/types.py +122 -0
  30. iad/core/pydantools/fixed_pydantic_yaml/compat/yaml_lib.py +104 -0
  31. iad/core/pydantools/fixed_pydantic_yaml/ext/__init__.py +1 -0
  32. iad/core/pydantools/fixed_pydantic_yaml/ext/semver.py +152 -0
  33. iad/core/pydantools/fixed_pydantic_yaml/ext/versioned_model.py +113 -0
  34. iad/core/pydantools/fixed_pydantic_yaml/main.py +30 -0
  35. iad/core/pydantools/fixed_pydantic_yaml/mixin.py +281 -0
  36. iad/core/pydantools/fixed_pydantic_yaml/model.py +20 -0
  37. iad/core/pydantools/fixed_pydantic_yaml/py.typed +1 -0
  38. iad/core/pydantools/fixed_pydantic_yaml/version.py +1 -0
  39. iad/core/pydantools/models.py +560 -0
  40. iad/core/regexp.py +348 -0
  41. iad/core/short.py +308 -0
  42. iad/core/strings.py +635 -0
  43. iad/core/unc_panda.py +270 -0
  44. iad/core/units.py +58 -0
  45. iad/core/wrap.py +420 -0
  46. ialdev_core-0.1.0.dist-info/METADATA +73 -0
  47. ialdev_core-0.1.0.dist-info/RECORD +48 -0
  48. ialdev_core-0.1.0.dist-info/WHEEL +4 -0
iad/core/array.py ADDED
@@ -0,0 +1,1961 @@
1
+ from __future__ import annotations
2
+
3
+ import sys
4
+ import warnings
5
+ import dataclasses as dcs
6
+ from enum import Enum
7
+ from typing import (Any, Literal, overload, Collection, Callable, Iterable, Tuple,
8
+ Union, Iterator, get_type_hints, get_args, Container)
9
+ import re
10
+ import numpy as np
11
+ import numba
12
+ from numbers import Number
13
+
14
+ from .codetools import NamedObj
15
+ from . import as_list
16
+ from . import nptools as npt
17
+ from .datatools import rm_keys
18
+
19
+ __all__ = ['DForm', 'Array', 'FormArray', 'form_array', 'Color', '_wC2G']
20
+
21
+ FLOAT = np.float32 # default float for images
22
+ _wC2G = np.array([0.2125, 0.7154, 0.0721], FLOAT).reshape(3, 1) # RGB -> Gray weights
23
+
24
+
25
+ def _cast_int(inp, out, round):
26
+ """
27
+ Simple logic to handle clipping of inf values where casting float to int, where np.clip returns garbage.
28
+ Also for clipping when doing int -> int casting.
29
+
30
+ :param inp: input array
31
+ :param out: output array
32
+ :param round: If true, use round function of python, else let astype do the rounding (would be np.fix)
33
+ :return: array after casting
34
+ """
35
+ out_type = out.dtype.type
36
+ inp_type = inp.dtype.type
37
+
38
+ lims = np.iinfo(out.dtype)
39
+ mn, mx = out_type(lims.min), out_type(lims.max)
40
+ inp = inp.reshape(inp.size)
41
+ out = out.reshape(inp.size)
42
+ if inp.dtype.kind == 'f':
43
+ inf, ninf = inp_type(np.inf), inp_type(-np.inf)
44
+ for i in range(inp.size):
45
+ if inp[i] > mx or inp[i] == inf:
46
+ out[i] = mx
47
+ elif inp[i] < mn or inp[i] == ninf:
48
+ out[i] = mn
49
+ elif round:
50
+ out[i] = out_type(np.round(inp[i]))
51
+ else:
52
+ out[i] = out_type(inp[i])
53
+ else:
54
+ for i in numba.prange(inp.size):
55
+ if inp[i] > mx:
56
+ out[i] = mx
57
+ elif inp[i] < mn:
58
+ out[i] = mn
59
+ else:
60
+ out[i] = out_type(inp[i])
61
+
62
+
63
+ _cast_parallel = numba.jit(nopython=True, fastmath=True, parallel=True)(_cast_int)
64
+ _cast_no_parallel = numba.jit(nopython=True, fastmath=True)(_cast_int)
65
+ _cast_parallel_size = 50000 # size to start parallel
66
+
67
+
68
+ def cast(a: np.ndarray, dtype: Union[str, np.dtype],
69
+ *, out=None, copy=True, round=True):
70
+ """
71
+ Casting function of arrays to better cast in problematic use-cases, as a replace for astype()
72
+
73
+ Casting could change value for different reasons:
74
+ 1. range of the out type is narrower: i2(255:u2) = -1
75
+ 2. precision of the output type is lower: f4(i4.max) - i4.max = 1
76
+
77
+ The improvement is by:
78
+ 1. Clipping better for integers.
79
+ 2. Rounding instead of np.fix for integers. # todo update optional
80
+ 3. Allowing to include an output array.
81
+
82
+ By default,
83
+ - out of range values for float outputs are *automatically* rendered into +-inf.
84
+ - outputs values in integers and floats are clipped *if needed*.
85
+ - precision of float is decreasing when lowering the size of the float (as astype)
86
+
87
+ :param a: input array to be cast to a new type
88
+ :param dtype: dtype to cast
89
+ :param out: If exists, it is the array to be used as the output. It MUST be in the shape of the input, and in
90
+ the same dtype of the dtype requested, else casting would fail.
91
+ If None, a new array would be created.
92
+ :param round: When True, using the round function to round float numbers to integers.
93
+ If False, it uses the astype default strategy (np.fix)
94
+ Default is True.
95
+ :param copy: Relevant only to casting to same dtype, and when there isn't an "out" parameter:
96
+ If True, there is a new copy of the input. The default.
97
+ If False, the output is a view of the input and not a copy.
98
+ :return: Array that was cast to new dtype
99
+ """
100
+
101
+ dtype = np.dtype(dtype)
102
+ if out is not None and out.dtype != dtype:
103
+ raise TypeError(f"Missmatch of types between out type : {out.dtype} and cast to type {dtype}")
104
+
105
+ if a.dtype == dtype: # special cases where there is no changing in dtype
106
+ if out is not None and (a is out or shared_view(a, out)): # a and out have same view, and no casting
107
+ return out
108
+ elif not copy: # not copying - only if same type
109
+ if out is None:
110
+ return a
111
+ raise ValueError(f"Can't cast without copying when out array is present")
112
+
113
+ if dtype.kind == a.dtype.kind and dtype.itemsize >= a.dtype.itemsize: # only increasing size of same kind
114
+ if out is None:
115
+ return a.astype(dtype, casting='safe')
116
+ else:
117
+ np.copyto(out, a.astype(dtype, casting='safe'))
118
+ return out
119
+ else:
120
+ if dtype.kind == 'f' or np.can_cast(a, dtype,
121
+ casting='safe'): # inf values created automatically by astype
122
+ if out is None:
123
+ return a.astype(dtype, casting='unsafe')
124
+ else:
125
+ np.copyto(out, a.astype(dtype, casting='unsafe'))
126
+ return out
127
+ else: # int out or can't safely cast - int16 -> int8, float16 -> int32
128
+ if out is None: # create out if not included
129
+ out = np.empty_like(a, dtype=dtype)
130
+ cast_int = _cast_parallel if a.size > _cast_parallel_size else _cast_no_parallel
131
+ cast_int(a.astype('f4') if a.dtype == np.dtype('f2') else a, # numba doesn't support float16
132
+ out, round)
133
+ return out
134
+
135
+
136
+ def shared_view(a: np.ndarray, b: np.ndarray, shape=True) -> bool:
137
+ """Checks if two arrays share same data in the memory.
138
+
139
+ If shape is ``False`` - they may be of different shape, but size must match -
140
+ it does not deal with partial overlaps, and its main purpose to recognize if
141
+ one array is a full views of each over or both are views of some other array.
142
+ """
143
+ if shape:
144
+ if a.shape != b.shape:
145
+ return False
146
+ elif a.size != b.size:
147
+ return False
148
+ return a.__array_interface__['data'][0] == b.__array_interface__['data'][0]
149
+
150
+
151
+ def _name_to_dtype(name: str) -> np.dtype | None:
152
+ """
153
+ Return dtype for given string or None if not possible
154
+ :param name:
155
+ :return: dtype
156
+ """
157
+ try:
158
+ return np.dtype(name)
159
+ except TypeError:
160
+ return None
161
+
162
+
163
+ def _dtype_str(t: np.dtype):
164
+ return f"{t.kind}{t.itemsize}"
165
+
166
+
167
+ def _collect_result_type(*v: np.ndarray | None):
168
+ """Result type of multiple vector or scalar objects:
169
+
170
+ >>> _collect_result_type([1, 0,1, np.array([1,2,3])])
171
+ """
172
+ v = (_ for _ in v if _ is not None)
173
+ return np.result_type(*map(np.min_scalar_type, v))
174
+
175
+
176
+ class LinT:
177
+ """Helper class to implement linear transformation ``k*a+b`` over ndarray ``a``
178
+ with carefully managed type conversions and data copying
179
+ """
180
+
181
+ def __init__(self, k=1., b=0.):
182
+ """
183
+ :param k: initial k (a number of vector of ``chn`` size)
184
+ :param b: initial b (a number of vector of ``chn`` size)
185
+ """
186
+ # self.channels = 1 if chn is None else chn
187
+ self.k, self.b = (np.array(v, dtype=float, ndmin=1) for v in (k, b))
188
+
189
+ @property
190
+ def identity(self):
191
+ """True if transform does not change the data"""
192
+ return np.allclose(self.k, 1) and np.allclose(self.b, 0)
193
+
194
+ def __repr__(self):
195
+ one_or_all = lambda x: x[0] if np.all(x == x[0]) else x
196
+ k, b = map(one_or_all, (self.k, self.b))
197
+ return f"<{type(self).__name__}> [{k = }, {b = }]"
198
+ # f" chn: {self.channels}"
199
+
200
+ def _chain_mult(self, k, b):
201
+ return self.k * k, k * self.b + b
202
+
203
+ def __mul__(self, kb):
204
+ return type(self)(*self._chain_mult(*kb))
205
+
206
+ def __imul__(self, kb):
207
+ self.k, self.b = self._chain_mult(*kb)
208
+ return self
209
+
210
+ def apply(self, a: np.ndarray, *, out: np.ndarray) -> np.ndarray:
211
+ """
212
+ To minimize unnecessary operations return input for equivalent transform.
213
+ Also perform multiplication with dtype defined by target dtype and the input.
214
+
215
+ :param a: array to apply the transform to
216
+ :param out: array for output results
217
+ """
218
+ inp_type, out_type = a.dtype, out.dtype
219
+
220
+ k, b = map(npt.as_min_type, (self.k, self.b))
221
+ k_not_1, b_not_0 = not np.allclose(k, 1), not np.allclose(b, 0)
222
+ float_out = out_type.kind == 'f'
223
+
224
+ if not (k_not_1 or b_not_0): # no transformation - just casting
225
+ return cast(a, out_type, out=out)
226
+
227
+ if float_out: # transformation into float does not require clipping
228
+ tmp = np.multiply(a, self.k) if k_not_1 else a
229
+ tmp = np.add(tmp, self.b) if b_not_0 else tmp
230
+ return cast(tmp, out_type, out=out)
231
+
232
+ # determine dtype for operations
233
+ mlt_type = np.result_type(inp_type, k.dtype) if k_not_1 else inp_type
234
+ add_type = np.result_type(mlt_type, b.dtype) if b_not_0 else mlt_type
235
+ # upgrade dtype for possibly range expansion during operations (partially heuristics)
236
+ if add_type.kind != 'f' and add_type.itemsize <= 2: # todo we don't handle itemsize == 4 and increasing to 8?
237
+ add_type = np.dtype(f'i{add_type.itemsize * 2}')
238
+ tmp = out if add_type is out_type else np.empty(a.shape, add_type)
239
+ np.add(np.multiply(a, k, out=tmp) if k_not_1 else a, b, out=tmp)
240
+ return cast(tmp, out_type, out=out)
241
+
242
+ def __bool__(self):
243
+ return not self.identity
244
+
245
+
246
+ class Kind(Enum):
247
+ NA = None
248
+ UND = None
249
+ DISP = 'disp'
250
+ IMG = 'image'
251
+ IMAGE = 'image'
252
+ DEP = 'depth'
253
+ DEPTH = 'depth'
254
+ RGN = 'region'
255
+ REGION = 'region'
256
+ CNF = 'conf'
257
+ CONF = 'conf'
258
+
259
+ def __bool__(self):
260
+ return bool(self.value)
261
+
262
+ @classmethod
263
+ def from_str(cls, s: str):
264
+ """Return Color instance from name if exists or return None"""
265
+ return cls.__members__.get(s.upper(), None) if isinstance(s, str) else None
266
+
267
+
268
+ class Color(Enum):
269
+ """Enum listing differnt color codings of the channels."""
270
+ BIN = 'BIN'
271
+ GRAY = 'GRAY'
272
+ RGB = 'RGB'
273
+ B = 'BIN'
274
+ G = 'GRAY'
275
+ C = 'RGB'
276
+
277
+ def __bool__(self):
278
+ return bool(self.value)
279
+
280
+ @classmethod
281
+ def from_str(cls, s: str) -> str | None:
282
+ """Return Color instance from name if exists or return None"""
283
+ return cls.__members__.get(s.upper(), None) if isinstance(s, str) else None
284
+
285
+ @property
286
+ def channels(self):
287
+ """Number of color channels"""
288
+ if self is self.RGB: return 3
289
+ if self in (self.GRAY, self.BIN): return 1
290
+ raise RuntimeError("All cases must return channels number")
291
+
292
+ def short_code(self):
293
+ """Return the shortest alias to the color"""
294
+ code = self.value
295
+ for name, v in self.__class__.__members__.items():
296
+ if self is v and len(name) < len(code): code = name
297
+ return code
298
+
299
+ def __str__(self):
300
+ return self.value
301
+
302
+
303
+ def field(default=None, *, init=True, repr=True,
304
+ hash: bool | Callable[[Any], int] = None,
305
+ compare: bool | Callable[[Any, Any], bool] = True,
306
+ **metadata):
307
+ """
308
+ Wraps and modifies ``dataclass.field`` interface.
309
+
310
+ Changes:
311
+ 1. ``default`` argument is optional and is None by default
312
+ 2. ``metadata`` is provided not as a argument but as a keyword arguments
313
+ 2. ``hash`` may additionally be a function defining custom hash calculation
314
+ 3. ``compare`` may additionally be a function defining custom comparison
315
+
316
+ ``metadata`` will contain all the keyword arguments not defined in ``datclass.field``.
317
+
318
+ More, if ``hash`` and ``compare`` are Callables they are passed as ``True``
319
+ into ``dataclass.field``, but additionally stored in ``metadata`` with their
320
+ function values.
321
+
322
+ All that allows to simplify definition of DForm fields with extended functionality,
323
+ not only by supporting custom hashing and comparison, but also to
324
+ define entirely new functionality through metadata.
325
+
326
+ In particular ``calc`` keyword argument defines function to estimate field value from the data.
327
+
328
+ :param default
329
+ :param init: add as ``__init__`` argument
330
+ :param repr: include in ``__repr__``
331
+ :param hash: True to include or Callable to calc hash with
332
+ :param compare: True to include in comparison or Callable for special __eq__
333
+ :metadata: additional field attribute as described below:
334
+
335
+ :key calc: function to calculate field from data
336
+
337
+ :return:
338
+ """
339
+ if isinstance(hash, Callable):
340
+ metadata['hash'], hash = hash, True
341
+
342
+ if isinstance(compare, Callable):
343
+ metadata['compare'], compare = compare, True
344
+
345
+ if 'default_factory' in metadata: # move from metadata into locals
346
+ default_factory = metadata.pop('default_factory')
347
+
348
+ if 'calc' in metadata and not isinstance(metadata['calc'], Callable):
349
+ raise ValueError("Argument `calc` must be a function")
350
+
351
+ return dcs.field(**locals())
352
+
353
+
354
+ FormKind = Union['DForm', np.dtype, str, type]
355
+ hash_array = lambda a: None if a is None else hash(tuple(a))
356
+
357
+
358
+ # noinspection PyShadowingBuiltins
359
+ @dcs.dataclass
360
+ class DForm:
361
+ """
362
+ Provides extended description of *form* and *content* of data arrays.
363
+
364
+ In addition to the standard ``dtype`` field, identical to the ``ndarray.dtype``,
365
+ may optionally include information about
366
+ - statistics (``min, max, avr, std``)
367
+ - color information: (``color``, ``cax``)
368
+
369
+ Provides method to automatically ``transform`` array of different forms.
370
+
371
+ Also supplies convenience properties derived from this information:
372
+ - ``range`` (``max`` - ``min``)
373
+ - ``ndim`` - dimension of data if its describe a color image
374
+ - ``norm`` - ``(avr, std)`` how data was normalized
375
+
376
+ All those return ``None`` if not relevant.
377
+
378
+ Initialization
379
+ ==============
380
+ All ``DForm`` arguments are initialized with ``None`` by default:
381
+ ::
382
+ dtype: np.dtype or convertable into dtype
383
+ name: str | DForm,
384
+ min: float, max: float
385
+ avr: float | list[float], std: float | list[float]
386
+ color: DForm.Color, cax: int
387
+
388
+ Color
389
+ -----
390
+ Color is described by
391
+ - ``color: DForm.Color`` ``enum`` containing all the supportted color spaces
392
+
393
+ - ``cax`` - index of the color axis in the data dimensions
394
+
395
+ could be ``None`` also for ``color` like ``BIN``, ``GRAY``, wih dimensions ``[H x W]``,
396
+
397
+ unless additional color dimention is used ``[H x W x 1]``
398
+
399
+ Empty Form
400
+ ----------
401
+ ``dtype`` must be provided, except of the case of *empty* form:
402
+
403
+ >>> bool(DForm()) is False # empty form
404
+
405
+ Name and Registrattion
406
+ ----------------------
407
+ If ``name`` string is provided, a resulting instance is automatiocally
408
+ registered under ``name.lower()``.
409
+
410
+ It then may be accessed using either:
411
+ - ``registered()`` query or
412
+ - constructor with name as only argument.
413
+
414
+ >>> dform = DForm('Cu1', dtype='uint8', color=DForm.Color.RGB)
415
+ >>> assert DForm.from_name('Cu1') is dform
416
+ >>> assert DForm('Cu1') == dform
417
+
418
+ Use ``DForm.list_registered()`` to list of all the registered forms.
419
+ Defining a ``DForm`` with same currently replaces the previous one with a warning.
420
+ Avoid that to prevent having different forms with same name.
421
+
422
+ Practical Examples
423
+ ==================
424
+ Not any combination of the forms attributes are meaningfull.
425
+ Below are some typical cases.
426
+
427
+ Colored
428
+ ------
429
+ Color requires either ``range``, or ``norm`` to be defined.
430
+
431
+ Range of float color is usually ``(0,1)`` or ``(-1,1)``, but not necessary.
432
+
433
+ (Curled brackets means every option is possible independent on the others)
434
+ ::
435
+
436
+ color: {RGB, GRAY}
437
+ - dtype: u8 # unsigned 8 bits per channel
438
+ range: (0, 255) # type based range
439
+
440
+ - dtype: u16 # unsigned 16 bits
441
+ range: {(0, 1023), # specific (10 bits for example)
442
+ (0, 65535)} # type based
443
+
444
+ - dtype: i16
445
+ - range: {(-32768, 32767), (-32768, 32767), ...}
446
+
447
+ - dtype: {f16, f32 f64}
448
+ range: {(0, 1), (-1, 1)}
449
+
450
+
451
+ Supported basic transformations
452
+ -------------------------------
453
+ As ``DForm`` extends ``dtype`` so this transform extends notion of casting:
454
+ - only obvious and safe data manipulations are applied automatically,
455
+ like range mappings, rescaling, etc,
456
+ - aim to safe type casting keeping data values, except of type range clipping.
457
+
458
+ Those basic kind of transformations can be easily defined separately:
459
+ 1. range -> range
460
+ 2. normalization (avr, std) -> (avr, std)
461
+ 3. color -> color
462
+ 4. dtype -> dtype
463
+
464
+ All the transformations (except of dtype) are performed if the corresponding attribute is
465
+ defined in both source and target DataForms.
466
+
467
+ Missing input attribute makes transformation impossible and it fails.
468
+
469
+ Missing output attribute keeps data (in its respect) unchanged, however
470
+ it could be changed because of another attributes.
471
+
472
+ In this example:
473
+ ``range, color:RGB -> color:GRAY``
474
+ input range is irrelevant, but the color transformation does occur.
475
+
476
+ Implementation tries to minimize number of calculations and data copy.
477
+ That can be facilitated even farther by providing the ``out`` argument
478
+ with array to write the data into.
479
+
480
+ This array must match both the source dimensions and target form.
481
+ If possible it is used along all the transformation stages to reduce copying.
482
+
483
+ Supported Combinations
484
+ ======================
485
+ Only some of the combinations of the attributes in the source and target forms can be allowed.
486
+
487
+ DType
488
+ ~~~~~
489
+ ``dtype`` is always defined for both source and target forms, and any conversion is possible
490
+ together with allowed transform of any other attributes.
491
+
492
+ Range
493
+ ~~~~~
494
+ Mathematically ``range`` is incompatible with ``normalization``, therefore
495
+ - ``range, norm -> range norm`` is not allowed
496
+ - ``range -> std`` is possible however
497
+ - ``std -> range`` is not allowed since the initial range is not known
498
+
499
+ When automatic transform is not possible, a ``TypeError`` is raised.
500
+
501
+ That sometimes can be avoided if desired transform is explicitly enabled or disabled using
502
+ ``do_range`` and ``do_norm`` arguments. Then exception is raised if a specific transform(s)
503
+ are explicitly enabled but is not possible.
504
+
505
+ Color
506
+ ~~~~~
507
+ ``color`` can be used in different combinations:
508
+ ::
509
+
510
+ color, range -> color, range
511
+ color, range -> color, norm
512
+ color, norm -> color, norm
513
+
514
+ """
515
+
516
+ _reg_names = {}
517
+ _reg_hashes = {}
518
+
519
+ _auto_dtype = True # allow guessing dtype when not provided
520
+ _auto_names = True
521
+ _default_dtype = np.dtype('f4')
522
+ _verify_cache = False or sys.gettrace() # for debugging. FIXME: remove after freeze
523
+
524
+ @classmethod
525
+ def auto_naming(cls, state: bool = None):
526
+ """
527
+ Controls over possibility to auto-generate ``DForms`` using name coding conventions
528
+
529
+ If turned ON (default) than accessing a ``DForm`` by name following this convention, instead of
530
+ raising an error, generates and registers it automatically.
531
+
532
+ :param state:True - AutoNaming On (default),
533
+ False - AutoNaming Off.
534
+ None - for getting the current mode.
535
+ :return: the current mode if `state` is ``None`` or ``None``
536
+
537
+ Examples:
538
+ >>> DForm('CNu1')
539
+ Works
540
+ >>> DForm.auto_naming(False)
541
+ >>> DForm('CNu2')
542
+ KeyError: "No registered DForm named 'CNu2'"
543
+ """
544
+ if state is None:
545
+ return cls._auto_names
546
+ cls._auto_names = state
547
+ return None
548
+
549
+ @classmethod
550
+ def _field_type(cls, fld, allowed: Container) -> type:
551
+ """Found in the field defintion first type listed in `allow` or raise ``TypeError``.
552
+ """
553
+ if not getattr(cls, '_fields_types', None):
554
+ cls._fields_types = get_type_hints(cls)
555
+
556
+ _type = cls._fields_types[fld]
557
+ type_options = [_type] if isinstance(_type, type) else get_args(_type)
558
+ for _type in allowed: # find first!
559
+ if _type in type_options:
560
+ return _type
561
+ raise TypeError(f"Neither {type_options=} for field {fld} are {allowed=}")
562
+
563
+ @classmethod
564
+ def _parse_name(cls, name: str) -> dict | None:
565
+ """
566
+ Parse through the name, validate and return fields as ``dict``"
567
+
568
+ :param name: The name of dform for parsing
569
+ :return: dict of all attributes of dform
570
+
571
+ **The Pattern**:
572
+ ::
573
+ [C|G|B][R[<min>:]<max>][N[<std>][|<avr>]]((u|i|f)<bytes>|(s|b)<bits>)
574
+
575
+ **Conventions**:
576
+ ::
577
+ (x|y|...) - selection group
578
+ [] - optional group
579
+ <name> - group containing value of variable `name`
580
+
581
+ C - var:color = 'RGB'
582
+ G - var:color = 'Grey'
583
+ B - var:color = 'Binary'
584
+
585
+ **Group ideyntification symbols starting definition of**:
586
+ ::
587
+ R - Range
588
+ N - Normalization
589
+
590
+ **Defaults for missing groups**:
591
+ ::
592
+ R<max> = R<0>:<max>
593
+ N = N<1>|<0>
594
+
595
+
596
+ The only *mandatory* group is for **type**, which may be given in two forms:
597
+ - as number of bits prefixed by 'b' or 's' (for signed) \n
598
+ only for `int` types, contaner ``dtype`` is deduced from that
599
+ - using ``np.dtype`` shortcut notations (u1 = uint8, f8 = float64, ...)
600
+
601
+ **Limitations**:
602
+ - Parameters of `range` and `normalization` are represented by ``float``,
603
+ which can be parsed only in this form:
604
+ ::
605
+ [-][<digits>.]<digits>
606
+
607
+ Though `DForm` supports defining normalization *per channel*,
608
+ that is **not** supported by the name parser.
609
+
610
+ Examples:
611
+ >>> DForm('CNu1') == DForm(color='RGB', std=1, avr=0, dtype=np.uint8)
612
+ True
613
+ >>> DForm('N2.5|-3f2') == DForm(std=2.5, avr=-3, dtype=np.float16)
614
+ True
615
+ >>> DForm('R1N|1.2f4') == DForm(min=0, max=1, avr=1.2, dtype=np.float32)
616
+ True
617
+ >>> DForm('Gb10') == DForm(color=Color.GRAY, min=0, max=1023, dtype=np.uint16)
618
+ True
619
+ >>> DForm('s2') == DForm(min=-1, max=1, dtype='i1')
620
+ True
621
+ """
622
+
623
+ _codes_rex = re.compile(r"""
624
+ (?P<color>[GCB])?
625
+ (?:R
626
+ (?:
627
+ (?P<min>-?\d*\.?\d+):
628
+ )?
629
+ (?P<max>-?\d*\.?\d+)
630
+ )?
631
+ (?P<norm>N
632
+ (?P<std>\d*\.?\d+)?
633
+ (?:\|
634
+ (?P<avr>-?\d*\.?\d+)
635
+ )?
636
+ )?
637
+ (?:
638
+ (?P<bs>[bs])(?P<bits>[1-9]\d*) |
639
+ (?P<dtype>[uif][1248])
640
+ )
641
+ """, re.VERBOSE)
642
+
643
+ def set_default(fld, val):
644
+ if attr[fld] is None:
645
+ attr[fld] = val
646
+
647
+ def cast_numeric_attr(fld):
648
+ if attr[fld] is not None:
649
+ attr[fld] = cls._field_type(fld, (float, int))(attr[fld])
650
+ return attr[fld]
651
+
652
+ if (m := _codes_rex.fullmatch(name)) and (attr := m.groupdict()):
653
+ # shortcuts
654
+ if attr['max'] is not None: # R10 -> R0:10
655
+ set_default('min', 0)
656
+ if attr['norm']: # N -> N1 -> N1|0
657
+ set_default('std', 1)
658
+ set_default('avr', 0)
659
+
660
+ r_min, r_max, *_ = map(cast_numeric_attr, ['min', 'max', 'std', 'avr', 'bits'])
661
+
662
+ if bits_code := attr.pop('bs'):
663
+ if (bits := attr['bits']) > 64:
664
+ raise ValueError(f"Invalid number of {bits=}")
665
+ if bits_code == 'b': # unsigned of size bits
666
+ t_min, t_max = 0, 2 ** bits - 1
667
+ attr['dtype'] = np.min_scalar_type(t_max)
668
+ else: # s - signed
669
+ t_min, t_max = -2 ** (bits - 1), 2 ** (bits - 1) - 1
670
+ attr['dtype'] = np.min_scalar_type(t_min)
671
+ else: # if not a bits code, then dtype is defined
672
+ dtype = attr['dtype'] = np.dtype(attr['dtype']) # ensure valid dtype
673
+ t_info = (np.finfo if dtype.kind == 'f' else np.iinfo)(dtype)
674
+ t_min, t_max = t_info.min, t_info.max
675
+
676
+ if not ((r_min is None or r_min >= t_min) and (
677
+ r_max is None or r_max <= t_max)):
678
+ raise ValueError(f"Range ({r_min},{r_max}) is inconsistent with {bits_code}{bits}")
679
+
680
+ attr['color'] = Color.from_str(attr['color'])
681
+
682
+ # remove parsed keys which are not valid fields
683
+ return rm_keys(attr, set(attr).difference(cls.fields()))
684
+
685
+ return None
686
+
687
+ @classmethod
688
+ def _register(cls, dform):
689
+ """ Add to the global registry """
690
+ h = hash(dform)
691
+ if found := cls._reg_hashes.get(h, None):
692
+ if dform.name and found.name and found.name != dform.name:
693
+ warnings.warn(f"DForm {dform} is already known under {found.name}")
694
+
695
+ if dform.name:
696
+ key = hash(dform.name.lower())
697
+ if found := cls._reg_names.get(key, None):
698
+ warnings.warn(f"{cls.__name__}('{dform.name}') overrides registered {found.name}")
699
+ cls._reg_names[key] = dform
700
+
701
+ @classmethod
702
+ def from_name(cls, name: str, *, auto: bool = None, reg=True) -> DForm:
703
+ """
704
+ Check if the name is registered as a dform in the registered list.
705
+ If the name is not registered, AutoNaming is On, and the name have values other than None,
706
+ it includes it in the registered list and return the dform created
707
+ Else we return the dform if it was found, or None if it wasn't found
708
+ :param name: The dform to check
709
+ :param reg: look into the registry
710
+ :param auto: allow automatic creation from name
711
+ :return: dform or None
712
+ """
713
+ if reg and (found := cls._reg_names.get(hash(name.lower()), None)):
714
+ return found
715
+ if auto is True or auto is None and cls._auto_names:
716
+ if attrs := cls._parse_name(name):
717
+ return DForm(name=name, **attrs)
718
+
719
+ @classmethod
720
+ def list_registered(cls, out=False) -> dict[int, DForm] | None:
721
+ """
722
+ Print list of the registered DataForms.
723
+
724
+ :param out: if True just silently return registration dict
725
+ :return: None or dict
726
+ """
727
+ if out:
728
+ return cls._reg_names
729
+
730
+ for i, (k, v) in enumerate(cls._reg_names.items()):
731
+ print(f"{i:3}. {v.name:20}: {v}")
732
+ return None
733
+
734
+ class Shaper:
735
+ def __init__(self, src_shape, src_form: DForm, trg_form: DForm):
736
+ """
737
+ Summary of possible color options and resulting re-shaping:
738
+ ::
739
+ case src trg re-shape
740
+ chn cax chn cax
741
+ 0 - - + error
742
+ 1 - - - - =
743
+ 2 + - - =
744
+ 3 1 - 1 - =
745
+ 4 1 - 1 + + dim
746
+ 5 1 - 3 + + dim
747
+ 6 1 + 1 - - dim
748
+ 7 3 + 1 - - dim
749
+ 8 1 + 1 + ? move
750
+ 9 3 + 1 + ? move
751
+ 10 1 + 3 + ? move
752
+
753
+ :param src_shape:
754
+ :param src_form:
755
+ :param trg_form:
756
+ """
757
+ # start with 'no shape changes' configuration:
758
+ self.src_shape = self.trg_shape = src_shape
759
+ self.channels, self.cax = None, None
760
+ # if that is actually the case - exit
761
+ if not (src_form and trg_form and src_form.color and trg_form.color): return
762
+ src_chn, trg_chn = src_form.color.channels, trg_form.color.channels
763
+ if src_chn == trg_chn and src_form.cax == trg_form.cax: return
764
+
765
+ self.channels = trg_chn != src_chn and (src_chn, trg_chn)
766
+ self.cax = trg_form.cax != src_form.cax and (src_form.cax, trg_form.cax)
767
+
768
+ shape = list(src_shape) # evaluate target shape
769
+ if src_form.cax is not None: shape.pop(src_form.cax)
770
+ if trg_form.cax is not None:
771
+ shape.insert(trg_form.cax % (len(shape) + 1), trg_chn)
772
+ self.trg_shape = tuple(shape)
773
+
774
+ def __bool__(self):
775
+ return self.src_shape != self.trg_shape
776
+
777
+ def move_cax(self, a: np.ndarray, src_trg: Literal['src', 'trg'],
778
+ state: bool):
779
+ """ Move cax (if needed) of the array
780
+
781
+ :param a: data array to move axis
782
+ :param src_trg: which data is passed: 'src' or 'trg'
783
+ :param state: ``True`` cax to the end, ``False`` - restore
784
+ :return: array with cax moved
785
+ """
786
+ if self.cax:
787
+ src_trg = src_trg == 'trg'
788
+ chn = self.channels[src_trg]
789
+ cax = self.cax[src_trg] % chn if self.cax[src_trg] else None
790
+
791
+ if cax and cax != (end := chn - 1):
792
+ from_to = (cax, end) if state else (end, cax)
793
+ return np.moveaxis(a, *from_to)
794
+ return a
795
+
796
+ CALC = NamedObj('CALC')
797
+
798
+ # =========================================================================
799
+ name: FormKind | None = field(hash=False, compare=False)
800
+ dtype: Any = None
801
+ bits: int = None
802
+ min: float = field(calc=np.nanmin)
803
+ max: float = field(calc=np.nanmax)
804
+ avr: float | np.ndarray | Collection[float] = field(hash=hash_array, calc=np.nanmean)
805
+ std: float | np.ndarray | Collection[float] = field(hash=hash_array, calc=np.nanstd)
806
+ color: Color | str = None
807
+ cax: int = None # IDEA: turn into axis: ordered list of axis names
808
+ units: float = None # IDEA: add support for physical units
809
+ kind: Kind = None
810
+
811
+ # ==========================================================================
812
+
813
+ @classmethod
814
+ def _meta(cls, fld: str, key: str, default=None):
815
+ """
816
+ Return key's value from the metadata of a given field
817
+ :param fld: name of the field
818
+ :param key: meta-key
819
+ :param default: return this if key is not in metadata
820
+ """
821
+ return cls.__dataclass_fields__[fld].metadata.get(key, default)
822
+
823
+ @classmethod
824
+ def _filter_apply_fields(cls, fld_proc: Callable[[dcs.Field], Tuple[Any, bool]], *,
825
+ include: str | Collection[str] | None = None,
826
+ exclude: str | Collection[str] | None = None):
827
+ """
828
+ Generator yields first item returned by applying given `fld_proc` function
829
+ on all or some of the dataclass fields.
830
+
831
+ Function `fld_proc` must accept `Field` object and return a tuple `(result, accept)`:
832
+ - `result` to be yielded ONLY if
833
+ - `accept` is True
834
+
835
+ Function is applied either *all* the dataclass fields, unless *one* of `include` or `exclude`
836
+ arguments says differently (``NameError`` raised if invalid field name is given)
837
+
838
+ :param fld_proc:
839
+ :param include: fields to include
840
+ :param exclude:
841
+ """
842
+ cls_fields = cls.__dataclass_fields__
843
+
844
+ if include:
845
+ if exclude:
846
+ raise ValueError("Both include AND exclude args are provided!")
847
+
848
+ include = as_list(include, collect=set)
849
+ if inv := include.difference(cls_fields):
850
+ raise NameError(f"Invalid fields: {inv}")
851
+ fields = include
852
+ elif exclude:
853
+ exclude = as_list(exclude, collect=set)
854
+ if inv := exclude.difference(cls_fields):
855
+ raise NameError(f"Invalid fields: {inv}")
856
+ fields = set(cls_fields) - exclude
857
+ else:
858
+ fields = cls_fields
859
+
860
+ for name in fields:
861
+ val, use = fld_proc(cls_fields[name])
862
+ if use:
863
+ yield val
864
+
865
+ @classmethod
866
+ def field_calc(cls, fld: str):
867
+ return cls._meta(fld, 'calc')
868
+
869
+ @classmethod
870
+ def from_data(cls, a: np.ndarray, /, name: str = None, *,
871
+ dtype: np.dtype = None, **attrs) -> DForm:
872
+ """
873
+ Create DForm by estimating some attributes from given ``data``, which
874
+ must be either ``ndarray`` or iterable convertible into it.
875
+
876
+ In this case its ``dtype`` is first defined in this order:
877
+ ``dform`` argument, ``dform.dtype``, ``DForm._default_dtype``
878
+
879
+ Attributes values may be inherited from given dform if provided.
880
+ That can be overriden by providing corresponding argument,
881
+ which could have different meanings:
882
+ - ``None``: not provided
883
+ - <value>: use it for the attribute
884
+ - "calculate": estimate from the data using internally defined function
885
+ - `Callable`: estimate from the data using this function
886
+
887
+ If attribute value is not defined by the argument and ``dform`` is ``None``,
888
+ it remains None, except of `dtype` which is then assigned `a.dtype``.
889
+
890
+ :param a: ndarray to use for calculations
891
+ :param dform: Provides base to inherit the attributes
892
+ :param dtype:
893
+ :param name: Optional name for new DForm
894
+ :keyword min: min value of the range
895
+ :keyword max: max value of the range
896
+ :keyword std: std of the data distribution
897
+ :keyword avr: avr of the data distribution
898
+ :keyword color: one of the `DForm.Color`
899
+ :keyword cax: ``int`` - color axes location (if color is defined)
900
+ :return: resulting dform
901
+ """
902
+ if not isinstance(a, np.ndarray):
903
+ a = np.array(a, dtype=dtype)
904
+ if not dtype:
905
+ dtype = a.dtype
906
+
907
+ def value(fld, v):
908
+ if v == 'calculate':
909
+ if calc := cls._meta(fld, 'calc'):
910
+ return calc(a)
911
+ raise NotImplementedError(f"Calc function not defined for '{fld}'")
912
+ if isinstance(v, Callable):
913
+ return v(a)
914
+ return v
915
+
916
+ attrs = {k: value(k, v) for k, v in attrs.items()}
917
+
918
+ return DForm(name=name, dtype=dtype, **attrs)
919
+
920
+ @classmethod
921
+ def from_merge(cls, dform: DForm | str | type = None, **fields) -> DForm:
922
+ """Create a new object by replacing one with new fields values.
923
+
924
+ - If ``dform`` is omitted, use fields defaults.
925
+ - If ``fields`` are *not provided either*, return ``None``!
926
+
927
+ :param dform to merge with in any form acceptable by DForm constructor
928
+ :param fields: any subset of valid DForm fields
929
+ """
930
+ if dform:
931
+ if not isinstance(dform, cls):
932
+ dform = cls(dform)
933
+ if not fields:
934
+ return dform
935
+ fields = dict(dform.items()) | fields
936
+ if fields:
937
+ return DForm(**fields)
938
+ raise ValueError("No arguments in from_merge")
939
+
940
+ def type_info(self) -> str:
941
+ base = f"{self.name or ''}{_dtype_str(self.dtype)}"
942
+ if not self.extra_info():
943
+ return f"☉{base}"
944
+ rng = "" if self.min is None else f"⎣{self.min:g}➔{self.max:g}⌉"
945
+ vals = lambda s: ', '.join(f"{_:.3g}" for _ in (s if hasattr(s, '__len__') else [s]))
946
+ avr = "" if self.avr is None else f"<{vals(self.avr)}>"
947
+ std = f"" if self.std is None else f"σ{vals(self.std)}"
948
+ return f"Ⓕ{base}{rng}{avr}{std}"
949
+
950
+ def __repr__(self):
951
+ if self.name and self.name == self._code_name():
952
+ return 'Ⓕ' + self.name
953
+ return self.type_info()
954
+
955
+ def same(self, other: FormKind, *, name: bool | Literal['case'] = 'case'):
956
+ """
957
+ Return ``True`` if other is the same as self including name.
958
+ Default comparison of names is case-sensitive, use ``True`` otherwise.
959
+
960
+ :param other: dform to compare with
961
+ :param name: 'case' - case-sensitive when comparing names. The default.
962
+ True - compare names. False - don't.
963
+ :return: bool
964
+ """
965
+ # Consider: replace that by hash comparison once DForm is frozen
966
+ cls = type(self)
967
+ if not isinstance(other, cls):
968
+ # noinspection PyBroadException
969
+ try:
970
+ other = cls(other) # creates default dform (f4) if dform of 'other' is None
971
+ except:
972
+ return False
973
+
974
+ if name not in (allowed := (True, False, 'case')):
975
+ raise ValueError(f'name argument {name} is not among {allowed=}')
976
+ if name and not (
977
+ self.name == other.name if (name is True or self.name is None or other.name is None) else
978
+ self.name.lower() == other.name.lower()
979
+ ): return False
980
+
981
+ def not_equal(f):
982
+ """Return only yield-able False"""
983
+
984
+ def size(v):
985
+ return 0 if v is None else 1 if not isinstance(v, np.ndarray) else len(v)
986
+
987
+ if f.compare:
988
+ v1, v2 = getattr(self, f.name), getattr(other, f.name)
989
+ size_v1 = size(v1)
990
+ size_v2 = size(v2) # Consider: cmp() is not enough?
991
+ if (cmp := f.metadata.get('compare', None)) and cmp(v1, v2) or \
992
+ (size_v1 == size_v2 and size_v1 > 1 and (v1 == v2).all()) or \
993
+ (size_v1 == size_v2 and size_v1 <= 1 and v1 == v2): # 1D
994
+ return None, False # don't yield (None not to be used)
995
+ else:
996
+ return False, True # yield False
997
+ return None, False # don't yield
998
+
999
+ for _ in self._filter_apply_fields(not_equal): return _
1000
+ return True
1001
+
1002
+ def __eq__(self, other: DForm):
1003
+ return self.same(other, name=False)
1004
+
1005
+ def _calc_hash(self):
1006
+ d = self.__dict__
1007
+
1008
+ def hash_filter(f):
1009
+ if f.hash is False:
1010
+ return None, False
1011
+
1012
+ v = d[f.name]
1013
+ if v is not None and (hash_fnc := f.metadata.get('hash', None)):
1014
+ v = hash_fnc(v)
1015
+ return v, True
1016
+
1017
+ return hash(tuple(self._filter_apply_fields(fld_proc=hash_filter)))
1018
+
1019
+ def __hash__(self):
1020
+ found_hash = getattr(self, '_cached_hash', None)
1021
+ if not found_hash or self._verify_cache: # FIXME: freeze DForm and remove verifications
1022
+ now_hash = self._calc_hash()
1023
+ if found_hash and now_hash != found_hash: # can be true only if self.__verify_cache
1024
+ raise RuntimeError(f"Inconsistent cache for {self}")
1025
+ self._cached_hash = now_hash
1026
+
1027
+ return self._cached_hash
1028
+
1029
+ def extra_info(self) -> dict: # ToDo: replace with hash based after DForm freeze
1030
+ """Return dict with extra type information (besides name and dtype) which is not None."""
1031
+ return dict(
1032
+ self._filter_apply_fields(
1033
+ exclude=('name', 'dtype'),
1034
+ fld_proc=lambda f: (
1035
+ (f.name, val := self.__dict__[f.name]),
1036
+ val is not None
1037
+ )
1038
+ )
1039
+ )
1040
+
1041
+ @classmethod
1042
+ def fields(cls, hash_only=False, exclude=None, include=None) -> Iterator[str]:
1043
+ """Return iterator over dataclass fields' names.
1044
+ :param hash_only: if ``True`` - only those used for hash calculation
1045
+ :param exclude: optionally exclude one or more of those
1046
+ :param include: only those, first validate their existence
1047
+ """
1048
+ return cls._filter_apply_fields(
1049
+ fld_proc=lambda f: (f.name, not (hash_only and f.hash is False)),
1050
+ exclude=exclude, include=include
1051
+ )
1052
+
1053
+ def items(self, hash_only=False) -> Iterator[tuple[str, Any]]:
1054
+ """Return iterator over fields items (name, value)
1055
+ :param hash_only: if True return only fields used in hash calculations
1056
+ """
1057
+ d = self.__dict__
1058
+ return ((k, d[k]) for k in self.fields(hash_only=hash_only))
1059
+
1060
+ @property
1061
+ def range(self) -> float | None:
1062
+ """None or length of the range (max - min)"""
1063
+ return None if self.min is None else self.max - self.min
1064
+
1065
+ @property
1066
+ def norm_dim(self) -> int:
1067
+ """
1068
+ Number of normalization channels, also can be used to check if normalized:
1069
+ - 0 - no normalization,
1070
+ - 1 - normalize without separation by channel
1071
+ - 1+ - number of separate channels
1072
+
1073
+ Normalization is positive if either avr or std is defined.
1074
+ """
1075
+ return self._norm_channels
1076
+
1077
+ def _code_name(self):
1078
+ # todo Why is this different name convention from type_info?
1079
+ # [C | G | B][R[ < min >:] < max >]][N[ < std >][ |][ < avr >]](u | i) < bytes > | (s | b) < bits >
1080
+ # Idea: produce signal if _parse_name(_code_name(x)) != x
1081
+ clr = self.color.short_code() if self.color else ''
1082
+
1083
+ if self.range:
1084
+ rng = 'R'
1085
+ if self.min: rng += f"{self.min:g}:"
1086
+ rng += f"{self.max:g}"
1087
+ else:
1088
+ rng = ''
1089
+
1090
+ if self.norm_dim > 1:
1091
+ nrm = 'N...'
1092
+ elif self.norm_dim == 1:
1093
+ std, avr = (x if x is None else x.item() for x in (self.std, self.avr))
1094
+ nrm = 'N'
1095
+ if not (std == 1 and avr == 0):
1096
+ if std is not None:
1097
+ nrm += f"{std:g}"
1098
+ if avr is not None:
1099
+ nrm += f"|{avr:g}"
1100
+ else:
1101
+ nrm = ''
1102
+
1103
+ typ = ''
1104
+ if self.dtype.kind == 'i':
1105
+ if self.range and self.min == -self.max and (b := np.log2(self.max + 1)).is_integer():
1106
+ rng = ''
1107
+ if (b := int(b + 1)) // 8 != self.dtype.itemsize:
1108
+ typ = f's{b}'
1109
+ elif self.dtype.kind == 'u':
1110
+ if self.range and self.min == 0 and (b := np.log2(self.max + 1)).is_integer():
1111
+ rng = ''
1112
+ if (b := int(b)) // 8 != self.dtype.itemsize:
1113
+ typ = f'b{int(b)}'
1114
+ return f"{clr}{rng}{nrm}{typ or _dtype_str(self.dtype)}"
1115
+
1116
+ def _copy_from(self, src: DForm | dict):
1117
+ """Set fields by copying them from other DForm or dict
1118
+ (if DForm _norm_channels are also copied).
1119
+
1120
+ :return: number of fields set with not `None` values
1121
+ """
1122
+ if isinstance(src, DForm):
1123
+ fields = src.fields(hash_only=False)
1124
+ self._norm_channels = src._norm_channels
1125
+ src = src.__dict__
1126
+ elif isinstance(src, dict):
1127
+ fields = self.fields(hash_only=False, include=src)
1128
+ else:
1129
+ raise TypeError(f"Invalid {type(src) = }!")
1130
+
1131
+ count_real_values = 0
1132
+ for f in fields:
1133
+ if (val := src[f]) is not None:
1134
+ count_real_values += 1
1135
+ setattr(self, f, val)
1136
+ return count_real_values
1137
+
1138
+ def __post_init__(self):
1139
+ self._norm_channels = 0 # number of normalization channels (0 - not normalized)
1140
+
1141
+ parsed = self._decode_name() # name parsed and return True only if only argument DForm(name)
1142
+ self._init_dtype()
1143
+ self._init_range()
1144
+ self._init_norm()
1145
+ self._init_color()
1146
+ self._init_kind()
1147
+
1148
+ # check name consistency for multi-fields init: DFom(name, color, dtype, range, ...)
1149
+ if (not parsed and self.name
1150
+ and (attrs := self._parse_name(self.name)) # name encodes attributes
1151
+ and self.same(DForm(**attrs), name=False)): # matching provided through arguments
1152
+ raise NameError(f"Name is inconsistent with form's attributes")
1153
+
1154
+ DForm._register(self)
1155
+
1156
+ def _decode_name(self) -> bool | int:
1157
+ """ Unpack attributes encoded in the name.
1158
+
1159
+ The `name` attribute string is parsed ONLY if it was the only argument.
1160
+
1161
+ :return: if state changed return ``True`` or number of fields set to not ``None``
1162
+ """
1163
+ if not (name := self.name):
1164
+ return False
1165
+
1166
+ name_is_str = isinstance(name, str)
1167
+ extra = self.extra_info() # the current state of extra attributes
1168
+
1169
+ if not (extra or self.dtype): # name was the only argument: DForm(name)
1170
+ if name_is_str: # from a single str argument: DForm('Cu1') or DForm('KnownType')
1171
+ if dform := self.from_name(name, auto=False): # query from DataForms registry
1172
+ return self._copy_from(dform) # copy from registered type
1173
+ elif attrs := self._parse_name(name):
1174
+ return self._copy_from(attrs)
1175
+ else:
1176
+ raise NameError(f"Unknown DForm '{self.name}'")
1177
+ elif isinstance(name, DForm):
1178
+ return self._copy_from(name) # including name
1179
+ elif isinstance(name, type): # first arg is type -> dtype
1180
+ self.name = None
1181
+ self.dtype = np.dtype(name)
1182
+ return True
1183
+ else:
1184
+ raise ValueError(f"Invalid name argument {self.name}")
1185
+ elif not name_is_str: # must be string if other arguments are given
1186
+ raise ValueError(f"Expected string for DForm name {self.name}")
1187
+ elif dtype := _name_to_dtype(name): # name encodes dtype
1188
+ if extra: # this form is more than pure dtype
1189
+ raise NameError(f'Assigning dtype {name=} to custom dform')
1190
+ if self.dtype and np.dtype(self.dtype) != dtype:
1191
+ raise NameError(f"Assigning dform.{name=} for dtype {self.dtype}")
1192
+
1193
+ self.name = None # name argument was dtype name - not to register
1194
+ self.dtype = dtype
1195
+ return True
1196
+ return False
1197
+
1198
+ def _init_color(self):
1199
+ if self.color:
1200
+ self.color = Color(self.color) # convert from possible string
1201
+ if self.cax is None and self.color.channels > 1:
1202
+ self.cax = - 1 # default for multiple color channels
1203
+ # here we assume that if normalization is defined per channels - that color channels
1204
+ if self.norm_dim > 1 and self.norm_dim != self.color.channels: raise ValueError(
1205
+ f"Number of color & normalization channels ({self.color.channels}) != {self.norm_dim}")
1206
+
1207
+ if self.color is Color.BIN:
1208
+ mn, mx = map(self.dtype.type, (0, 1))
1209
+ if self.min is None:
1210
+ self.min, self.max = mn, mx
1211
+ elif (self.min, self.max) != (mn, mx):
1212
+ raise ValueError(f"DForm {self} with BW color must have range [0,1]")
1213
+
1214
+ if self.color is not Color.BIN and not (self.norm_dim or self.range):
1215
+ if self.dtype in (np.uint8, np.uint16):
1216
+ info = np.iinfo(self.dtype)
1217
+ self.min, self.max = info.min, info.max
1218
+ else:
1219
+ raise TypeError(f"Color form of type {self.dtype} requires norm or range")
1220
+ elif self.cax is not None:
1221
+ raise ValueError(f"Color axis is defined {self.cax=} without color")
1222
+
1223
+ def _init_dtype(self) -> bool:
1224
+ """Process state to determine and set dtype.
1225
+ :return: ``True`` if found and changed
1226
+ """
1227
+ if self.dtype is None:
1228
+ if not self._auto_dtype: # see if guessing is allowed
1229
+ raise TypeError('Missing dtype in DForm definition')
1230
+ elif self.color and Color(self.color) is Color.RGB:
1231
+ self.dtype = 'uint8'
1232
+ else:
1233
+ self.dtype = self._default_dtype
1234
+ return True
1235
+
1236
+ self.dtype = np.dtype(self.dtype)
1237
+ if self.dtype.kind not in 'biuf':
1238
+ raise TypeError(f"Not a numeric type for dtype!")
1239
+ return False
1240
+
1241
+ def _init_range(self):
1242
+ """:return: ``True`` if range been set"""
1243
+
1244
+ if (self.min, self.max) == (None, None):
1245
+ return False
1246
+
1247
+ if self.min is None:
1248
+ self.min = 0
1249
+ elif self.max is None:
1250
+ raise ValueError("DForm min is defined without max")
1251
+
1252
+ dtype = _collect_result_type(self.min, self.max)
1253
+ if not np.can_cast(dtype, self.dtype):
1254
+ TypeError(f"Can't cast range {dtype=} into {self.dtype}")
1255
+
1256
+ self.min, self.max = map(self.dtype.type, (self.min, self.max))
1257
+ if self.max <= self.min:
1258
+ raise ValueError(f"DForm {self} has invalid range")
1259
+ return True
1260
+
1261
+ def _init_norm(self):
1262
+ """
1263
+ Calculate, update and return `_norm_channels`.
1264
+ :return: 0 means neither `std` nor `avr` are defined
1265
+ """
1266
+ if self.std is not None:
1267
+ self.std = np.asarray(self.std).flatten()
1268
+ self._norm_channels = max(self._norm_channels, self.std.size)
1269
+ if self.std.min() < 10 * np.finfo('float32').eps:
1270
+ raise ValueError(f"Expected DForm std = {self.std.min()} > eps")
1271
+
1272
+ if self.avr is not None:
1273
+ self.avr = np.asarray(self.avr).flatten()
1274
+ self._norm_channels = max(self._norm_channels, self.avr.size)
1275
+
1276
+ return self._norm_channels
1277
+
1278
+ def _init_kind(self):
1279
+ """Ensure that `color` consistent with IMAGE `kind`"""
1280
+ if not self.kind and self.color:
1281
+ self.kind = Kind.IMAGE
1282
+ elif self.kind != Kind.IMAGE and self.color:
1283
+ raise ValueError(f"{self.color} implies {Kind.IMAGE} found {self.kind}")
1284
+
1285
+ def _verify_color(self, a: np.ndarray) -> str | None:
1286
+ """Verify that array matches the color info.
1287
+
1288
+ Return str message describing why not or None if OK
1289
+ """
1290
+ if self.color:
1291
+ # 1 chn: MxN or MxNx1 | 3 chn: MxNx3
1292
+ if not (self.color.channels == 1 and a.ndim in (2, 3) or a.ndim == 3):
1293
+ return f"Array's dims {a.ndim} incompatible with color {self.color}"
1294
+
1295
+ if not (self.cax is None or # no color channel
1296
+ a.ndim == 2 and self.color.channels == 1 or # 2D image with no separate color channel
1297
+ a.shape[
1298
+ self.cax] == self.color.channels): # color channel location and size
1299
+ return f"Wrong color channels # {a.shape}[{self.cax}] != {self.color.channels}"
1300
+ return None
1301
+
1302
+ def verify_array(self, a: np.ndarray, exc=TypeError):
1303
+ """Verify that array matches the data form
1304
+
1305
+ :param a: array to verify
1306
+ :param exc: Exception class to raise on error,
1307
+ if None - return error message instead
1308
+ :raises: If shape does not match color information
1309
+ :return: None if no error, otherwise Error message (if not exc)
1310
+ """
1311
+ assert isinstance(a, np.ndarray)
1312
+
1313
+ def failure(message):
1314
+ if exc: raise exc(message)
1315
+ return message
1316
+
1317
+ if self.dtype is not a.dtype: failure(f"Array and form dtypes mismatch {a.dtype}!={self.dtype}")
1318
+ if msg := self._verify_color(a):
1319
+ return failure(msg)
1320
+ return None
1321
+
1322
+ @classmethod
1323
+ def guess_from_data(cls, a: np.ndarray, fail=False) -> DForm | None:
1324
+ """
1325
+ Using arrays shape and dtype try to guess its ``DForm``.
1326
+ If fails return ``None`` or raise ``TypeError`` if ``fail is True``
1327
+ :param a: np.ndarray
1328
+ :param fail: default = False return None
1329
+ :return: guessed dform or None
1330
+ """
1331
+ if a.dtype == np.uint8: # currently only this case is supported
1332
+ return cls(a.dtype, min=0, max=255,
1333
+ color=Color.RGB if a.ndim == 3 and (a.shape[-1] == 3)
1334
+ else Color.GRAY if a.ndim == 1 else None)
1335
+ if fail: raise TypeError(f"Can't guess {cls.__name__} from array's properties")
1336
+
1337
+ def _range_trans_params(self, src_form) -> LinT:
1338
+ """
1339
+ Calculate linear parameters (k, b) in y = kx+b for normalization amd range transforms
1340
+ If do_range and do_norm not provided, then searching first for range changing, and
1341
+ if there is not, searching for norm changing.
1342
+ :param src_form: source data format
1343
+ :return: k, b - tuple, (1, 0) if transform is not required
1344
+ """
1345
+ if not src_form.range: # RANGE -> RANGE
1346
+ raise TypeError(f"Can't convert into range from not range-limited source {src_form}")
1347
+
1348
+ k = (self.max - self.min) / (src_form.max - src_form.min)
1349
+ b = self.min - k * src_form.min
1350
+ return LinT(k, b)
1351
+
1352
+ def _norm_trans_params(self, src_form: DForm) -> LinT:
1353
+ """
1354
+ Calculate linear parameters (k, b) in y = kx+b for normalization amd range transforms
1355
+ If do_range and do_norm not provided, then search first for range changing, and
1356
+ if there is not, search for norm changing.
1357
+ :param src: the source array to be transformed
1358
+ :param src_form: source data format
1359
+ :return: k, b - tuple, (1, 0) if transform is not required
1360
+ """
1361
+ src_avr = src_form.avr
1362
+
1363
+ # TODO: how to continue if src_avr is None?
1364
+ if self.std is not None: # output std requires src std or given or calculated
1365
+ src_std = src_form.std
1366
+ if self.avr is None: # output avr not defined - only scaling
1367
+ return LinT(self.std / src_std, 0)
1368
+ return LinT(self.std / src_std, self.avr - src_form.avr * self.std / src_std)
1369
+ self.avr: float
1370
+ return LinT(1, self.avr - src_avr) # no std conversion means k=1
1371
+
1372
+ def _apply_transform(self, src, src_form: DForm, out) -> np.ndarray:
1373
+ if self.range and src_form.range and self.norm_dim:
1374
+ # TODO: need to think of all the possibilities of norm and range combinations
1375
+ if self.min != src_form.min or self.max != src_form.max:
1376
+ raise ValueError("Simultaneous Range mapping AND Normalization is not supported")
1377
+
1378
+ lt = (self.range and self._range_trans_params(src_form) or
1379
+ self.norm_dim and self._norm_trans_params(src_form) or
1380
+ LinT())
1381
+
1382
+ return lt.apply(src, out=out)
1383
+
1384
+ def transform(self, src: np.ndarray | Array,
1385
+ src_form: DForm | Literal['guess', 'cast', 'ignore'] = None, *,
1386
+ round=True, out: np.ndarray = None, copy: bool = None) -> np.ndarray:
1387
+ """
1388
+ Convert ``src`` array from its DForm ``src_form`` into``self`` form.
1389
+
1390
+ Argument ``src_form`` must be one of those:
1391
+ - a ``DForm`` (or cast-able into) object (overrides with warning possible ``src.dform``)
1392
+ - 'guess' str to attempt guessing DForm from ``src`` (or inherit its ``dform`` if available)
1393
+ - 'ignore' try to ``self.dform`` to src ignoring its possible ``dform`` with minimal changes.
1394
+ - ``None``, ``src`` must have ``.dform`` in this case
1395
+
1396
+ If ``out`` is provided with ``.dform`` it must either match ``self.dform``
1397
+ or be compatible: same ``dtype`` and implied shape (color channels).
1398
+
1399
+
1400
+
1401
+ :param src: ndarray to transform
1402
+ :param src_form: DForm of the source array or 'guess' to request guessing
1403
+ :param out: provide array to fill the transform in
1404
+ :param copy: if ``True`` enforce copy even if data has not been changed
1405
+ :return: ndarray converted into self DForm
1406
+ """
1407
+ src, src_form = self._prepare_src_form(src, src_form) # None means src_form ignore is requested!
1408
+ if src_form in ('ignore', self): # semantically equal or ignore src_form
1409
+ return self._bypass_transform(src, out, copy)
1410
+
1411
+ shaper = DForm.Shaper(src.shape, src_form, self)
1412
+ if out is None: # allocate output array if not provided
1413
+ out = np.empty(shape=shaper.trg_shape, dtype=self.dtype) # created with cax always last!
1414
+ else: # or verify it if it is provided
1415
+ if out.dtype != self.dtype: raise TypeError(f"Mismatch {out.dtype=} != {self.dtype}")
1416
+ if out.shape != shaper.trg_shape: raise TypeError(f"Mismatch {out.shape=} != {shaper.trg_shape}")
1417
+ out = out.view(np.ndarray) # strip possible dform
1418
+
1419
+ src = shaper.move_cax(src, 'src', True)
1420
+ out = shaper.move_cax(out, 'trg', True)
1421
+
1422
+ src, src_form = self._color_pre_transform(src, src_form, out)
1423
+ self._apply_transform(src, src_form, out)
1424
+
1425
+ return shaper.move_cax(out, 'trg', False)
1426
+
1427
+ @staticmethod
1428
+ def _bypass_transform(src, out, copy):
1429
+ if out is None:
1430
+ out = src.copy() if copy else src
1431
+ else:
1432
+ if out.shape != src.shape or out.dtype != src.dtype:
1433
+ raise TypeError(f"Incompatible out: {out.shape}{out.dtype} with {src.shape}{src.dtype}")
1434
+ if npt.shared_view(out, src) and copy is True:
1435
+ raise ValueError("Can't copy: out shares src data")
1436
+ if copy:
1437
+ np.copyto(out, src, casting='no')
1438
+ return out
1439
+
1440
+ def _color_pre_transform(self, src, src_form: DForm, out):
1441
+ """
1442
+ Color transformation can be achieved by pre-transforming ``src``, ``src_form``
1443
+ before passed to the ``_apply_transform`` method.
1444
+
1445
+ :return: prepared ``src``, ``src_form``
1446
+ """
1447
+ if self.color and self.color != src_form.color: # only if target color is defined
1448
+ clr = Color
1449
+ colors = (src_form.color, self.color)
1450
+
1451
+ if not src_form.color: raise TypeError("Color transform from undefined color")
1452
+ # if src_form.std is not None and self.norm: # TODO: check this condition
1453
+ # raise TypeError("Can't transform color from std-ized image")
1454
+ if clr.BIN in colors: raise TypeError("Can't convert between any color and binary")
1455
+
1456
+ if colors == (clr.RGB, clr.GRAY): # MxNx3 -> MxN
1457
+ src = np.dot(src, _wC2G).squeeze()
1458
+ if src_form.norm_dim: # in this case possible per channel norm should be scaled too
1459
+ weighted = lambda _: None if _ is None else np.dot(np.resize(_, 3), _wC2G).squeeze()
1460
+ src_form = DForm(dtype=src.dtype, color=Color.GRAY,
1461
+ min=src_form.min, max=src_form.max,
1462
+ std=weighted(src_form.std),
1463
+ avr=weighted(src_form.avr))
1464
+ elif colors == (clr.GRAY, clr.RGB):
1465
+ src = np.dstack([src] * colors[1].channels)
1466
+ else:
1467
+ raise NotImplementedError(f'Not supported transform {colors=}')
1468
+ return src, src_form
1469
+
1470
+ def _prepare_src_form(self, src: Array | np.ndarray,
1471
+ src_form: Literal['ignore', 'guess'] | DForm | type
1472
+ ) -> tuple[np.ndarray, DForm | Literal['ignore']]:
1473
+ """
1474
+ From various forms of arguments ``src`` and ``src_form`` produce tuple:
1475
+ - ``src`` as ``ndarray`` (stripped from dform) and
1476
+ - ``src_form`` as ``DForm`` (or 'ignore')
1477
+
1478
+ If both define different DForm ``src_form != src.dform`` use ``src_form`` and issue warning.
1479
+
1480
+ Raise error if:
1481
+ 1. ``src_form="guess"`` but can't guess ``DForm`` from the data
1482
+ 2. ``src_form=None`` and ``src`` does not have DForm information
1483
+ 3. ``src_form`` can not be cast into ``DForm``
1484
+ """
1485
+
1486
+ if not_array := not isinstance(src, np.ndarray):
1487
+ if src_form in (None, 'ignore', 'guess'):
1488
+ raise TypeError(f"Neither src nor {src_form=} contain dform information!")
1489
+ dform_in_src = None
1490
+ elif dform_in_src := getattr(src, 'dform', None): # recover dform from src
1491
+ src = src.view(np.ndarray) # before stripping
1492
+
1493
+ if src_form is None:
1494
+ return src, dform_in_src or DForm(dtype=src.dtype)
1495
+ if src_form == 'ignore': return src, src_form # dissociate any dform from src
1496
+ if src_form == 'guess': return src, dform_in_src or self.guess_from_data(src, fail=True)
1497
+
1498
+ if not isinstance(src_form, DForm): src_form = DForm(src_form)
1499
+ if not_array: src = np.array(src, dtype=src_form.dtype)
1500
+
1501
+ if not dform_in_src: # src_form is provided separately - check validity
1502
+ if msg := src_form.verify_array(src): raise TypeError(msg)
1503
+ elif src_form != dform_in_src: # IDEA: allowed case and correct behaviour?
1504
+ warnings.warn(f"Overriding {dform_in_src} by {src_form}")
1505
+
1506
+ return src, src_form
1507
+
1508
+
1509
+ _format_options = dict(
1510
+ rows=16,
1511
+ cols=12,
1512
+ precision=3,
1513
+ rng=True,
1514
+ norm=False,
1515
+ nans=True,
1516
+ stats=(9, 1e7),
1517
+ info=True
1518
+ )
1519
+
1520
+
1521
+ class FormArray(type):
1522
+ """
1523
+ Metaclass for creating Array-like classes.
1524
+ """
1525
+
1526
+ EnableCond = Union[None, bool, Number, tuple[Number, Number]]
1527
+ _UND = type('_U', (), {'__str__': lambda _: 'UND'})()
1528
+
1529
+ _reg: dict[FormArray, FormArray] = {}
1530
+ _np_options = np.get_printoptions()
1531
+ _def_options = _format_options.copy()
1532
+
1533
+ def __new__(mcs, name: str | DForm | type, bases=(), attrs=None, *, dform: str | DForm = None,
1534
+ _base=None) -> Array:
1535
+ if _base: # used to create base classes without dform, not for user!
1536
+ if dform: raise RuntimeError("Base Array class can't define dform")
1537
+ cls = super().__new__(mcs, name, bases, attrs or {})
1538
+ cls.dform = None
1539
+ return cls
1540
+
1541
+ if dform is None: # dfrom may be passed through name argument
1542
+ if ( # FormArray('NCu1') - name is a registered dform or follows coding convention
1543
+ isinstance(name, str) and ( # FixMe: same as DForm.from_name(name)?
1544
+ dform := DForm.from_name(name, auto=True)) and DForm.auto_naming()
1545
+ or # FormArray(dform) - explicitly provided DForm
1546
+ isinstance(name, DForm) and (dform := name) # assign to dform
1547
+ ):
1548
+ name = f'Array{dform.name or "_"}'
1549
+ elif not bases and not attrs: # FormArray('uint8') - from dtype
1550
+ dform = DForm(dtype=name)
1551
+ name = f'Array_{_dtype_str(dform.dtype)}'
1552
+
1553
+ if not isinstance(dform, DForm): # 1. FormArray() - default dtype DForm
1554
+ dform = DForm(dform) # 2. FormArray('Name', dform=int)
1555
+
1556
+ if not any(issubclass(b, np.ndarray) for b in bases):
1557
+ bases = (Array, *bases)
1558
+ cls = super().__new__(mcs, name, bases, attrs or {})
1559
+ cls.dform = dform
1560
+ cls._format_options = FormArray._def_options.copy()
1561
+ return FormArray._reg.setdefault(cls, cls)
1562
+
1563
+ @classmethod # fixme: make it self method
1564
+ def from_array(cls, data: np.ndarray | Iterable | Array, *, dform: FormKind = None,
1565
+ copy=True, **dform_attrs) -> Array:
1566
+ """From given array create instance of Array-based class,
1567
+ with dform derived from the data or optional ``dform`` argument.
1568
+
1569
+ General compatibility requirements between the data and the dform applies.
1570
+
1571
+ In the minimal case with no additional arguments when only ``dtype`` can be deduced from
1572
+ the data, the resulting Array will be based on ``DForm(data.dtype)``
1573
+
1574
+ - If given ``dform`` argument overrides data type in all the cases!
1575
+ - In addition or alternatively DForm attributes may be provided as ``kws``.
1576
+
1577
+ If data is an iterable, then first an ``ndarray`` is created and then
1578
+ ``dtype`` is determined.
1579
+
1580
+ Even if data comes with its own ``dform``, it is replaced **without transformation**!
1581
+
1582
+ Also, available as a direct function in this module:
1583
+ ::
1584
+ form_array(data, dform, **dform_attrs)
1585
+
1586
+ :param data: ndarray or Array or iterable
1587
+ :param dform: DForm in any of supported form
1588
+ :param copy: if False try using view when possible
1589
+ :param dform_attrs: DForm attributes as key-value pairs
1590
+ """
1591
+ if not isinstance(data, np.ndarray):
1592
+ data = np.array(data)
1593
+
1594
+ data_form = getattr(data, 'dform', None)
1595
+ data_type = getattr(data, 'dtype', None)
1596
+ dform = DForm.from_merge(dform, **dform_attrs)
1597
+
1598
+ if dform:
1599
+ if not data_type:
1600
+ return FormArray(dform)(np.array(data, dtype=dform.dtype, copy=copy))
1601
+
1602
+ if data_form: # noinspection PyUnresolvedReferences
1603
+ return FormArray(dform)(data, copy=copy)
1604
+ else:
1605
+ return FormArray(dform)(data, copy=copy)
1606
+ else:
1607
+ if data_form:
1608
+ return np.asanyarray(data)
1609
+ else:
1610
+ return FormArray(data.dtype)(data)
1611
+
1612
+ @classmethod
1613
+ def global_print_options(mcs, *, rows=_UND, cols=_UND, precision=_UND, info=_UND,
1614
+ stats: EnableCond = _UND, rng: EnableCond = _UND,
1615
+ norm: EnableCond = _UND, nans: EnableCond = _UND, **options):
1616
+ """
1617
+ Set or get print options for ALL the FormArray classes.
1618
+
1619
+ Arguments ``stats, rng, norm, nans`` define conditions when to calculate
1620
+ specific statistics, and could be:
1621
+ - set explicitly (True|False)
1622
+ - defined (min, max) limits of array's size
1623
+ - only by the maximum limit
1624
+ - set to None, in which case it follows `stats` condition
1625
+
1626
+ ``stats`` is a general condition, if it evaluates to False, no calculations will be performed.
1627
+ Otherwise, other arguments will define their specific condition, as described above.
1628
+
1629
+ :param cols: maximal columns to show content
1630
+ :param rows: maximal rows to show content
1631
+ :param info: bool - show info summary, (None - automatic)
1632
+ :param precision (one of ``np.set_printoptions``)
1633
+ :param stats: condition to calculate all the stats
1634
+ :param rng: condition to calculate data range (min, max)
1635
+ :param norm: condition to calculate avr and std
1636
+ :param nans: condition to count found ``nan``s (only for floating point arrays)
1637
+ :param options: other options for ``np.set_printoptions``
1638
+ :return:
1639
+ """
1640
+ loc = locals().copy()
1641
+ for k in ['mcs', 'options']: del loc[k]
1642
+
1643
+ if inv := set(options).difference(mcs._np_options):
1644
+ raise KeyError(f"Unknown numpy print options: {inv}")
1645
+
1646
+ defined = {k: v for k, v in loc.items() if v is not mcs._UND}
1647
+ mcs._def_options.update(**defined)
1648
+ mcs._np_options.update(**options)
1649
+
1650
+ if defined or options:
1651
+ for cls in mcs._reg:
1652
+ cls.print_options(**defined, **options)
1653
+ else:
1654
+ return mcs._np_options | mcs._def_options
1655
+
1656
+ def print_options(cls, *, rows=_UND, cols=_UND, precision=_UND, info=_UND,
1657
+ stats: EnableCond = _UND, rng: EnableCond = _UND,
1658
+ norm: EnableCond = _UND, nans: EnableCond = _UND, **options):
1659
+ """
1660
+ Set or get print options for specific class of Arrays.
1661
+
1662
+ When called without arguments returns a copy of the current options.
1663
+
1664
+ Arguments ``stats, rng, norm, nans`` define conditions when to calculate
1665
+ specific statistics, and could be:
1666
+ - set explicitly (True|False)
1667
+ - defined (min, max) limits of array's size
1668
+ - only by the maximum limit
1669
+ - set to None, in which case it follows `stats` condition
1670
+
1671
+ ``stats`` is a general condition, if it evaluates to False, no calculations will be performed.
1672
+ Otherwise, other arguments will define their specific condition, as described above.
1673
+
1674
+ :param cols: maximal columns to show content
1675
+ :param rows: maximal rows to show content
1676
+ :param info: bool - show info summary, (None - automatic)
1677
+ :param precision (one of ``np.set_printoptions``)
1678
+ :param stats: condition to calculate all the stats
1679
+ :param rng: condition to calculate data range (min, max)
1680
+ :param norm: condition to calculate avr and std
1681
+ :param nans: condition to count found ``nan``s (only for floating point arrays)
1682
+ :param options: other options for ``np.set_printoptions``
1683
+ :return:
1684
+ """
1685
+ loc = locals().copy()
1686
+ for k in ['cls', 'options']: del loc[k]
1687
+
1688
+ if inv := set(options).difference(cls._np_options):
1689
+ raise KeyError(f"Unknown numpy print options: {inv}")
1690
+
1691
+ defined = {k: v for k, v in loc.items() if v is not cls._UND}
1692
+
1693
+ if options or defined:
1694
+ cls._format_options.update(**options, **defined)
1695
+ else:
1696
+ return cls._format_options.copy()
1697
+
1698
+ @overload
1699
+ def __init__(cls, class_name: str, *, dform: str | DForm):
1700
+ ...
1701
+
1702
+ @overload
1703
+ def __init__(cls, dform: str | DForm | type):
1704
+ pass
1705
+
1706
+ def __init__(cls, name: str | DForm | type, bases=(), attrs: dict = None, *,
1707
+ dform: str | DForm = None, _base=None):
1708
+ """
1709
+ Create a new ``Array`` like class with associated ``DForm``.
1710
+ Supported initializations:
1711
+
1712
+ >>> FormArray('MyArray', (Array,), {}, dform=DForm('Cu1') ) # full form
1713
+ >>> FormArray('MyArray', dform='Cu1') # Assuming Cu8 is a registered DForm
1714
+ >>> FormArray('Cu1') == FormArray('ArrayCu8', 'Cu1') == FormArray(DForm('Cu1'))
1715
+
1716
+ :param name: class name or dform (registered name or an instance)
1717
+ :param bases: base classes include at least one subclass of ``np.ndarray``.
1718
+ Otherwise, ``Array`` is automatically.
1719
+ :param attrs: optional additional attributes
1720
+ :param dform: ``DForm``- name or instance. ``None`` equivalent to ``DForm()``
1721
+ """
1722
+
1723
+ def __init_subclass__(cls, *, dform=None):
1724
+ if dform is None:
1725
+ dform = DForm()
1726
+ elif not isinstance(dform, DForm):
1727
+ raise TypeError(f"dform must be None or DForm")
1728
+ cls.dform = dform
1729
+
1730
+ def __eq__(self, other: FormArray):
1731
+ if isinstance(other, FormArray):
1732
+ return self.dform == other.dform
1733
+ return False
1734
+
1735
+ def __hash__(self):
1736
+ return hash(self.dform) + hash(self.__name__)
1737
+
1738
+ def __repr__(self):
1739
+ return self.__qualname__
1740
+
1741
+
1742
+ # noinspection PyShadowingBuiltins
1743
+ class Array(np.ndarray, metaclass=FormArray, _base=True):
1744
+ dform: DForm
1745
+
1746
+ @property
1747
+ def ndarray(self):
1748
+ return np.asarray(self)
1749
+
1750
+ def __repr__(self):
1751
+ return npt.array_info_func(**self._format_options)(self)
1752
+
1753
+ def __str__(self):
1754
+ return self.__repr__()
1755
+
1756
+ def __new__(cls, obj, dform=None, *, trans=False, round=False, copy=False):
1757
+ """
1758
+ Creating a new instance of an Array-like class.
1759
+ Notice! This class can create instances of another classes, not only Array.
1760
+ Use cases:
1761
+
1762
+ >>> Array([]) # same as Array([], 'f4').from_array([])
1763
+ >>> Array([], dform='Nf2') # creates instance of ArrayNf2
1764
+
1765
+ >>> ArrayCu1([]) # ok
1766
+ >>> ArrayCu1([], dform='Nf2') # error
1767
+
1768
+ :param obj: data as iterable or ndarray or Array-like with dfrom
1769
+ :param trans: allow data transfromation when casting into new form
1770
+ :param copy: if False try to avoid memory copy (use view)
1771
+ :param dform: only allowed in base class (Array)
1772
+ """
1773
+ if cls.dform: # ArrayF(a)
1774
+ if dform and dform != cls.dform: # ArrayF1(af, f1) !ArrayF1.dform == f1
1775
+ raise TypeError(f"argument {dform=} clashes with {cls.dform=}")
1776
+ else: # Array(obj, dform='Nu1')
1777
+ return FormArray(dform or FLOAT)(obj, trans=trans, round=round, copy=copy)
1778
+
1779
+ if type(obj) == cls: # Array(obj:Frm, Frm) !From -> Frm
1780
+ return obj.copy() if copy else obj
1781
+
1782
+ if not isinstance(obj, np.ndarray): # ArrayFrm([1,2])
1783
+ obj = np.array(obj) # - create ndarray
1784
+ copy = False
1785
+ if obj.dtype.kind not in 'bifu':
1786
+ raise TypeError('Array data must be of numeric type')
1787
+
1788
+ if trans: # ArrayF(a:ndarray)
1789
+ obj = cls.dform.transform(obj, copy=copy, round=round)
1790
+ else:
1791
+ obj = cast(obj, dtype=cls.dform.dtype, round=round, copy=copy)
1792
+ return obj.view(cls)
1793
+
1794
+ # noinspection PyMethodOverriding
1795
+ def __array_finalize__(self, obj):
1796
+ if obj is None: return
1797
+ if self.dform.dtype and self.dform.dtype != self.dtype:
1798
+ raise TypeError(f"dtype {self.dtype} incompatible with {self.dform}'s {self.dform.dtype}")
1799
+
1800
+ def __array_ufunc__(self, ufunc, method, *inputs, out=None, **kwargs):
1801
+ # Based on: https://numpy.org/doc/stable/user/basics.subclassing.html
1802
+ # Required to make sure ndarray are provided where expected in Universal functions (Ufunc).
1803
+ # Also avoids automatic conversion of scalar results into Array
1804
+
1805
+ # Unless out arrays are provided, all the resulting arrays are returned as Array (empty DForm)
1806
+ # leaving to the user reassignment to the proper DForm if needed.
1807
+ # That because an automatic propagation of the DForm would be too tricky in most the cases.
1808
+ # TODO: should we implement those few clear cases?
1809
+
1810
+ args = tuple(input_.view(np.ndarray) if isinstance(type(input_), FormArray) else
1811
+ input_ for input_ in inputs) # view inputs as regular ndarrays
1812
+
1813
+ if out: # also view as ndarrays optional outputs
1814
+ outputs = out
1815
+ kwargs['out'] = tuple(output.view(np.ndarray) if isinstance(type(output), FormArray) else
1816
+ output for output in outputs)
1817
+ else:
1818
+ outputs = (None,) * ufunc.nout
1819
+
1820
+ results = super().__array_ufunc__(ufunc, method, *args, **kwargs)
1821
+ if results is NotImplemented: return NotImplemented
1822
+ if method == 'at': return
1823
+
1824
+ def select(r, o):
1825
+ if o is not None: return o
1826
+ if not isinstance(r, np.ndarray): return r
1827
+ return np.asarray(r).view(FormArray(r.dtype))
1828
+
1829
+ if ufunc.nout == 1: return select(results, outputs[0])
1830
+ return tuple(select(result, output) for result, output in zip(results, outputs))
1831
+
1832
+ # def __init__(self, obj, dform=None, *, trans=False, copy=False):
1833
+ # """
1834
+ # Something
1835
+ #
1836
+ # :param obj:
1837
+ # :param dform:
1838
+ # :param trans:
1839
+ # :param copy:
1840
+ # """
1841
+ # ...
1842
+
1843
+ def asform(self, dform: str | DForm = None, **kws):
1844
+ """
1845
+ Return present array as a different `DForm`, based on the:
1846
+ 1. empty `DForm()` if ``dform = None (and no kws)``
1847
+ 2. provided as DForm instance
1848
+ 3. provided as a name to registered DForm instance
1849
+
1850
+ Additional keyword arguments may be provided to override specific ``DForm`` attributes:
1851
+ ``name, color, cax, std, avr, min, max``
1852
+
1853
+ Returned array is of a different type (FormArray sub-class).
1854
+
1855
+ Use this method when specificity of the form of an existing data must be updated.
1856
+ Initial transform is ignored
1857
+
1858
+ Notice:
1859
+ - only attributes are replaced
1860
+ - ``dtype`` can not be changed
1861
+ - resulting array is another view on same data
1862
+
1863
+ :return: An instance of a new Array subclass with different ``DForm`` but same data.
1864
+ """
1865
+ if isinstance(dform, str):
1866
+ dform = DForm(dform)
1867
+ elif dform is None:
1868
+ if not kws: # neither dform nor kws defined - strip dform
1869
+ return FormArray(self.dtype)(self.ndarray)
1870
+ dform = self.dform # inherit from self.dform with updates from kws
1871
+ elif not isinstance(dform, DForm):
1872
+ raise ValueError(f"Invalid argument {dform=}")
1873
+
1874
+ if kws:
1875
+ attrs = dcs.fields(DForm)
1876
+ if unknown := set(kws).difference(attrs):
1877
+ raise KeyError(f"DForm has no attribute: {unknown}")
1878
+
1879
+ # inherit undefined attributes from the dform
1880
+ if 'dtype' in kws: raise TypeError("dtype can not be overriden")
1881
+ kws = {f.name: kws.get(f.name, getattr(dform, f.name))
1882
+ for f in attrs if f.init}
1883
+ return FormArray(DForm(**kws))(self.ndarray)
1884
+ elif dform == self.dform:
1885
+ return np.asanyarray(self)
1886
+ else:
1887
+ return dform.transform(self, src_form='ignore')
1888
+
1889
+ def transform(self, dform: DForm, *, guess=True, copy=True, out=None):
1890
+ """Creates a NEW object of required form.
1891
+ If ``dform`` is same as ``self.dform`` data is not transformed.
1892
+
1893
+ If ``self.dform`` is not defined, attempt to guess it before failing.
1894
+ Undefined ``dform`` always fails.
1895
+
1896
+ :param dform: target DForm of resulting array
1897
+ :param guess: if ``self.dform`` is not defined, guess from the data
1898
+ :return: new array instance with requested ``dform``.
1899
+ """
1900
+ # todo implement copy and out (with and without dform). If out neglect copy
1901
+ if isinstance(dform, str): dform = DForm(dform)
1902
+ if dform == self.dform: return type(self)(self)
1903
+ src_form = 'guess' if (self.dform is None and guess) else self.dform
1904
+ return FormArray(dform)(dform.transform(self, src_form=src_form))
1905
+
1906
+
1907
+ # Consider auto-generation based on the naming convention:
1908
+ #
1909
+ # [C|G[<bits>b]][R[<min>[:]]<max>|N[<std>[:<avr>]]
1910
+ # C - color (RGB)
1911
+ # G - Gray scale
1912
+ # R - Range
1913
+ # N - Normalized (by default std=1, avr=0)
1914
+ #
1915
+ # In R<min>:<max> ':' optional if min and max single digit
1916
+ # That makes Rxy ambiguous: R0:xy or Rx:y. Second is assumed
1917
+
1918
+
1919
+ # Some standard data forms
1920
+ # DForm('R01f4', FLOAT, min=0, max=1) # any dim
1921
+ # DForm('R-11f4', dtype=FLOAT, min=-1, max=1) # any dim
1922
+ #
1923
+ # DForm('CR01f4', dtype=FLOAT, min=0, max=1, color=DForm.Color.RGB) # HxWx3
1924
+ # DForm('CNf4', FLOAT, color=DForm.Color.RGB, avr=0, std=1)
1925
+ # DForm('Nf4', dtype=FLOAT, avr=0, std=1)
1926
+ #
1927
+ # DForm('Cu1', dtype='u1', color=DForm.Color.RGB)
1928
+ # DForm('Gb10', dtype='u2', min=0, max=1023, color=DForm.Color.GRAY)
1929
+
1930
+ @overload
1931
+ def form_array(data: Iterable | np.ndarray, *, dform: FormKind = None, copy=False) -> Array: ...
1932
+
1933
+
1934
+ @overload
1935
+ def form_array(data: Iterable | np.ndarray, *, copy=False, **dform_attrs) -> Array: ...
1936
+
1937
+
1938
+ def form_array(data: Iterable | np.ndarray | Array, *,
1939
+ dform: FormArray | FormKind = None, copy=False,
1940
+ **dform_attrs) -> Array:
1941
+ """
1942
+ From given array create instance of Array-based class,
1943
+ with dform derived from the data or optional ``dform`` argument.
1944
+
1945
+ In basic case resulting Array will be based on ``DForm(data.dtype)``
1946
+
1947
+ - If given ``dform`` argument overrides data type in all the cases!
1948
+ - In addition or alternatively DForm attributes may be provided as ``kws``.
1949
+
1950
+ If data is an iterable, then first an ``ndarray`` is created and then
1951
+ ``dtype`` is determined.
1952
+
1953
+ Equivalent to:
1954
+ ::
1955
+ FormArray.from_array(data, dform=dform, **dform_attrs)
1956
+
1957
+ :param data: ndarray or Array or iterable
1958
+ :param dform: Array-like class or DForm in any of supported form
1959
+ :param dform_attrs: DForm attributes as key-value pairs
1960
+ """
1961
+ return FormArray.from_array(data, dform=dform, copy=copy, **dform_attrs)