ialdev-core 0.2.5__tar.gz → 0.2.6__tar.gz

{ialdev_core-0.2.5 → ialdev_core-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ialdev-core
-Version: 0.2.5
+Version: 0.2.6
 Summary: iad.core — algorithmic utilities: data tools, parameters, paths, caching, and more
 Author: ipcoder
 Requires-Python: >=3.10

{ialdev_core-0.2.5 → ialdev_core-0.2.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "ialdev-core"
-version = "0.2.5"
+version = "0.2.6"
 description = "iad.core — algorithmic utilities: data tools, parameters, paths, caching, and more"
 readme = "README.md"
 requires-python = ">=3.10"

ialdev_core-0.2.6/src/iad/core/data/labeled.py

@@ -0,0 +1,249 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from .label import Labels
+
+if TYPE_CHECKING:
+    from .pdtools import DTable
+
+__all__ = ['Labeled', 'LabeledType']
+
+
+class Labeled(Labels):
+    """
+    Base ABSTRACT class for Labeled data types - an extended dict containing data
+    in one field and descriptive labels in the others.
+
+    Actual Labeled Data types are created for specific collection of labels
+    (by ``LabeledType`` metaclass) using ``labeled_type()`` or ``LT`` alias
+    """
+
+    @classmethod
+    def type_keys(cls) -> tuple:
+        return *cls._defaults, *cls._undefined
+
+    UNDEF = object()
+
+    def __eq__(self, other):
+        if not (isinstance(other, dict) and len(self) == len(other)):
+            return False
+        no_key = object()
+        try:
+            for k, vo in other.items():
+                if (v := self.get(k, no_key)) is no_key:
+                    return False
+                na = isinstance(v, np.ndarray) + isinstance(vo, np.ndarray)
+                if na == 1 or na == 0 and vo != v or na == 2 and not np.array_equal(vo, v):
+                    return False
+        except Exception as ex:  # various not comparable types
+            print(ex)
+            return False
+        return True
+
+    def __init__(self, data=UNDEF, **kws):
+        """
+        Support dict-like initializations:
+            A(data=2, b=3) and A({'data':2, 'y':3})
+        and silent data field:
+            A(2, b=3)  == A(data=2, b=3)
+        :param data: not defined or data or dict with labels (like kws)
+        :param kws: labels key, values
+        """
+        cls = type(self)
+        if cls is Labeled:
+            raise RuntimeError(f"Class {cls} is a mixin class not intended for instantiation.")
+
+        # ---- STEP 1 - convert inputs into homogeneous form
+        if data is not self.UNDEF:
+            # support Labeled(dict) like initialization
+            if isinstance(data, dict) and self._all_keys.issuperset(data):
+                kws.update(data)
+            else:
+                assert 'data' in self._all_keys, "unnamed argument allowed only if 'data' field defined!"
+                kws['data'] = data
+
+        # ---- STEP 2 - check all the conditions are met
+        if missing := set(self._undefined).difference(kws):
+            raise KeyError(f"labels {missing} not initialized in {cls}!")
+
+        if violated := '\n\t'.join(
+                f"{k}={assigned} ({expected=})"
+                for k in self._frozen_defaults.intersection(kws)
+                if (expected := self._defaults[k]) != (assigned := kws[k])
+        ): raise KeyError(f"Initialization of {cls} violates frozen defaults:\n {violated}")
+
+        if self._frozen_keys and not self._all_keys.issuperset(kws):
+            raise KeyError(f"Undefined keys {set(kws).difference(self._all_keys)} in frozen {cls}")
+
+        # ---- STEP 3 - initialize the items
+        super().__init__(self._defaults)  # first fill with defaults
+        # then assign from the arguments
+        for k, v in kws.items():
+            if self._validate_type and \
+                    (tp := self._undefined.get(k, False)) and \
+                    type(tp) is type and not isinstance(v, tp):
+                v = tp(v)
+            self[k] = v
+
+    def flat(self, *, sep='_', exclude='data', data='data', fmt=None) -> tuple[str, np.ndarray]:
+        """
+        Merge labels into a single string and produce a tuple (flatten_label, data)
+
+        :param sep: separator when joining
+        :param exclude: keys exclude from the join
+        :param data: key of the data field returned as second item in the tuple
+        :param fmt: optional str.format sting, if provided ignores arguments:
+                sep, exclude, keys
+        :return: tuple(joint_label, data_item)
+        """
+        from iad.core import as_iter
+        data = self[data]
+        if fmt:
+            return fmt.format(**self), data
+
+        excluded = set(as_iter(exclude))
+        included = lambda _: _[0] not in excluded
+        str_val = lambda _: str(_[1])
+        return sep.join(map(str_val, filter(included, self.items()))), data
+
+    def __setattr__(self, item, val):
+        if item in self:
+            self[item] = val
+        else:
+            super().__setattr__(val)
+
+    @classmethod
+    def from_table(cls, db: DTable, *, missing=KeyError, undef=()):
+        """
+        Create instance of this Labeled type from :cls:`iad.core.data.pdtools.DataTable`
+        All the pre-defined keys are used to query the index.
+        All the undefined keys must be in the table and are initialized
+
+        If table does not contain given label, then depending on the value of ``missing``:
+         - `issubclass(missing, BaseException)` - raise `missing(f"{type(ex)}({ex.args})")`
+         - `isinstance(missing, BaseException)` - raise `missing`
+         - otherwise, return `missing`
+
+        :param db:
+        :param missing: Exception class, instance, or value to return
+        :param undef: keys listed here are forced to be considered as undefined
+        :return:
+        """
+        if undef:
+            undef = set(undef)
+            if undef := undef.difference(cls._undefined):  # undef not already undefined
+                if not_defined := undef.difference(cls._defaults):  # must be defined!
+                    raise KeyError(f"Unknown keys: {not_defined}")
+            # move defined undef from defined into undefined
+            defined = {k: v for k, v in cls._defaults.items() if k not in undef}
+            undefined = [*undef, cls._undefined]
+        else:
+            defined = cls._defaults
+            undefined = [*cls._undefined]
+
+        try:
+            res = db.select(defined)
+        except (KeyError, LookupError) as ex:
+            if isinstance(missing, BaseException):
+                raise missing
+            elif isinstance(missing, type) and issubclass(missing, BaseException):
+                raise missing(f"{type(ex)}({ex.args})")
+            return missing
+        if hasattr(res, 'columns'):
+            res = res[undefined]
+        elif not (len(undefined) == 1 and res.name == undefined[0]):
+            raise KeyError(f"Table does not contain required keys {undefined}")
+        return cls(**res.reset_index().iloc[0].to_dict())
+
+
+class LabeledType(type):
+    """ Metaclass to create Labeled Data classes extending ``dict``,
+    but built around predefined keys with optional default values.
+
+    Main purpose of such classes to ensure that during instantiation all
+    those keys are initialized either by their defaults or as arguments.
+
+    Note: additional keys except those defined in the class may be added.
+
+    Classes are supposed to be created by directly calling this meta:
+
+    >>> LabeledXY = LabeledType(x=10, y=int)
+    >>> d = LabeledXY(y=2)   # LT(x=2) raise exception for undefined y!
+    >>> assert d['y'] == 10
+    >>> assert type(d.__name__) == 'Labeled'  # default class name
+    >>> LabeledData = LabeledType('LabeledData', x=10, y=int)
+    >>> assert LabeledData.__name__ == 'LabeledData'  # custom class name
+
+    To encorage usage of 'data' key for data its positional initialization
+    is supported if data key is introduced explicitely or using `LD` function
+    >>> LabeledData = LabeledType(data=NDArray, y=int) # equivalent = LD(y=int)
+    >>> d = LabeledData([1,2,3], y=20)
+    >>> assert d['data'] == [1,2,3]
+    """
+
+    def __repr__(cls):
+        undefined = (f"{k}: {v.__name__}" for k, v in cls._undefined.items())
+        defaults = (f"{k}={v}" for k, v in cls._defaults.items())
+        return f"{cls.__qualname__}<{', '.join((*undefined, *defaults))}>"
+
+    def __new__(mcs, _name=None, *, validate_type=False,
+                frozen_defaults=False, frozen_keys=False,
+                **labels):
+        """
+        :param _name: Name of the created class
+        :param validate_type: perform casting on keys initialization if possible
+        :param frozen_defaults: allow change default values (False)
+               use True to define templates
+        :param frozen_keys: allow not defined keys
+        :param labels: dict with {key: default_value} and {key: type} items
+        """
+        _name = _name or f"Labeled"
+        # 'undefined' are keys assigned by types, not
+        # default values which would be called 'defaults'.
+        cls = super().__new__(mcs, _name, (Labeled,), {})
+
+        cls._undefined = {k: labels.pop(k) for k in
+                          [_ for _, v in labels.items() if type(v) is type]}
+        cls._defaults = labels
+        cls._all_keys = {*cls._undefined, *cls._defaults}
+
+        frozen_defaults = cls._defaults if frozen_defaults is True else \
+            [] if frozen_defaults is False else frozen_defaults
+        cls._frozen_defaults = {*frozen_defaults}
+        cls._frozen_keys = frozen_keys
+        cls._validate_type = validate_type
+        return cls
+
+    def __init__(mcs, _name=None, *, validate_type=False,
+                 frozen_defaults=False, frozen_keys=False,
+                 **labels):
+        """
+        :param _name: Name of the created class
+        :param validate_type: perform casting on keys initialization if possible
+        :param frozen_defaults: allow change default values (False)
+               use True to define templates
+        :param frozen_keys: allow not defined keys
+        :param labels: dict with {key: default_value} and {key: type} items
+        """
+        pass  # to avoid calling type.__init__ with unusual arguments
+
+    def __contains__(cls, item):
+        return item in cls._defaults or item in cls._undefined
+
+    @property
+    def undefined(cls):
+        """keys with undefined default values"""
+        return cls._undefined.keys()
+
+    @property
+    def default(cls):
+        """keys with defined default values"""
+        return cls._defaults.keys()
+
+    @property
+    def defined(cls):
+        """set of ALL the keys with undefined AND default values"""
+        return cls._all_keys

{ialdev_core-0.2.5 → ialdev_core-0.2.6}/src/iad/core/data/pdtools.py

@@ -1050,6 +1050,56 @@ def _expand_vec_values(pairs: list[tuple[Scalar, Scalar | Vector]], prev_expande
         yield prev_expanded  # return as received - start aggregation
 
 
+def pivot_flat(df, base: str | list[str], *, pivot: str | list[str], 
+               index: bool | list[str] | str | None = None, sep=','):
+    """
+    In the given dataframe creates multi-index columns by adding `pivot` levels to the `base` columns` 
+    and flattens them into str using sep.
+
+    All the operations are done using on both df.columns and df.index.names.
+
+    Sets `index` columns as multi-index levels.
+     - if `index` is True, use all the remaining columns not mentioned in `pivot`.
+     - if `index` is False, index is reset completely
+     - if `index` is None, leaves original index levels (except of pivoted).    
+    """
+    remove = lambda x, c : [_ for _ in x if _ not in c]
+
+    org_index = df.index.names if isinstance(df.index, pd.MultiIndex) else []
+    org_columns = df.columns    
+    all_labels = [*org_index, *org_columns]
+
+    invalid = list(filter(lambda _: not isinstance(_, (str, type(None))), org_columns))
+    assert not invalid, f"Some of columns/ index levels are not strings: {invalid}"
+
+    def check(x, name):
+        x = as_list(x)    
+        assert x, f"At least one {x} needed" 
+        assert not (_:= set(x).difference(all_labels)), f"Unknown {name}: {_}"
+        return x
+
+    base, pivot = check(base, 'base'), check(pivot, 'pivot')
+
+    # leave only base as columns, and then pivot reuired levels     
+    df = df.set_index(remove(all_labels, base)).unstack(pivot)
+    df.columns = [sep.join(col) for col in df.columns] # flatten columns names into str
+
+    if index is True:
+        index = df.index.names
+    elif index is False:
+        index = []
+    elif index is None:
+        index = re|move(org_index, pivot)
+    else:
+        index = as_list(index)
+        assert not (_:= set(index).difference(df.index.names)), f"Invaid index levels: {_}"
+
+    pivot_columns = list(df.columns)
+    to_columns = remove(df.index.names, index)
+    df = df.reset_index(to_columns)
+    return df, pivot_columns
+
+
 def _row_queries_gen(labels, levels, levels_map) -> Generator[list[tuple[int, Scalar]]]:
     """
     Generator of mult-index row queries in form of:
@@ -1790,18 +1840,32 @@ class _TableMixIn:
 
     def qix(self: PTable, *anonymous,
             drop_level: bool | Collection = False, keep: Collection = None,
-            axis: AxisT = None, key_err=True, **named) -> PTable:
+            axis: AxisT = None, key_err=True, reorder=False, **named) -> PTable:
         """
         Filter a dataframe or series using fuzzy query of its multi-index.
-        Arguments in different forms describe the index values to be searched for.
-
-        Some index levels in the output can be dropped by specifying either
-        ``drop_level`` or ``keep`` (mutually exclusive) arguments.
 
-        Use ``drop_level=True`` to drop all the redundant levels used in the query, or
-        just provide one (or list) of specific levels to drop.
-        Would not drop if the level is important for indexing the data (there is no redundancy),
-        even if specified in the drop_level list
+        ``qix`` is a FILTER, not a selector: each requested value may match
+        zero, one, or many rows, and results follow the table's storage
+        order. For structured, ordered, one-row-per-label retrieval (e.g.
+        ``img, gt = ...``) use :meth:`select`.
+
+        Capabilities, from simple to advanced:
+
+        1. One level by name:        ``dt.qix(w='a')``
+        2. Several levels (AND):     ``dt.qix(w='a', x='1')``
+        3. Several values per level: ``dt.qix(w=['b', 'a'])``
+           (membership; rows stay in storage order)
+        4. Both axes auto-matched:   ``dt.qix(w='a', y='4')``
+           - restrict with ``axis='index'``/``'columns'`` (or 0/1)
+        5. Anonymous values:         ``dt.qix('a', '3')``
+           - search a value without naming its level (error if ambiguous)
+        6. Collapse redundant levels with ``drop_level`` / ``keep``:
+           ``dt.qix(w='a', drop_level=True)`` (drops only when one value remains)
+        7. Tolerate misses with ``key_err=False`` (returns empty table)
+        8. Opt-in ordering:          ``dt.qix(w=['b', 'a'], reorder=True)``
+           - with ``reorder=True`` the listed values define each level's
+             output order (first keyword = primary key; unqueried levels keep
+             storage order), and duplicate listed values raise ``ValueError``.
 
         :param self: The table to filter
         :param anonymous: list of values from one of the index levels
@@ -1811,6 +1875,10 @@ class _TableMixIn:
         :param keep: a collection of levels to keep - excludes using drop_level
         :param axis: if specified query only this axis
         :param key_err: raise KeyError if index not found or return empty
+        :param reorder: if True, reorder results to the order values are listed
+                     per level (kwargs order = precedence) and raise ValueError
+                     on duplicate listed values. Default False (pure filter,
+                     storage order).
         :param named:  {level: value} - specific levels to find,
                             eliminates exhaustive search in all levels
         :return: Filtered table
@@ -1862,6 +1930,17 @@ class _TableMixIn:
         x
         1      1      5
         2      1      5
+
+        By default qix is a filter and keeps storage order. Pass
+        ``reorder=True`` to order results by the values you list.
+        Example 4 - opt-in ordering (note y follows the listed [5, 3]):
+
+        >>> dt.qix(x='1', y=['5','3'], drop_level=True, reorder=True) # doctest: +NORMALIZE_WHITESPACE
+        z data1       data2
+        w     a b c d     a b c d
+        y
+        5     1 2 3 4     5 6 7 8
+        3     1 2 3 4     5 6 7 8
         """
         # ToDo: add option to search not only in the index
         # FixMe: self.empty check twice!
@@ -1908,6 +1987,17 @@ class _TableMixIn:
             logging.debug(f'qix:\n{err}')
             return self.iloc[0:0]
 
+        if reorder:
+            for lvl, val in named.items():
+                vals = as_list(val)
+                if len(vals) != len(set(vals)):
+                    raise ValueError(f"Duplicate values with reorder=True: {lvl=}, {vals=}")
+            for ax, axis_kws in enumerate(axs_kws):
+                # follow kwargs (named) order so the first listed level is primary
+                ordered = {lvl: named[lvl] for lvl in named if lvl in axis_kws}
+                if ordered and (keys := order_keys(res.axes[ax], ordered)):
+                    res = res.take(np.lexsort(keys[::-1]), axis=ax)
+
         if drop_level:
             to_set = lambda x: {x} if isinstance(x, (str, int)) else set(x)
             for ax, (kws, axis) in enumerate(zip(axs_kws, res.axes)):
@@ -2073,6 +2163,30 @@ class _TableMixIn:
         return hash_value.hexdigest()[-n:]
 
 
+def order_keys(axis_index: pd.Index | pd.MultiIndex, kws) -> list[np.ndarray]:
+    """ Positional sort keys to reorder rows by the order values are listed.
+
+    For each level whose requested value is a list of two or more values,
+    map every axis entry to the position of its value in that list. Levels
+    with a scalar / single value are skipped (no reordering needed).
+
+    Args:
+        axis_index: the (multi-)index of the already-filtered axis
+        kws: {level: value | [values]} requested for this axis
+
+    Returns: list of int key arrays, one per listed level (kwargs order)
+    """
+    keys = []
+    for lvl, val in kws.items():
+        vals = as_list(val)
+        if len(vals) < 2:
+            continue
+        pos = {v: i for i, v in enumerate(vals)}
+        level_vals = axis_index.get_level_values(lvl) if axis_index.nlevels > 1 else axis_index
+        keys.append(level_vals.map(pos).to_numpy())
+    return keys
+
+
 def slicer(index: pd.Index | pd.MultiIndex, kws):
     """ Slicing the index with kws requested
     Args:

{ialdev_core-0.2.5 → ialdev_core-0.2.6}/src/iad/core/fs/filesproc.py

@@ -215,6 +215,32 @@ def rename_files_by_ref(files: Union[Iterable, str], ref_files: Union[Iterable,
     return old_new_pairs
 
 
+def inc_file_sfx(path, sep='_'):
+    """Given full path (<dir>/<stem>.<ext>) find if there are any files with such name 
+    or with additional suffix (<dir>/<stem><sep><num>.<ext>'),
+    and create a new name with suffix incremented over the maximal found.
+
+     - if only `path` start with '_1'
+     - if `path` is not found:
+         - found with suffixes - increment max suffix
+         - otherwise return the path    
+    """
+    path = Path(path).expanduser()
+    base = path.stem + sep
+    pat = re.compile(re.escape(base) + r'(\d+)')
+    last = max((
+        int(m.group(1))
+        for p in path.parent.iterdir() 
+        if p.is_file() and (m := pat.fullmatch(p.stem))
+    ), default=None)
+    
+    if last is None:    
+        stem = base + '1' if path.exists() else path.stem
+    else:
+        stem = base + str(last+1)
+    return path.with_stem(stem)
+
+
 def normalize(path: PathT, out: Type[Path] | Type[str] | None = Path):
     """
     Translate path into canonical form expanding user but not resolving links.

{ialdev_core-0.2.5 → ialdev_core-0.2.6}/src/iad/core/wrap.py

@@ -150,7 +150,6 @@ class NamedTupleI:
     ::
             isinstance(type(obj), NamedTupleMeta)
     """
-    from .data.nptools import array_info_str
 
     _fields: list[str]
     __iter__: Callable
@@ -158,7 +157,8 @@ class NamedTupleI:
     def __init__(self, *args, **kws): ...
 
     def __repr__(self):
-        val_str = lambda v: (NamedTupleI.array_info_str(v, stats=False)
+        from .data.nptools import array_info_str
+        val_str = lambda v: (array_info_str(v, stats=False)
                              if hasattr(v, 'shape') and hasattr(v, 'ndim') else str(v))
         fields_str = (f'{k}: {val_str(v)}' for k, v in zip(self._fields, self))
 

{ialdev_core-0.2.5 → ialdev_core-0.2.6}/tests/test_filesproc.py

@@ -1,6 +1,6 @@
 import pytest
 
-from iad.core.fs.filesproc import Locator, Path
+from iad.core.fs.filesproc import Locator, Path, inc_file_sfx
 
 
 def test_locator(tmp_path):
@@ -95,3 +95,32 @@ def test_locator_not_responsive(tmp_path):
     loc.validate(alarm=True)   # removes the "invalid" folder
     assert not any(loc.defined()), "Must have failed to validate in 0 time"
 
+
+def test_inc_file_sfx_returns_unused_path(tmp_path):
+    path = tmp_path / 'result.txt'
+
+    assert inc_file_sfx(path) == path
+
+
+def test_inc_file_sfx_starts_suffix_when_file_exists(tmp_path):
+    path = tmp_path / 'result.txt'
+    path.touch()
+
+    assert inc_file_sfx(path) == tmp_path / 'result_1.txt'
+
+
+def test_inc_file_sfx_increments_max_suffix(tmp_path):
+    path = tmp_path / 'result.txt'
+    (tmp_path / 'result_1.txt').touch()
+    (tmp_path / 'result_3.txt').touch()
+    (tmp_path / 'result_other.txt').touch()
+
+    assert inc_file_sfx(path) == tmp_path / 'result_4.txt'
+
+
+def test_inc_file_sfx_supports_custom_separator(tmp_path):
+    path = tmp_path / 'result.txt'
+    (tmp_path / 'result-2.txt').touch()
+
+    assert inc_file_sfx(path, sep='-') == tmp_path / 'result-3.txt'
+

{ialdev_core-0.2.5 → ialdev_core-0.2.6}/tests/test_pdtools.py

@@ -326,6 +326,34 @@ def test_qix_anonymous(sdt):
         sdt.qix('Im not there')
 
 
+def test_qix_reorder(sdt):
+    # default is a pure filter: storage order, not requested order
+    qixed = sdt.qix(x='1', y=['5', '3'])
+    assert list(qixed.index.get_level_values('y')) == ['3', '5']
+    # opt-in reorder follows the listed order, single index level
+    qixed = sdt.qix(x='1', y=['5', '3'], reorder=True)
+    assert list(qixed.index.get_level_values('y')) == ['5', '3']
+    # reorder on the columns axis
+    qixed = sdt.qix(z='data1', w=['c', 'a'], reorder=True)
+    assert list(qixed.columns.get_level_values('w')) == ['c', 'a']
+    # multi-level: first kwarg ('y') is the primary key
+    qixed = sdt.qix(y=['5', '3'], x=['2', '1'], reorder=True)
+    assert list(qixed.index.get_level_values('y'))[:2] == ['5', '5']
+    assert list(qixed.index.get_level_values('x'))[:2] == ['2', '1']
+
+
+def test_qix_reorder_duplicates_raise(sdt):
+    # duplicates are rejected only under reorder=True
+    with pytest.raises(ValueError):
+        sdt.qix(y=['3', '3'], reorder=True)
+    with pytest.raises(ValueError):
+        sdt.qix(w=['a', 'a'], reorder=True)
+    with pytest.raises(ValueError):
+        sdt.qix('a', w='a', reorder=True)
+    # without reorder the same duplicates are tolerated (no raise)
+    assert sdt.qix(y=['3', '3']) is not None
+
+
 def test_freeze(multi_label_data_table):
     dt = multi_label_data_table
     initial_id = id(dt)
@@ -618,6 +646,8 @@ if __name__ == '__main__':
     test_qix_basic(sdt)
     test_qix_specified(sdt)
     test_qix_anonymous(sdt)
+    test_qix_reorder(sdt)
+    test_qix_reorder_duplicates_raise(sdt)
     test_freeze()
     test_kron()
     test_outer()