ialdev-core 0.2.4__tar.gz → 0.2.6__tar.gz

{ialdev_core-0.2.4 → ialdev_core-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ialdev-core
-Version: 0.2.4
+Version: 0.2.6
 Summary: iad.core — algorithmic utilities: data tools, parameters, paths, caching, and more
 Author: ipcoder
 Requires-Python: >=3.10

{ialdev_core-0.2.4 → ialdev_core-0.2.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "ialdev-core"
-version = "0.2.4"
+version = "0.2.6"
 description = "iad.core — algorithmic utilities: data tools, parameters, paths, caching, and more"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -63,6 +63,7 @@ name = "iad.core"
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
+addopts = "--import-mode=importlib"
 python_files = ["test_*.py"]
 python_classes = ["Test*"]
 python_functions = ["test_*"]

ialdev_core-0.2.6/src/iad/core/data/labeled.py

@@ -0,0 +1,249 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from .label import Labels
+
+if TYPE_CHECKING:
+    from .pdtools import DTable
+
+__all__ = ['Labeled', 'LabeledType']
+
+
+class Labeled(Labels):
+    """
+    Base ABSTRACT class for Labeled data types - an extended dict containing data
+    in one field and descriptive labels in the others.
+
+    Actual Labeled Data types are created for specific collection of labels
+    (by ``LabeledType`` metaclass) using ``labeled_type()`` or ``LT`` alias
+    """
+
+    @classmethod
+    def type_keys(cls) -> tuple:
+        return *cls._defaults, *cls._undefined
+
+    UNDEF = object()
+
+    def __eq__(self, other):
+        if not (isinstance(other, dict) and len(self) == len(other)):
+            return False
+        no_key = object()
+        try:
+            for k, vo in other.items():
+                if (v := self.get(k, no_key)) is no_key:
+                    return False
+                na = isinstance(v, np.ndarray) + isinstance(vo, np.ndarray)
+                if na == 1 or na == 0 and vo != v or na == 2 and not np.array_equal(vo, v):
+                    return False
+        except Exception as ex:  # various not comparable types
+            print(ex)
+            return False
+        return True
+
+    def __init__(self, data=UNDEF, **kws):
+        """
+        Support dict-like initializations:
+            A(data=2, b=3) and A({'data':2, 'y':3})
+        and silent data field:
+            A(2, b=3)  == A(data=2, b=3)
+        :param data: not defined or data or dict with labels (like kws)
+        :param kws: labels key, values
+        """
+        cls = type(self)
+        if cls is Labeled:
+            raise RuntimeError(f"Class {cls} is a mixin class not intended for instantiation.")
+
+        # ---- STEP 1 - convert inputs into homogeneous form
+        if data is not self.UNDEF:
+            # support Labeled(dict) like initialization
+            if isinstance(data, dict) and self._all_keys.issuperset(data):
+                kws.update(data)
+            else:
+                assert 'data' in self._all_keys, "unnamed argument allowed only if 'data' field defined!"
+                kws['data'] = data
+
+        # ---- STEP 2 - check all the conditions are met
+        if missing := set(self._undefined).difference(kws):
+            raise KeyError(f"labels {missing} not initialized in {cls}!")
+
+        if violated := '\n\t'.join(
+                f"{k}={assigned} ({expected=})"
+                for k in self._frozen_defaults.intersection(kws)
+                if (expected := self._defaults[k]) != (assigned := kws[k])
+        ): raise KeyError(f"Initialization of {cls} violates frozen defaults:\n {violated}")
+
+        if self._frozen_keys and not self._all_keys.issuperset(kws):
+            raise KeyError(f"Undefined keys {set(kws).difference(self._all_keys)} in frozen {cls}")
+
+        # ---- STEP 3 - initialize the items
+        super().__init__(self._defaults)  # first fill with defaults
+        # then assign from the arguments
+        for k, v in kws.items():
+            if self._validate_type and \
+                    (tp := self._undefined.get(k, False)) and \
+                    type(tp) is type and not isinstance(v, tp):
+                v = tp(v)
+            self[k] = v
+
+    def flat(self, *, sep='_', exclude='data', data='data', fmt=None) -> tuple[str, np.ndarray]:
+        """
+        Merge labels into a single string and produce a tuple (flatten_label, data)
+
+        :param sep: separator when joining
+        :param exclude: keys exclude from the join
+        :param data: key of the data field returned as second item in the tuple
+        :param fmt: optional str.format sting, if provided ignores arguments:
+                sep, exclude, keys
+        :return: tuple(joint_label, data_item)
+        """
+        from iad.core import as_iter
+        data = self[data]
+        if fmt:
+            return fmt.format(**self), data
+
+        excluded = set(as_iter(exclude))
+        included = lambda _: _[0] not in excluded
+        str_val = lambda _: str(_[1])
+        return sep.join(map(str_val, filter(included, self.items()))), data
+
+    def __setattr__(self, item, val):
+        if item in self:
+            self[item] = val
+        else:
+            super().__setattr__(val)
+
+    @classmethod
+    def from_table(cls, db: DTable, *, missing=KeyError, undef=()):
+        """
+        Create instance of this Labeled type from :cls:`iad.core.data.pdtools.DataTable`
+        All the pre-defined keys are used to query the index.
+        All the undefined keys must be in the table and are initialized
+
+        If table does not contain given label, then depending on the value of ``missing``:
+         - `issubclass(missing, BaseException)` - raise `missing(f"{type(ex)}({ex.args})")`
+         - `isinstance(missing, BaseException)` - raise `missing`
+         - otherwise, return `missing`
+
+        :param db:
+        :param missing: Exception class, instance, or value to return
+        :param undef: keys listed here are forced to be considered as undefined
+        :return:
+        """
+        if undef:
+            undef = set(undef)
+            if undef := undef.difference(cls._undefined):  # undef not already undefined
+                if not_defined := undef.difference(cls._defaults):  # must be defined!
+                    raise KeyError(f"Unknown keys: {not_defined}")
+            # move defined undef from defined into undefined
+            defined = {k: v for k, v in cls._defaults.items() if k not in undef}
+            undefined = [*undef, cls._undefined]
+        else:
+            defined = cls._defaults
+            undefined = [*cls._undefined]
+
+        try:
+            res = db.select(defined)
+        except (KeyError, LookupError) as ex:
+            if isinstance(missing, BaseException):
+                raise missing
+            elif isinstance(missing, type) and issubclass(missing, BaseException):
+                raise missing(f"{type(ex)}({ex.args})")
+            return missing
+        if hasattr(res, 'columns'):
+            res = res[undefined]
+        elif not (len(undefined) == 1 and res.name == undefined[0]):
+            raise KeyError(f"Table does not contain required keys {undefined}")
+        return cls(**res.reset_index().iloc[0].to_dict())
+
+
+class LabeledType(type):
+    """ Metaclass to create Labeled Data classes extending ``dict``,
+    but built around predefined keys with optional default values.
+
+    Main purpose of such classes to ensure that during instantiation all
+    those keys are initialized either by their defaults or as arguments.
+
+    Note: additional keys except those defined in the class may be added.
+
+    Classes are supposed to be created by directly calling this meta:
+
+    >>> LabeledXY = LabeledType(x=10, y=int)
+    >>> d = LabeledXY(y=2)   # LT(x=2) raise exception for undefined y!
+    >>> assert d['y'] == 10
+    >>> assert type(d.__name__) == 'Labeled'  # default class name
+    >>> LabeledData = LabeledType('LabeledData', x=10, y=int)
+    >>> assert LabeledData.__name__ == 'LabeledData'  # custom class name
+
+    To encorage usage of 'data' key for data its positional initialization
+    is supported if data key is introduced explicitely or using `LD` function
+    >>> LabeledData = LabeledType(data=NDArray, y=int) # equivalent = LD(y=int)
+    >>> d = LabeledData([1,2,3], y=20)
+    >>> assert d['data'] == [1,2,3]
+    """
+
+    def __repr__(cls):
+        undefined = (f"{k}: {v.__name__}" for k, v in cls._undefined.items())
+        defaults = (f"{k}={v}" for k, v in cls._defaults.items())
+        return f"{cls.__qualname__}<{', '.join((*undefined, *defaults))}>"
+
+    def __new__(mcs, _name=None, *, validate_type=False,
+                frozen_defaults=False, frozen_keys=False,
+                **labels):
+        """
+        :param _name: Name of the created class
+        :param validate_type: perform casting on keys initialization if possible
+        :param frozen_defaults: allow change default values (False)
+               use True to define templates
+        :param frozen_keys: allow not defined keys
+        :param labels: dict with {key: default_value} and {key: type} items
+        """
+        _name = _name or f"Labeled"
+        # 'undefined' are keys assigned by types, not
+        # default values which would be called 'defaults'.
+        cls = super().__new__(mcs, _name, (Labeled,), {})
+
+        cls._undefined = {k: labels.pop(k) for k in
+                          [_ for _, v in labels.items() if type(v) is type]}
+        cls._defaults = labels
+        cls._all_keys = {*cls._undefined, *cls._defaults}
+
+        frozen_defaults = cls._defaults if frozen_defaults is True else \
+            [] if frozen_defaults is False else frozen_defaults
+        cls._frozen_defaults = {*frozen_defaults}
+        cls._frozen_keys = frozen_keys
+        cls._validate_type = validate_type
+        return cls
+
+    def __init__(mcs, _name=None, *, validate_type=False,
+                 frozen_defaults=False, frozen_keys=False,
+                 **labels):
+        """
+        :param _name: Name of the created class
+        :param validate_type: perform casting on keys initialization if possible
+        :param frozen_defaults: allow change default values (False)
+               use True to define templates
+        :param frozen_keys: allow not defined keys
+        :param labels: dict with {key: default_value} and {key: type} items
+        """
+        pass  # to avoid calling type.__init__ with unusual arguments
+
+    def __contains__(cls, item):
+        return item in cls._defaults or item in cls._undefined
+
+    @property
+    def undefined(cls):
+        """keys with undefined default values"""
+        return cls._undefined.keys()
+
+    @property
+    def default(cls):
+        """keys with defined default values"""
+        return cls._defaults.keys()
+
+    @property
+    def defined(cls):
+        """set of ALL the keys with undefined AND default values"""
+        return cls._all_keys

{ialdev_core-0.2.4 → ialdev_core-0.2.6}/src/iad/core/data/pdtools.py

@@ -4,6 +4,7 @@ import logging
 import re
 import warnings
 from collections import namedtuple
+from collections.abc import Mapping
 from dataclasses import dataclass
 from functools import cmp_to_key
 from io import StringIO
@@ -767,7 +768,7 @@ class TablePainter:
             styled.set_caption(self.cap)
 
         if self.prec is not None:
-            styled.set_precision(self.prec)
+            styled.format(precision=self.prec)
 
         if not self.disp:
             return styled
@@ -817,8 +818,8 @@ def side_tables(*tables, caps=None, hide_index=None, painter=None, **kws):
                     index.names = [None] * len(index.names)
                     columns = table.columns[[0]]
                     t = pd.DataFrame(data=np.arange(len(table.index)), index=index, columns=columns).style
-                    yield t.hide_columns([*columns]), None
-                yield table.hide_index(), cap
+                    yield t.hide([*columns], axis="columns"), None
+                yield table.hide(axis="index"), cap
             else:
                 yield table, cap
 
@@ -1049,6 +1050,56 @@ def _expand_vec_values(pairs: list[tuple[Scalar, Scalar | Vector]], prev_expande
         yield prev_expanded  # return as received - start aggregation
 
 
+def pivot_flat(df, base: str | list[str], *, pivot: str | list[str], 
+               index: bool | list[str] | str | None = None, sep=','):
+    """
+    In the given dataframe creates multi-index columns by adding `pivot` levels to the `base` columns` 
+    and flattens them into str using sep.
+
+    All the operations are done using on both df.columns and df.index.names.
+
+    Sets `index` columns as multi-index levels.
+     - if `index` is True, use all the remaining columns not mentioned in `pivot`.
+     - if `index` is False, index is reset completely
+     - if `index` is None, leaves original index levels (except of pivoted).    
+    """
+    remove = lambda x, c : [_ for _ in x if _ not in c]
+
+    org_index = df.index.names if isinstance(df.index, pd.MultiIndex) else []
+    org_columns = df.columns    
+    all_labels = [*org_index, *org_columns]
+
+    invalid = list(filter(lambda _: not isinstance(_, (str, type(None))), org_columns))
+    assert not invalid, f"Some of columns/ index levels are not strings: {invalid}"
+
+    def check(x, name):
+        x = as_list(x)    
+        assert x, f"At least one {x} needed" 
+        assert not (_:= set(x).difference(all_labels)), f"Unknown {name}: {_}"
+        return x
+
+    base, pivot = check(base, 'base'), check(pivot, 'pivot')
+
+    # leave only base as columns, and then pivot reuired levels     
+    df = df.set_index(remove(all_labels, base)).unstack(pivot)
+    df.columns = [sep.join(col) for col in df.columns] # flatten columns names into str
+
+    if index is True:
+        index = df.index.names
+    elif index is False:
+        index = []
+    elif index is None:
+        index = re|move(org_index, pivot)
+    else:
+        index = as_list(index)
+        assert not (_:= set(index).difference(df.index.names)), f"Invaid index levels: {_}"
+
+    pivot_columns = list(df.columns)
+    to_columns = remove(df.index.names, index)
+    df = df.reset_index(to_columns)
+    return df, pivot_columns
+
+
 def _row_queries_gen(labels, levels, levels_map) -> Generator[list[tuple[int, Scalar]]]:
     """
     Generator of mult-index row queries in form of:
@@ -1320,6 +1371,24 @@ def trans(s, max_col_width=20):
     return ss
 
 
+def _format_display_cell(value, fnc):
+    """Apply a ``TableFormats`` cell formatter, while leaving null cells empty.
+
+    NA detection is guarded to scalars so array-like cells (the very ones we
+    want to compact, e.g. ndarrays) never reach the ambiguous ``pd.isna``
+    truth-value check.
+    """
+    if value is None:
+        return ""
+    if np.ndim(value) == 0:
+        try:
+            if pd.isna(value):
+                return ""
+        except (TypeError, ValueError):
+            pass
+    return fnc(value)
+
+
 def _invert_levels(levels, names):
     if isinstance(levels, (str, int)):
         levels = [levels]
@@ -1329,8 +1398,71 @@ def _invert_levels(levels, names):
         return names.difference(levels)
 
 
+def _reduce(obj: PTable, op: str, *, axis=0, level=None, numeric_only=None):
+    """Apply a reduction (``'mean'`` | ``'sum'`` | ``'count'``) over an axis, or,
+    if ``level`` is given, by grouping that index level.
+
+    This replaces the ``level=`` argument of the aggregation methods, which was
+    removed in pandas 2.0 in favour of ``groupby(level=...)``.
+    """
+    num_kws = {} if (numeric_only is None or op == 'count') else dict(numeric_only=numeric_only)
+    if level is not None:
+        # GroupBy methods are the stock pandas implementations, so this also
+        # bypasses any aggregation overrides on the table subclasses.
+        return getattr(obj.groupby(level=level), op)(**num_kws)
+    # Call the unbound pandas method explicitly to bypass subclass overrides
+    # (e.g. _TableMixIn.mean), matching the pre-pandas-2 behaviour.
+    base = pd.Series if isinstance(obj, pd.Series) else pd.DataFrame
+    if base is pd.Series:
+        return getattr(base, op)(obj, **num_kws)
+    return getattr(base, op)(obj, axis=axis, **num_kws)
+
+
 class _TableMixIn:
 
+    def _display_frame(self) -> pd.DataFrame:
+        """Build a plain ``pd.DataFrame`` copy for marimo's interactive table.
+
+        Object columns (which may hold large ndarrays, dicts, paths, ...) are
+        replaced by their compact ``TableFormats`` string representation, the
+        same one used by ``_repr_html_``. Numeric / bool / datetime / string
+        columns are left native so marimo keeps real sorting and filtering.
+
+        A plain ``pd.DataFrame`` (not a ``DataTable``) is returned on purpose:
+        it has no ``_display_`` of its own, so marimo applies its built-in
+        opinionated formatter and renders an interactive, sortable, filterable
+        table whose cells are already short.
+        """
+        src = self if self.ndim == 2 else self._prep_repr()
+        plain = pd.DataFrame(src)
+        formatters = TableFormats.formatters(plain)
+        for i, fnc in enumerate(formatters):
+            if fnc is None or plain.dtypes.iloc[i] != object:
+                continue
+            col = plain.iloc[:, i]
+            plain.isetitem(i, col.map(lambda v, _f=fnc: _format_display_cell(v, _f)))
+        return plain
+
+    def _display_(self):
+        """marimo display protocol.
+
+        Has the highest precedence in marimo's formatter resolution, so it
+        overrides marimo's built-in opinionated formatter for ``pd.DataFrame``
+        / ``pd.Series`` (which would otherwise dump full ndarray contents for
+        the ``data`` column and similar).
+
+        We return a plain ``pd.DataFrame`` with object columns pre-rendered to
+        compact strings, so marimo still shows its interactive table (sorting,
+        filtering, pagination) but without dumping large arrays. On any failure
+        we fall back to the static rich HTML used by Jupyter.
+        """
+        try:
+            return self._display_frame()
+        except Exception:
+            import marimo as mo
+            html = self._repr_html_()
+            return mo.plain(self) if html is None else mo.Html(html)
+
     def as_labels(self, index=True, data=False, squeeze=False) -> Labels | list[Labels]:
         """
         Convert into list of datacast.labels.Labels objects.
@@ -1436,7 +1568,7 @@ class _TableMixIn:
         if strict:
             missing = set(levels).difference(index_names)
             if missing:
-                tbl = tbl.stack([*missing])  # will raise her if not found in columns
+                tbl = tbl.stack([*missing], future_stack=True)  # will raise her if not found in columns
                 index_names.update(missing)
 
         tbl = tbl.unstack([*index_names.difference(levels)], **kws)
@@ -1469,7 +1601,13 @@ class _TableMixIn:
                     raise KeyError(f"Levels {missing} not found in index {tbl.index.names}")
                 col_names.update(missing)
 
-        tbl = tbl.stack([*col_names.difference(levels)], **kws)  # Need remove NULL rows?
+        # pandas 2: the new stack (future_stack=True) no longer accepts/handles
+        # `dropna`/`sort` and never drops NaN rows, so emulate the old default.
+        dropna = kws.pop('dropna', True)
+        kws.pop('sort', None)
+        tbl = tbl.stack([*col_names.difference(levels)], future_stack=True, **kws)
+        if dropna:
+            tbl = tbl.dropna(how='all') if tbl.ndim > 1 else tbl.dropna()
 
         if tbl.ndim > 1 and tbl.columns.nlevels > 1:  # if there are col levels
             levels = [lvl for lvl in levels if lvl in col_names]
@@ -1702,18 +1840,32 @@ class _TableMixIn:
 
     def qix(self: PTable, *anonymous,
             drop_level: bool | Collection = False, keep: Collection = None,
-            axis: AxisT = None, key_err=True, **named) -> PTable:
+            axis: AxisT = None, key_err=True, reorder=False, **named) -> PTable:
         """
         Filter a dataframe or series using fuzzy query of its multi-index.
-        Arguments in different forms describe the index values to be searched for.
 
-        Some index levels in the output can be dropped by specifying either
-        ``drop_level`` or ``keep`` (mutually exclusive) arguments.
-
-        Use ``drop_level=True`` to drop all the redundant levels used in the query, or
-        just provide one (or list) of specific levels to drop.
-        Would not drop if the level is important for indexing the data (there is no redundancy),
-        even if specified in the drop_level list
+        ``qix`` is a FILTER, not a selector: each requested value may match
+        zero, one, or many rows, and results follow the table's storage
+        order. For structured, ordered, one-row-per-label retrieval (e.g.
+        ``img, gt = ...``) use :meth:`select`.
+
+        Capabilities, from simple to advanced:
+
+        1. One level by name:        ``dt.qix(w='a')``
+        2. Several levels (AND):     ``dt.qix(w='a', x='1')``
+        3. Several values per level: ``dt.qix(w=['b', 'a'])``
+           (membership; rows stay in storage order)
+        4. Both axes auto-matched:   ``dt.qix(w='a', y='4')``
+           - restrict with ``axis='index'``/``'columns'`` (or 0/1)
+        5. Anonymous values:         ``dt.qix('a', '3')``
+           - search a value without naming its level (error if ambiguous)
+        6. Collapse redundant levels with ``drop_level`` / ``keep``:
+           ``dt.qix(w='a', drop_level=True)`` (drops only when one value remains)
+        7. Tolerate misses with ``key_err=False`` (returns empty table)
+        8. Opt-in ordering:          ``dt.qix(w=['b', 'a'], reorder=True)``
+           - with ``reorder=True`` the listed values define each level's
+             output order (first keyword = primary key; unqueried levels keep
+             storage order), and duplicate listed values raise ``ValueError``.
 
         :param self: The table to filter
         :param anonymous: list of values from one of the index levels
@@ -1723,6 +1875,10 @@ class _TableMixIn:
         :param keep: a collection of levels to keep - excludes using drop_level
         :param axis: if specified query only this axis
         :param key_err: raise KeyError if index not found or return empty
+        :param reorder: if True, reorder results to the order values are listed
+                     per level (kwargs order = precedence) and raise ValueError
+                     on duplicate listed values. Default False (pure filter,
+                     storage order).
         :param named:  {level: value} - specific levels to find,
                             eliminates exhaustive search in all levels
         :return: Filtered table
@@ -1730,8 +1886,6 @@ class _TableMixIn:
         Tutorial:
         Using this DataTable:
 
-        >>> from iad.core.tests.test_pdtools import sdt_general
-        >>> dt= sdt_general()
         >>> dt # doctest: +NORMALIZE_WHITESPACE
         z   data1          data2
         w       a  b  c  d     a  b  c  d
@@ -1776,6 +1930,17 @@ class _TableMixIn:
         x
         1      1      5
         2      1      5
+
+        By default qix is a filter and keeps storage order. Pass
+        ``reorder=True`` to order results by the values you list.
+        Example 4 - opt-in ordering (note y follows the listed [5, 3]):
+
+        >>> dt.qix(x='1', y=['5','3'], drop_level=True, reorder=True) # doctest: +NORMALIZE_WHITESPACE
+        z data1       data2
+        w     a b c d     a b c d
+        y
+        5     1 2 3 4     5 6 7 8
+        3     1 2 3 4     5 6 7 8
         """
         # ToDo: add option to search not only in the index
         # FixMe: self.empty check twice!
@@ -1822,6 +1987,17 @@ class _TableMixIn:
             logging.debug(f'qix:\n{err}')
             return self.iloc[0:0]
 
+        if reorder:
+            for lvl, val in named.items():
+                vals = as_list(val)
+                if len(vals) != len(set(vals)):
+                    raise ValueError(f"Duplicate values with reorder=True: {lvl=}, {vals=}")
+            for ax, axis_kws in enumerate(axs_kws):
+                # follow kwargs (named) order so the first listed level is primary
+                ordered = {lvl: named[lvl] for lvl in named if lvl in axis_kws}
+                if ordered and (keys := order_keys(res.axes[ax], ordered)):
+                    res = res.take(np.lexsort(keys[::-1]), axis=ax)
+
         if drop_level:
             to_set = lambda x: {x} if isinstance(x, (str, int)) else set(x)
             for ax, (kws, axis) in enumerate(zip(axs_kws, res.axes)):
@@ -1896,15 +2072,13 @@ class _TableMixIn:
             Include only float, int, boolean columns. If None, will attempt to use
             everything, then use only numeric data. Not implemented for Series.
         """
-        series = isinstance(self, pd.Series)
-        kws = dict(axis=axis, level=level, numeric_only=numeric_only)
-        mean = (pd.Series if series else pd.DataFrame).mean(self.nominal_value, **kws)
+        mean = _reduce(self.nominal_value, 'mean', axis=axis, level=level, numeric_only=numeric_only)
         if not (hasattr(mean, 'index') and hasattr(self, 'nominal_value')) or self.std_dev.sum().sum() == 0:
             return mean
 
-        kws.update(level=level)
-        num = self.std_dev.count(**(dict(level=level) if series else kws))
-        sd = ((self.std_dev ** 2).sum(**kws) / num) ** 0.5
+        num = _reduce(self.std_dev, 'count', axis=axis, level=level)
+        sd = (_reduce(self.std_dev ** 2, 'sum', axis=axis, level=level,
+                      numeric_only=numeric_only) / num) ** 0.5
         from uncertainties.unumpy import uarray
         return mean.__class__(uarray(mean, sd), index=mean.index,
                               **({} if mean.ndim == 1 else {'columns': mean.columns}))
@@ -1930,11 +2104,14 @@ class _TableMixIn:
             self = self.unstack_but(ws.index.names)  # align indices
             assert ws.shape[0] == self.shape[0], f"{ws.shape=} vs {self.shape=}"
 
-        ws = ws / ws.sum(level=level)
-        avr = (self.multiply(ws, 0)).sum(level=level).unstack_but(level)
+        if level is None:
+            ws = ws / ws.sum()
+        else:
+            ws = ws / ws.groupby(level=level).transform('sum')
+        avr = _reduce(self.multiply(ws, 0), 'sum', level=level).unstack_but(level)
         if unc:
             from uncertainties.unumpy import uarray
-            sd = ((self.sub(avr) ** 2).multiply(ws, 0).sum(level=level)) ** 0.5
+            sd = (_reduce((self.sub(avr) ** 2).multiply(ws, 0), 'sum', level=level)) ** 0.5
             if unc is True:
                 data = uarray(avr.values, sd.values)
                 return DataSeries(data, index=avr.index) if avr.ndim == 1 else \
@@ -1948,11 +2125,17 @@ class _TableMixIn:
         index_names = self.index.names
         return pd.concat([self.reset_index(), other.reset_index()]).set_index(list(index_names))
 
-    def __or__(self: DTable, other: dict):
+    def __or__(self: DTable, other):
+        # Only the dict form adds index levels; delegate everything else
+        # (e.g. boolean masks ``mask_a | mask_b``) to the pandas operator.
+        if not isinstance(other, Mapping):
+            return super().__or__(other)
         (out := self.copy()).__ior__(other)
         return out
 
-    def __ior__(self: DTable, other: dict):
+    def __ior__(self: DTable, other):
+        if not isinstance(other, Mapping):
+            return super().__ior__(other)
         new_levels = list(other)
         self[new_levels] = [*other.values()]
         self.set_index(new_levels, append=True, inplace=True)
@@ -1980,6 +2163,30 @@ class _TableMixIn:
         return hash_value.hexdigest()[-n:]
 
 
+def order_keys(axis_index: pd.Index | pd.MultiIndex, kws) -> list[np.ndarray]:
+    """ Positional sort keys to reorder rows by the order values are listed.
+
+    For each level whose requested value is a list of two or more values,
+    map every axis entry to the position of its value in that list. Levels
+    with a scalar / single value are skipped (no reordering needed).
+
+    Args:
+        axis_index: the (multi-)index of the already-filtered axis
+        kws: {level: value | [values]} requested for this axis
+
+    Returns: list of int key arrays, one per listed level (kwargs order)
+    """
+    keys = []
+    for lvl, val in kws.items():
+        vals = as_list(val)
+        if len(vals) < 2:
+            continue
+        pos = {v: i for i, v in enumerate(vals)}
+        level_vals = axis_index.get_level_values(lvl) if axis_index.nlevels > 1 else axis_index
+        keys.append(level_vals.map(pos).to_numpy())
+    return keys
+
+
 def slicer(index: pd.Index | pd.MultiIndex, kws):
     """ Slicing the index with kws requested
     Args:
@@ -2021,7 +2228,8 @@ def path_fixer(root, fixer_name, cols='path') -> Callable[[pd.DataFrame], pd.Dat
 
     def fix_path(df):
         if columns := [c for c in df.columns if c in cols]:
-            df[columns] = df[columns].applymap(fixer(root))
+            df = df.copy()
+            df[columns] = df[columns].map(fixer(root))
         return df
 
     return fix_path
@@ -2262,8 +2470,9 @@ def append_col(df: pd.DataFrame,
                    by cols and the added into  `df`
     :param levels: optionally provide names for the levels,
                    must be same number of them as the final levels
-    :return: return the input df (the operations are inplace)
+    :return: a new data frame with the column(s) added (the input is not modified)
     """
+    df = df.copy()
 
     def add_levels(n):
         df.columns = pd.MultiIndex.from_product([df.columns, *[*[['']] * n]])
@@ -2516,7 +2725,7 @@ def filter_full_groups(df, group) -> (pd.DataFrame, int):
     group = as_list(group)
     mv_idx = df.index.names.difference(group)
     df = df.reset_index(mv_idx)  # leave only 'scene' in the index
-    full_groups = df.groupby(group).apply(lambda x: issubset_report(x.kind, ))
+    full_groups = df.groupby(group).apply(lambda x: issubset_report(x.kind, ), include_groups=False)
     df = df.loc[full_groups[full_groups].index]
     bad_scenes = (full_groups == False).sum()
     groups_num = len(full_groups) - bad_scenes

{ialdev_core-0.2.4 → ialdev_core-0.2.6}/src/iad/core/fs/filesproc.py

@@ -215,6 +215,32 @@ def rename_files_by_ref(files: Union[Iterable, str], ref_files: Union[Iterable,
     return old_new_pairs
 
 
+def inc_file_sfx(path, sep='_'):
+    """Given full path (<dir>/<stem>.<ext>) find if there are any files with such name 
+    or with additional suffix (<dir>/<stem><sep><num>.<ext>'),
+    and create a new name with suffix incremented over the maximal found.
+
+     - if only `path` start with '_1'
+     - if `path` is not found:
+         - found with suffixes - increment max suffix
+         - otherwise return the path    
+    """
+    path = Path(path).expanduser()
+    base = path.stem + sep
+    pat = re.compile(re.escape(base) + r'(\d+)')
+    last = max((
+        int(m.group(1))
+        for p in path.parent.iterdir() 
+        if p.is_file() and (m := pat.fullmatch(p.stem))
+    ), default=None)
+    
+    if last is None:    
+        stem = base + '1' if path.exists() else path.stem
+    else:
+        stem = base + str(last+1)
+    return path.with_stem(stem)
+
+
 def normalize(path: PathT, out: Type[Path] | Type[str] | None = Path):
     """
     Translate path into canonical form expanding user but not resolving links.

{ialdev_core-0.2.4 → ialdev_core-0.2.6}/src/iad/core/wrap.py

@@ -150,7 +150,6 @@ class NamedTupleI:
     ::
             isinstance(type(obj), NamedTupleMeta)
     """
-    from .data.nptools import array_info_str
 
     _fields: list[str]
     __iter__: Callable
@@ -158,7 +157,8 @@ class NamedTupleI:
     def __init__(self, *args, **kws): ...
 
     def __repr__(self):
-        val_str = lambda v: (NamedTupleI.array_info_str(v, stats=False)
+        from .data.nptools import array_info_str
+        val_str = lambda v: (array_info_str(v, stats=False)
                              if hasattr(v, 'shape') and hasattr(v, 'ndim') else str(v))
         fields_str = (f'{k}: {val_str(v)}' for k, v in zip(self._fields, self))
 

{ialdev_core-0.2.4 → ialdev_core-0.2.6}/tests/test_filesproc.py

@@ -1,6 +1,6 @@
 import pytest
 
-from iad.core.fs.filesproc import Locator, Path
+from iad.core.fs.filesproc import Locator, Path, inc_file_sfx
 
 
 def test_locator(tmp_path):
@@ -95,3 +95,32 @@ def test_locator_not_responsive(tmp_path):
     loc.validate(alarm=True)   # removes the "invalid" folder
     assert not any(loc.defined()), "Must have failed to validate in 0 time"
 
+
+def test_inc_file_sfx_returns_unused_path(tmp_path):
+    path = tmp_path / 'result.txt'
+
+    assert inc_file_sfx(path) == path
+
+
+def test_inc_file_sfx_starts_suffix_when_file_exists(tmp_path):
+    path = tmp_path / 'result.txt'
+    path.touch()
+
+    assert inc_file_sfx(path) == tmp_path / 'result_1.txt'
+
+
+def test_inc_file_sfx_increments_max_suffix(tmp_path):
+    path = tmp_path / 'result.txt'
+    (tmp_path / 'result_1.txt').touch()
+    (tmp_path / 'result_3.txt').touch()
+    (tmp_path / 'result_other.txt').touch()
+
+    assert inc_file_sfx(path) == tmp_path / 'result_4.txt'
+
+
+def test_inc_file_sfx_supports_custom_separator(tmp_path):
+    path = tmp_path / 'result.txt'
+    (tmp_path / 'result-2.txt').touch()
+
+    assert inc_file_sfx(path, sep='-') == tmp_path / 'result-3.txt'
+

{ialdev_core-0.2.4 → ialdev_core-0.2.6}/tests/test_pdtools.py

@@ -326,6 +326,34 @@ def test_qix_anonymous(sdt):
         sdt.qix('Im not there')
 
 
+def test_qix_reorder(sdt):
+    # default is a pure filter: storage order, not requested order
+    qixed = sdt.qix(x='1', y=['5', '3'])
+    assert list(qixed.index.get_level_values('y')) == ['3', '5']
+    # opt-in reorder follows the listed order, single index level
+    qixed = sdt.qix(x='1', y=['5', '3'], reorder=True)
+    assert list(qixed.index.get_level_values('y')) == ['5', '3']
+    # reorder on the columns axis
+    qixed = sdt.qix(z='data1', w=['c', 'a'], reorder=True)
+    assert list(qixed.columns.get_level_values('w')) == ['c', 'a']
+    # multi-level: first kwarg ('y') is the primary key
+    qixed = sdt.qix(y=['5', '3'], x=['2', '1'], reorder=True)
+    assert list(qixed.index.get_level_values('y'))[:2] == ['5', '5']
+    assert list(qixed.index.get_level_values('x'))[:2] == ['2', '1']
+
+
+def test_qix_reorder_duplicates_raise(sdt):
+    # duplicates are rejected only under reorder=True
+    with pytest.raises(ValueError):
+        sdt.qix(y=['3', '3'], reorder=True)
+    with pytest.raises(ValueError):
+        sdt.qix(w=['a', 'a'], reorder=True)
+    with pytest.raises(ValueError):
+        sdt.qix('a', w='a', reorder=True)
+    # without reorder the same duplicates are tolerated (no raise)
+    assert sdt.qix(y=['3', '3']) is not None
+
+
 def test_freeze(multi_label_data_table):
     dt = multi_label_data_table
     initial_id = id(dt)
@@ -373,11 +401,58 @@ def test_append_col():
                   [4, 5, 6],
                   [7, 8, 9]], index=pd.MultiIndex.from_tuples([(1, 1), (1, 2), (1, 3)]))
     dt.columns = pd.MultiIndex.from_tuples([(1, 1), (1, 2), (1, 3)])
-    append_col(dt, col='4', values=[10, 11, 12])
-    append_col(dt, col='4', values=[3, 1, 11])
+    dt = append_col(dt, col='4', values=[10, 11, 12])
+    dt = append_col(dt, col='4', values=[3, 1, 11])
     assert int(dt.loc[(1, 3), '4']) == 11
 
 
+def test_append_col_does_not_mutate_input():
+    """append_col must return a new frame and leave the input untouched (pandas 2 / CoW)."""
+    base = DT(data=[[1], [2]], columns=['a'])
+    out = pdt.append_col(base, col='b', values=[10, 20])
+    assert 'b' not in base.columns       # input untouched
+    assert out['b'].tolist() == [10, 20]  # result carries the new column
+
+
+def test_reduce_level_aggregation():
+    """`_reduce` replaces the removed pandas-1 `level=` aggregation argument."""
+    idx = pd.MultiIndex.from_product([['a', 'b'], [1, 2]], names=['g', 'i'])
+    s = DS([1.0, 3.0, 10.0, 20.0], index=idx)
+    df = DT({'v': [1.0, 3.0, 10.0, 20.0], 'w': [2.0, 4.0, 6.0, 8.0]}, index=idx)
+
+    # by level -> grouped result indexed by that level
+    assert pdt._reduce(s, 'mean', level='g').to_dict() == {'a': 2.0, 'b': 15.0}
+    assert pdt._reduce(s, 'sum', level='g').to_dict() == {'a': 4.0, 'b': 30.0}
+    assert pdt._reduce(s, 'count', level='g').to_dict() == {'a': 2, 'b': 2}
+    assert pdt._reduce(df, 'sum', level='g').loc['a'].to_dict() == {'v': 4.0, 'w': 6.0}
+
+    # no level -> plain reduction
+    assert pdt._reduce(s, 'sum') == 34.0
+    assert pdt._reduce(df, 'mean')['v'] == 8.5
+
+
+def test_bitwise_or_is_elementwise_for_masks():
+    """The custom __or__ must not hijack boolean-mask combination (`mask | mask`)."""
+    idx = pd.MultiIndex.from_product([['a', 'b'], [1, 2]], names=['g', 'i'])
+    s = DS([1.0, np.nan, np.inf, -np.inf], index=idx)
+    assert (s.isnull() | np.isinf(s)).tolist() == [False, True, True, True]
+
+    df = DT({'x': [1.0, np.nan], 'y': [np.inf, 2.0]})
+    assert (df.isnull() | ~np.isfinite(df)).values.tolist() == [[False, True], [True, False]]
+
+
+def test_or_dict_still_adds_index_level():
+    """The dict form of `|` keeps its index-level-appending behaviour."""
+    dt = DT({'v': [1, 2]})
+    out = dt | dict(mode='X', fid='001')
+    assert 'mode' in out.index.names and 'fid' in out.index.names
+    assert out.index.get_level_values('mode').tolist() == ['X', 'X']
+
+    dt2 = DT({'v': [1, 2]})
+    dt2 |= dict(mode='Y')
+    assert dt2.index.get_level_values('mode').tolist() == ['Y', 'Y']
+
+
 def test_sample(sdt):
     sample = pdt.sample
     assert len(sample(sdt, 2)) == 2
@@ -571,6 +646,8 @@ if __name__ == '__main__':
     test_qix_basic(sdt)
     test_qix_specified(sdt)
     test_qix_anonymous(sdt)
+    test_qix_reorder(sdt)
+    test_qix_reorder_duplicates_raise(sdt)
     test_freeze()
     test_kron()
     test_outer()