microdf-python 1.4.0__tar.gz → 1.4.1__tar.gz

{microdf_python-1.4.0/microdf_python.egg-info → microdf_python-1.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: microdf-python
-Version: 1.4.0
+Version: 1.4.1
 Summary: Weighted pandas DataFrames and Series for survey microdata
 Author-email: Max Ghenis <max@policyengine.org>
 License: MIT

microdf_python-1.4.1/microdf/_weights.py

@@ -0,0 +1,257 @@
+"""Pandas subclass hooks for copying weights and retaining row positions."""
+
+import numpy as np
+import pandas as pd
+
+
+def weight_series(values, index):
+    """Own an independent float array aligned positionally to an index."""
+    return pd.Series(np.array(values, dtype=float, copy=True), index=index)
+
+
+def aligned_weights(source, index):
+    """Align only when row identity can be established without guessing."""
+    if source.index.equals(index):
+        return weight_series(source.weights, index)
+    if source.index.is_unique:
+        positions = source.index.get_indexer(index)
+        if (positions >= 0).all():
+            return weight_series(source.weights.iloc[positions], index)
+    raise ValueError(
+        "Cannot propagate weights unambiguously to these rows. "
+        "Use positional selection, or explicitly supply weights for the result."
+    )
+
+
+def finalize_weights(result, source, method, previous_weights):
+    """Propagate row weights after pandas has finalized its own metadata."""
+    if method == "transpose" and result.ndim == 2:
+        raise ValueError(
+            "Cannot transpose row weights onto columns. "
+            "Convert to a pandas DataFrame first and supply explicit result weights."
+        )
+    if method == "concat":
+        objects = source.objs
+        if not all(isinstance(obj, WeightPropagationMixin) for obj in objects):
+            raise ValueError(
+                "Cannot concatenate weighted and unweighted objects: "
+                "provide explicit weights for every input."
+            )
+        weights = concat_weights(result, objects, getattr(source, "axis", None))
+        result.weights = weights
+        if result.ndim == 2:
+            names = [obj.__dict__.get("weights_col") for obj in objects]
+            result.weights_col = (
+                names[0] if all(name == names[0] for name in names) else None
+            )
+    elif isinstance(source, WeightPropagationMixin):
+        if method == "reset_index" and source.ndim == result.ndim == 1:
+            # Series.reset_index(drop=True) changes labels, never row order.
+            # pandas 2 constructs a result; pandas 3 relabels a shallow copy.
+            result.weights = weight_series(source.weights, result.index)
+        elif method == "rename" and previous_weights is not None:
+            result.weights = weight_series(previous_weights, result.index)
+        else:
+            result.weights = aligned_weights(source, result.index)
+    return result
+
+
+def concat_weights(result, objects, axis):
+    """Recover concat row identity without relying on pandas 2-only state."""
+    if objects[0].ndim == 1 and result.ndim == 2:
+        axis = 1
+    row_weights = None
+    if axis != 1 and len(result) == sum(len(obj) for obj in objects):
+        combined_index = objects[0].index.append([obj.index for obj in objects[1:]])
+        same_rows = result.index.equals(combined_index)
+        reset_rows = result.index.equals(pd.RangeIndex(len(result)))
+        keyed_rows = False
+        if (
+            isinstance(result.index, pd.MultiIndex)
+            and result.index.nlevels > combined_index.nlevels
+        ):
+            trailing = result.index.droplevel(
+                list(range(result.index.nlevels - combined_index.nlevels))
+            )
+            keyed_rows = trailing.equals(combined_index)
+        if same_rows or reset_rows or keyed_rows:
+            row_weights = np.concatenate([np.asarray(obj.weights) for obj in objects])
+    column_weights = None
+    if (
+        axis != 0
+        and result.ndim == 2
+        and len(result.columns)
+        == sum(obj.shape[1] if obj.ndim == 2 else 1 for obj in objects)
+    ):
+        values = np.empty(len(result))
+        known = np.zeros(len(result), dtype=bool)
+        conflict = False
+        for obj in objects:
+            if obj.index.equals(result.index):
+                positions = np.arange(len(result))
+            elif obj.index.is_unique:
+                positions = obj.index.get_indexer(result.index)
+            else:
+                conflict = True
+                break
+            present = positions >= 0
+            incoming = np.asarray(obj.weights)[positions[present]]
+            overlap = known[present]
+            if not np.array_equal(
+                values[present][overlap], incoming[overlap], equal_nan=True
+            ):
+                conflict = True
+                break
+            values[present] = incoming
+            known[present] = True
+        if not conflict and known.all():
+            column_weights = values
+    if row_weights is not None:
+        if column_weights is not None and not np.array_equal(
+            row_weights, column_weights, equal_nan=True
+        ):
+            raise ValueError(
+                "Ambiguous weights: pandas did not expose the concat axis."
+            )
+        return weight_series(row_weights, result.index)
+    if column_weights is not None:
+        return weight_series(column_weights, result.index)
+    raise ValueError(
+        "Cannot propagate concat weights: conflicting or ambiguous row weights."
+    )
+
+
+class WeightPropagationMixin:
+    """Use positional provenance for pandas operations that select rows."""
+
+    def _plain(self):
+        return (
+            pd.Series(self, copy=False)
+            if self.ndim == 1
+            else pd.DataFrame(self, copy=False)
+        )
+
+    def _weighted_result(self, plain, weights):
+        result = type(self)(plain, weights=weight_series(weights, plain.index))
+        if self.ndim == 2:
+            result.weights_col = self.__dict__.get("weights_col")
+        return result
+
+    def _finish_row_operation(self, plain, positions, inplace=False):
+        result = self._weighted_result(plain, self.weights.iloc[positions])
+        if inplace:
+            self._update_inplace(result)
+            self.weights = result.weights
+            return None
+        return result
+
+    def take(self, indices, axis=0, **kwargs):
+        """Take values and weights using the same positional indexer."""
+        axis = self._get_axis_number(axis)
+        plain = self._plain().take(indices, axis=axis, **kwargs)
+        weights = self.weights.iloc[indices] if axis == 0 else self.weights
+        return self._weighted_result(plain, weights)
+
+    def _slice(self, slobj, axis=0):
+        plain = self._plain()._slice(slobj, axis=axis)
+        weights = self.weights.iloc[slobj] if axis == 0 else self.weights
+        return self._weighted_result(plain, weights)
+
+    def _reindex_with_indexers(
+        self, reindexers, fill_value=None, copy=False, allow_dups=False
+    ):
+        plain = self._plain()._reindex_with_indexers(
+            reindexers, fill_value=fill_value, allow_dups=allow_dups
+        )
+        if copy:
+            plain = plain.copy()
+        indexer = reindexers.get(0, (None, None))[1]
+        if indexer is not None:
+            if (np.asarray(indexer) < 0).any():
+                raise ValueError(
+                    "Cannot invent weights for new rows introduced by reindex."
+                )
+            weights = self.weights.iloc[indexer]
+        else:
+            weights = self.weights
+        return self._weighted_result(plain, weights)
+
+    def drop(
+        self,
+        labels=None,
+        axis=0,
+        index=None,
+        columns=None,
+        level=None,
+        inplace=False,
+        errors="raise",
+    ):
+        plain = self._plain().drop(
+            labels=labels,
+            axis=axis,
+            index=index,
+            columns=columns,
+            level=level,
+            errors=errors,
+        )
+        positions = pd.Series(np.arange(len(self)), index=self.index)
+        row_labels = (
+            index
+            if index is not None
+            else labels
+            if self._get_axis_number(axis) == 0
+            else None
+        )
+        if row_labels is not None:
+            positions = positions.drop(row_labels, level=level, errors=errors)
+        return self._finish_row_operation(plain, np.asarray(positions), inplace)
+
+    def sample(self, *args, **kwargs):
+        """Sample weights with the selected rows, including replacements."""
+        result = super().sample(*args, **kwargs)
+        result.weights = weight_series(result.weights, result.index)
+        return result
+
+    def sort_values(self, *args, **kwargs):
+        """Sort rows and weights together even when labels are duplicated."""
+        inplace = kwargs.pop("inplace", False)
+        axis = self._get_axis_number(kwargs.get("axis", 0))
+        if self.ndim == 2:
+            plain = self._plain()
+            if axis == 1:
+                plain = plain.sort_values(*args, **kwargs)
+                return self._finish_row_operation(plain, np.arange(len(self)), inplace)
+            marker = object()
+            plain = plain.copy(deep=False)
+            plain[marker] = np.arange(len(self))
+            plain = plain.sort_values(*args, **kwargs)
+            positions = np.asarray(plain.pop(marker), dtype=int)
+        else:
+            values = self._plain()
+            positions = pd.Series(
+                np.arange(len(self)), index=self.index, name=self.name
+            )
+            key = kwargs.pop("key", None)
+            positions = positions.sort_values(
+                *args,
+                key=lambda unused: values if key is None else key(values),
+                **kwargs,
+            )
+            plain = values.iloc[np.asarray(positions)]
+            plain.index = positions.index
+            positions = np.asarray(positions, dtype=int)
+        return self._finish_row_operation(plain, positions, inplace)
+
+    def sort_index(self, *args, **kwargs):
+        """Sort the index while preserving positional weight provenance."""
+        inplace = kwargs.pop("inplace", False)
+        axis = self._get_axis_number(kwargs.get("axis", 0))
+        if axis == 1:
+            plain = self._plain().sort_index(*args, **kwargs)
+            return self._finish_row_operation(plain, np.arange(len(self)), inplace)
+        positions = pd.Series(np.arange(len(self)), index=self.index).sort_index(
+            *args, **kwargs
+        )
+        plain = self._plain().iloc[np.asarray(positions)]
+        plain.index = positions.index
+        return self._finish_row_operation(plain, np.asarray(positions), inplace)

{microdf_python-1.4.0 → microdf_python-1.4.1}/microdf/microdataframe.py

@@ -8,93 +8,17 @@ import numpy as np
 import pandas as pd
 
 from microdf.microseries import MicroSeries, MicroSeriesGroupBy
+from microdf._weights import (
+    WeightPropagationMixin,
+    aligned_weights,
+    finalize_weights,
+    weight_series,
+)
 
 logger = logging.getLogger(__name__)
 
 
-class _MicroLocIndexer:
-    """Custom loc indexer that returns MicroDataFrame with proper weights."""
-
-    def __init__(self, mdf: "MicroDataFrame"):
-        self._mdf = mdf
-        # Get the parent's loc indexer
-        self._parent_loc = pd.DataFrame.loc.fget(mdf)
-
-    def __getitem__(self, key):
-        # Use the parent DataFrame's loc indexer
-        result = self._parent_loc[key]
-
-        if isinstance(result, pd.DataFrame):
-            # Get the filtered weights based on the result's index
-            new_weights = self._mdf.weights.reindex(result.index)
-            return MicroDataFrame(result, weights=new_weights)
-        elif isinstance(result, pd.Series):
-            # Single row or column selected
-            if result.name in self._mdf.columns:
-                # Column was selected - return MicroSeries with all weights
-                return MicroSeries(result, weights=self._mdf.weights)
-            else:
-                # Row was selected - return as-is (scalar values for each col)
-                return result
-        else:
-            # Scalar value
-            return result
-
-    def __setitem__(self, key, value):
-        self._parent_loc[key] = value
-        self._mdf._link_all_weights()
-
-    def __getattr__(self, name):
-        """Delegate unknown attributes to the parent loc indexer."""
-        return getattr(self._parent_loc, name)
-
-
-class _MicroILocIndexer:
-    """Custom iloc indexer that returns MicroDataFrame with proper weights."""
-
-    def __init__(self, mdf: "MicroDataFrame"):
-        self._mdf = mdf
-        # Get the parent's iloc indexer
-        self._parent_iloc = pd.DataFrame.iloc.fget(mdf)
-
-    def __getitem__(self, key):
-        # Use the parent DataFrame's iloc indexer
-        result = self._parent_iloc[key]
-
-        if isinstance(result, pd.DataFrame):
-            # Get the filtered weights based on the result's index
-            new_weights = self._mdf.weights.iloc[
-                self._mdf.index.get_indexer(result.index)
-            ]
-            new_weights = pd.Series(new_weights.values, index=result.index)
-            return MicroDataFrame(result, weights=new_weights)
-        elif isinstance(result, pd.Series):
-            # Single row or column selected
-            if isinstance(key, tuple) and len(key) == 2:
-                # df.iloc[:, col_idx] - column selection
-                row_key = key[0]
-                if isinstance(row_key, slice) and row_key == slice(None):
-                    # All rows selected for a column
-                    return MicroSeries(result, weights=self._mdf.weights)
-            # Check if this is a column (result index matches mdf index)
-            if result.index.equals(self._mdf.index):
-                return MicroSeries(result, weights=self._mdf.weights)
-            # Row selection - return as-is
-            return result
-        else:
-            # Scalar value
-            return result
-
-    def __setitem__(self, key, value):
-        self._parent_iloc[key] = value
-        self._mdf._link_all_weights()
-
-    def __getattr__(self, name):
-        """Delegate unknown attributes to the parent iloc indexer."""
-        return getattr(self._parent_iloc, name)
-
-
-class MicroDataFrame(pd.DataFrame):
+class MicroDataFrame(WeightPropagationMixin, pd.DataFrame):
     # Declare weight state as pandas metadata. pandas includes
     # _metadata attributes in the pickle state, so weights now survive
     # pickling, to_pickle/read_pickle and copy.deepcopy instead of
@@ -112,20 +36,54 @@ class MicroDataFrame(pd.DataFrame):
         :type weights: np.array
         """
         super().__init__(*args, **kwargs)
-        self.weights = None
+        # pandas normalizes mixed-dimensional concat inputs through this
+        # constructor, either as a Series or a one-column mapping. Preserve
+        # that Series' row weights before concat loses the original input.
+        weight_source = args[0] if args else kwargs.get("data")
+        if isinstance(weight_source, dict) and len(weight_source) == 1:
+            weight_source = next(iter(weight_source.values()))
+        if weights is None and isinstance(weight_source, MicroSeries):
+            weights = aligned_weights(weight_source, self.index)
+        self.weights = weight_series(np.ones(len(self)), self.index)
+        self.weights_col = None
         self.set_weights(weights)
         self._link_all_weights()
         self.override_df_functions()
 
-    def __finalize__(self, other, method=None, **kwargs) -> "MicroDataFrame":
-        """Retain copied weights when pandas finalizes a renamed result."""
-        copied_weights = getattr(self, "weights", None) if method == "rename" else None
+    @property
+    def _constructor(self):
+        return MicroDataFrame
+
+    # A row or a column-summary Series has no per-observation weights.
+    # _ixs wraps column selections using their unambiguous row provenance.
+    _constructor_sliced = pd.Series
+
+    def _ixs(self, i, axis=0):
+        result = pd.DataFrame(self, copy=False)._ixs(i, axis=axis)
+        if axis == 1:
+            return MicroSeries(result, weights=self.weights)
+        return result
+
+    def _get_item_cache(self, item):
+        # Weight arrays are independently mutable; cached column wrappers would
+        # retain stale copies after an in-place edit to frame.weights.
+        return self._ixs(self.columns.get_loc(item), axis=1)
+
+    def __finalize__(self, other, method=None, **kwargs):
+        previous = self.__dict__.get("weights")
         super().__finalize__(other, method=method, **kwargs)
-        if copied_weights is not None:
-            # rename already called copy(); metadata propagation must not
-            # replace those weights with the source's mutable Series.
-            self.weights = copied_weights
-        return self
+        return finalize_weights(self, other, method, previous)
+
+    @wraps(pd.DataFrame.cov)
+    def cov(self, *args, **kwargs) -> pd.DataFrame:
+        # Column summaries have no observation weights, even if labels match.
+        result = pd.DataFrame(self, copy=False).cov(*args, **kwargs)
+        return result.__finalize__(self, method="cov")
+
+    @wraps(pd.DataFrame.corr)
+    def corr(self, *args, **kwargs) -> pd.DataFrame:
+        result = pd.DataFrame(self, copy=False).corr(*args, **kwargs)
+        return result.__finalize__(self, method="corr")
 
     def __setstate__(self, state) -> None:
         """Restore a pickled MicroDataFrame.
@@ -141,23 +99,6 @@ class MicroDataFrame(pd.DataFrame):
             self._link_all_weights()
         self.override_df_functions()
 
-    @property
-    def loc(self) -> _MicroLocIndexer:
-        """Label-based indexer that preserves MicroDataFrame type and weights.
-
-        :return: Custom loc indexer for MicroDataFrame
-        """
-        return _MicroLocIndexer(self)
-
-    @property
-    def iloc(self) -> _MicroILocIndexer:
-        """Integer-based indexer that preserves MicroDataFrame type and
-        weights.
-
-        :return: Custom iloc indexer for MicroDataFrame
-        """
-        return _MicroILocIndexer(self)
-
     def override_df_functions(self) -> None:
         """Override DataFrame functions to work with weighted operations."""
         for name in MicroSeries.FUNCTIONS:
@@ -378,7 +319,7 @@ class MicroDataFrame(pd.DataFrame):
         pass
 
     def _link_all_weights(self) -> None:
-        if self.weights is None:
+        if self.weights is None or len(self.weights) == 0:
             if len(self) > 0:
                 self.set_weights(np.ones((len(self))))
         # In pandas 3.0+, columns are wrapped as MicroSeries on access via
@@ -472,33 +413,19 @@ class MicroDataFrame(pd.DataFrame):
         # that treats it as a Series (equals(), reindex() in __getitem__).
         self.set_weights(np.ones(len(self)))
 
-    def __getitem__(
-        self, key: Union[str, List]
-    ) -> Union[MicroSeries, "MicroDataFrame"]:
-        # Let pandas handle the initial slicing
-        result = super().__getitem__(key)
-
-        # If the result is a DataFrame, re-synchronize the weights
-        if isinstance(result, pd.DataFrame):
-            new_weights = self.weights.reindex(result.index)
-            return MicroDataFrame(result, weights=new_weights)
-
-        # If the result is a Series (single column), wrap as MicroSeries
-        if isinstance(result, pd.Series):
-            return MicroSeries(result, weights=self.weights)
-
-        # Otherwise, the result is a scalar, so just return it
-        return result
+    def __getitem__(self, key):
+        return super().__getitem__(key)
 
     def catch_series_relapse(self) -> None:
         # In pandas 3.0+, we don't need to track series class changes since
         # __getitem__ always wraps columns as MicroSeries on access.
         pass
 
-    def __setattr__(self, key, value) -> None:
+    def __setattr__(self, key, value):
+        weights = self.__dict__.get("weights") if key == "index" else None
         super().__setattr__(key, value)
-        # No need to call catch_series_relapse in pandas 3.0+ since we wrap
-        # on access rather than store MicroSeries internally.
+        if weights is not None and len(weights) == len(self.index):
+            self.weights = weight_series(weights, self.index)
 
     def reset_index(
         self,
@@ -567,13 +494,7 @@ class MicroDataFrame(pd.DataFrame):
             return out
 
     def copy(self, deep: Optional[bool] = True) -> "MicroDataFrame":
-        res = super().copy(deep)
-        # super().copy() corrupts self's column types to plain Series.
-        # Restore them in O(N) instead of O(N²) by calling
-        # _link_all_weights once rather than per-column __setitem__.
-        self._link_all_weights()
-        res = MicroDataFrame(res, weights=self.weights.copy(deep))
-        return res
+        return super().copy(deep)
 
     def drop(
         self,
@@ -605,55 +526,15 @@ class MicroDataFrame(pd.DataFrame):
             dropped.
         :return: MicroDataFrame or None if inplace=True.
         """
-        row_drop = axis in (0, "index") or index is not None
-        if inplace:
-            # Snapshot the pre-drop weights keyed by the pre-drop index so
-            # we can reindex to the surviving rows after the drop.
-            pre_drop_weights = pd.Series(self.weights.values, index=self.index.copy())
-            # Perform in-place drop on the parent DataFrame
-            super().drop(
-                labels=labels,
-                axis=axis,
-                index=index,
-                columns=columns,
-                level=level,
-                inplace=True,
-                errors=errors,
-            )
-            if row_drop:
-                surviving = pre_drop_weights.reindex(self.index)
-                self.weights = pd.Series(
-                    surviving.values, index=self.index, dtype=float
-                )
-            else:
-                self.weights = pd.Series(
-                    pre_drop_weights.values, index=self.index, dtype=float
-                )
-            self._link_all_weights()
-            return None
-        else:
-            res = super().drop(
-                labels=labels,
-                axis=axis,
-                index=index,
-                columns=columns,
-                level=level,
-                inplace=False,
-                errors=errors,
-            )
-            if row_drop:
-                # Row drop: keep only the weights for surviving rows,
-                # in the order of the resulting DataFrame.
-                pre_drop_weights = pd.Series(self.weights.values, index=self.index)
-                new_weights = pre_drop_weights.reindex(res.index).values
-            else:
-                new_weights = self.weights.values
-            out = MicroDataFrame(res, weights=new_weights)
-            # Guard against the set_weights path building weights with a
-            # default RangeIndex, which would misalign against res.index
-            # and silently zero weighted aggregations.
-            out.weights = pd.Series(new_weights, index=out.index, dtype=float)
-            return out
+        return super().drop(
+            labels=labels,
+            axis=axis,
+            index=index,
+            columns=columns,
+            level=level,
+            inplace=inplace,
+            errors=errors,
+        )
 
     def merge(
         self,

{microdf_python-1.4.0 → microdf_python-1.4.1}/microdf/microseries.py

@@ -6,6 +6,8 @@ from typing import Callable, List, Optional, Union
 import numpy as np
 import pandas as pd
 
+from microdf._weights import WeightPropagationMixin, finalize_weights, weight_series
+
 logger = logging.getLogger(__name__)
 
 
@@ -84,7 +86,7 @@ def _weighted_top_share(
     return top_sum / total_sum
 
 
-class MicroSeries(pd.Series):
+class MicroSeries(WeightPropagationMixin, pd.Series):
     # Declare ``weights`` as pandas metadata. pandas includes
     # _metadata attributes in the pickle state, so weights now survive
     # pickling, to_pickle/read_pickle and copy.deepcopy instead of
@@ -103,15 +105,26 @@ class MicroSeries(pd.Series):
         super().__init__(*args, **kwargs)
         self.set_weights(weights)
 
-    def __finalize__(self, other, method=None, **kwargs) -> "MicroSeries":
-        """Retain copied weights when pandas finalizes a renamed result."""
-        copied_weights = getattr(self, "weights", None) if method == "rename" else None
+    @property
+    def _constructor(self):
+        return MicroSeries
+
+    @property
+    def _constructor_expanddim(self):
+        from microdf.microdataframe import MicroDataFrame
+
+        return MicroDataFrame
+
+    def __finalize__(self, other, method=None, **kwargs):
+        previous = self.__dict__.get("weights")
         super().__finalize__(other, method=method, **kwargs)
-        if copied_weights is not None:
-            # rename already called copy(); metadata propagation must not
-            # replace those weights with the source's mutable Series.
-            self.weights = copied_weights
-        return self
+        return finalize_weights(self, other, method, previous)
+
+    def __setattr__(self, name, value):
+        weights = self.__dict__.get("weights") if name == "index" else None
+        super().__setattr__(name, value)
+        if weights is not None and len(weights) == len(self.index):
+            self.weights = weight_series(weights, self.index)
 
     @property
     def _values(self):
@@ -178,12 +191,7 @@ class MicroSeries(pd.Series):
         :type weights: np.array.
         """
         if weights is None:
-            if len(self) > 0:
-                self.weights = pd.Series(
-                    np.ones_like(self._values),
-                    index=self.index,
-                    dtype=float,
-                )
+            self.weights = weight_series(np.ones(len(self)), self.index)
         else:
             if len(weights) != len(self):
                 raise ValueError(
@@ -201,7 +209,7 @@ class MicroSeries(pd.Series):
             # its index first so we position-align rather than label-align.
             if isinstance(weights, pd.Series):
                 weights = weights.values
-            self.weights = pd.Series(np.asarray(weights), index=self.index, dtype=float)
+            self.weights = weight_series(weights, self.index)
 
     def nullify_weights(self) -> None:
         """Set all weights to 1, effectively making the Series unweighted.
@@ -824,16 +832,14 @@ class MicroSeries(pd.Series):
         )
 
     def groupby(self, *args, **kwargs) -> "MicroSeriesGroupBy":
-        gb = super().groupby(*args, **kwargs)
+        gb = pd.Series(self, copy=False).groupby(*args, **kwargs)
         gb.__class__ = MicroSeriesGroupBy
         gb._init()
         gb.weights = pd.Series(self.weights).groupby(*args, **kwargs)
         return gb
 
     def copy(self, deep: Optional[bool] = True):
-        res = super().copy(deep)
-        res = MicroSeries(res, weights=self.weights.copy(deep))
-        return res
+        return super().copy(deep)
 
     def clip(
         self,
@@ -865,15 +871,22 @@ class MicroSeries(pd.Series):
         equal_weights = self.weights.equals(other.weights)
         return equal_values and equal_weights
 
-    def __getitem__(
-        self, key: Union[str, int, slice, List, np.ndarray]
-    ) -> Union["MicroSeries", pd.Series]:
-        result = super().__getitem__(key)
+    def __getitem__(self, key):
+        if callable(key):
+            key = key(self)
+        result = pd.Series(self, copy=False).__getitem__(key)
         if isinstance(result, pd.Series):
-            weights = self.weights.__getitem__(key)
-            return MicroSeries(result, weights=weights)
+            positions = pd.Series(np.arange(len(self)), index=self.index).__getitem__(
+                key
+            )
+            return MicroSeries(result, weights=self.weights.iloc[np.asarray(positions)])
         return result
 
+    def repeat(self, repeats, axis=None):
+        # Use pandas to validate the repeat counts and axis argument.
+        positions = pd.Series(np.arange(len(self))).repeat(repeats, axis=axis)
+        return self.take(np.asarray(positions))
+
     def __getattr__(self, name: str) -> "MicroSeries":
         return MicroSeries(super().__getattr__(name), weights=self.weights)
 

microdf_python-1.4.1/microdf/tests/test_weight_propagation.py

@@ -0,0 +1,478 @@
+"""Weighted pandas operations preserve row identity, not just row labels."""
+
+import copy
+import pickle
+
+import numpy as np
+import pandas as pd
+import pytest
+
+import microdf as mdf
+
+
+def make_data(kind):
+    values = pd.Series([30.0, np.nan, 10.0, 20.0], index=[7, 7, 3, 7], name="x")
+    weights = [2.0, 5.0, 11.0, 17.0]
+    if kind == "series":
+        return mdf.MicroSeries(values, weights=weights)
+    return mdf.MicroDataFrame(values.to_frame(), weights=weights)
+
+
+def assert_rows(result, original, positions, values=None):
+    assert type(result) is type(original)
+    expected = original.weights.to_numpy()[positions]
+    np.testing.assert_array_equal(result.weights.to_numpy(), expected)
+    assert result.weights.index.equals(result.index)
+    assert result.weights is not original.weights
+    data = np.asarray(result if isinstance(result, mdf.MicroSeries) else result["x"])
+    if values is not None:
+        np.testing.assert_allclose(data, values, equal_nan=True)
+    total = np.nansum(data * expected)
+    assert (
+        result.sum() if isinstance(result, mdf.MicroSeries) else result.sum()["x"]
+    ) == total
+
+
+@pytest.mark.parametrize("kind", ["series", "frame"])
+@pytest.mark.parametrize("ascending", [True, False])
+@pytest.mark.parametrize("ignore_index", [True, False])
+def test_sort_values_preserves_positional_weights(kind, ascending, ignore_index):
+    original = make_data(kind)
+    args = () if kind == "series" else ("x",)
+    result = original.sort_values(*args, ascending=ascending, ignore_index=ignore_index)
+    positions = [2, 3, 0, 1] if ascending else [0, 3, 2, 1]
+    assert_rows(result, original, positions)
+    if ignore_index:
+        assert result.index.equals(pd.RangeIndex(4))
+    assert_rows(original, original.copy(), np.arange(4))
+
+
+@pytest.mark.parametrize("kind", ["series", "frame"])
+def test_fillna_and_row_transforms_preserve_weights(kind):
+    original = make_data(kind)
+    filled = original.fillna(4)
+    assert_rows(filled, original, np.arange(4), [30, 4, 10, 20])
+    assert_rows(filled.replace(30, 40), filled, np.arange(4), [40, 4, 10, 20])
+    assert_rows(filled.astype(float), filled, np.arange(4))
+    assert_rows(filled + 1, filled, np.arange(4), [31, 5, 11, 21])
+    filled.weights.iloc[0] = 100
+    assert original.weights.iloc[0] == 2
+
+
+@pytest.mark.parametrize("kind", ["series", "frame"])
+@pytest.mark.parametrize("ignore_index", [True, False])
+def test_sample_with_replacement_and_duplicate_labels(kind, ignore_index):
+    original = make_data(kind)
+    positions = (
+        pd.Series(np.arange(4)).sample(n=12, replace=True, random_state=17).to_numpy()
+    )
+    result = original.sample(
+        n=12, replace=True, random_state=17, ignore_index=ignore_index
+    )
+    assert_rows(result, original, positions)
+    assert len(set(positions)) < len(positions)
+
+
+@pytest.mark.parametrize("kind", ["series", "frame"])
+def test_duplicate_label_selections_and_sort_index(kind):
+    original = make_data(kind)
+    assert_rows(original.iloc[[3, 0, 3]], original, [3, 0, 3])
+    assert_rows(original.iloc[1:3], original, [1, 2])
+    assert_rows(original.loc[[3, 7]], original, [2, 0, 1, 3])
+    assert_rows(original.sort_index(kind="stable"), original, [2, 0, 1, 3])
+    if kind == "frame":
+        column = original.loc[[3, 7], "x"]
+        assert isinstance(column, mdf.MicroSeries)
+        np.testing.assert_array_equal(column.weights, [11, 2, 5, 17])
+        row = original.iloc[0]
+        assert type(row) is pd.Series
+
+
+@pytest.mark.parametrize("kind", ["series", "frame"])
+@pytest.mark.parametrize("options", [{}, {"ignore_index": True}, {"keys": ["a", "b"]}])
+def test_concat_rows_preserves_duplicate_and_repeated_weights(kind, options):
+    original = make_data(kind)
+    pieces = [original.iloc[[3, 0]], original.iloc[[2, 3]]]
+    result = pd.concat(pieces, **options)
+    assert_rows(result, original, [3, 0, 2, 3])
+
+
+def test_concat_columns_requires_consistent_row_weights():
+    first = mdf.MicroDataFrame({"x": [1, 2]}, index=[9, 3], weights=[4, 5])
+    second = mdf.MicroDataFrame({"y": [6, 7]}, index=[3, 9], weights=[5, 4])
+    result = pd.concat([first, second], axis=1)
+    assert isinstance(result, mdf.MicroDataFrame)
+    np.testing.assert_array_equal(result.weights, [4, 5])
+    assert result.sum().to_dict() == {"x": 14, "y": 58}
+    second.set_weights([50, 40])
+    with pytest.raises(ValueError, match="weights"):
+        pd.concat([first, second], axis=1)
+
+
+@pytest.mark.parametrize("kind", ["series", "frame"])
+def test_concat_with_unweighted_objects_is_explicit(kind):
+    weighted = make_data(kind)
+    plain = pd.Series([1, 2]) if kind == "series" else pd.DataFrame({"x": [1, 2]})
+    with pytest.raises(ValueError, match="weights"):
+        pd.concat([weighted, plain])
+
+
+@pytest.mark.parametrize("kind", ["series", "frame"])
+def test_reindex_new_rows_cannot_invent_weights(kind):
+    weighted = make_data(kind).iloc[:1]
+    with pytest.raises(ValueError, match="weights"):
+        weighted.reindex([7, 99])
+
+
+@pytest.mark.parametrize("kind", ["series", "frame"])
+def test_propagated_objects_keep_serialization_and_rename_isolation(kind):
+    original = make_data(kind).fillna(4)
+    renamed = (
+        original.rename("income")
+        if kind == "series"
+        else original.rename(columns={"x": "income"})
+    )
+    renamed.weights.iloc[0] = 99
+    assert original.weights.iloc[0] == 2
+    for result in [copy.deepcopy(original), pickle.loads(pickle.dumps(original))]:
+        assert_rows(result, original, np.arange(4))
+    if kind == "series":
+        assert original.name == "x"
+
+
+@pytest.mark.parametrize("kind", ["series", "frame"])
+def test_inplace_sort_and_fillna_preserve_weights(kind):
+    original = make_data(kind)
+    expected = original.copy()
+    args = () if kind == "series" else ("x",)
+    assert original.sort_values(*args, inplace=True, ascending=False) is None
+    assert_rows(original, expected, [0, 3, 2, 1])
+    original.fillna(4, inplace=True)
+    assert_rows(original, expected, [0, 3, 2, 1], [30, 20, 10, 4])
+
+
+@pytest.mark.parametrize("kind", ["series", "frame"])
+def test_boolean_selection_and_drop_with_duplicate_indexes(kind):
+    original = make_data(kind)
+    mask = np.array([True, False, True, False])
+    assert_rows(original[mask], original, [0, 2])
+    assert_rows(original.drop(index=7), original, [2])
+    assert_rows(original.loc[mask], original, [0, 2])
+    if kind == "series":
+        assert_rows(original.repeat(2), original, [0, 0, 1, 1, 2, 2, 3, 3])
+
+
+def test_column_access_uses_current_weights_without_mutating_earlier_series():
+    frame = mdf.MicroDataFrame({"x": [1, 2]}, weights=[3, 4])
+    earlier = frame["x"]
+    assert earlier.sum() == 11
+    frame.weights.iloc[0] = 10
+    assert frame["x"].sum() == 18
+    assert frame.loc[:, "x"].sum() == 18
+    assert earlier.sum() == 11
+
+
+def test_transpose_cannot_reinterpret_row_weights_as_column_weights():
+    frame = mdf.MicroDataFrame(
+        [[1, 2], [3, 4]], index=[0, 1], columns=[0, 1], weights=[5, 7]
+    )
+    with pytest.raises(ValueError, match="weights"):
+        frame.transpose()
+
+
+@pytest.mark.parametrize("inplace", [False, True])
+@pytest.mark.parametrize(
+    "index,level",
+    [
+        (pd.Index(["a", "a", "b"], name="row"), None),
+        (
+            pd.MultiIndex.from_tuples(
+                [("a", 1), ("a", 1), ("b", 2)], names=["group", "row"]
+            ),
+            None,
+        ),
+        (
+            pd.MultiIndex.from_tuples(
+                [("a", 1), ("a", 1), ("b", 2)], names=["group", "row"]
+            ),
+            "group",
+        ),
+    ],
+)
+def test_series_reset_index_drop_preserves_positional_weights(index, level, inplace):
+    original = mdf.MicroSeries(
+        [10, 20, 30], index=index, name="income", weights=[2, 3, 5]
+    )
+    source_weights = original.weights
+    expected = pd.Series(original).reset_index(level=level, drop=True, name="ignored")
+    result = original.reset_index(
+        level=level, drop=True, name="ignored", inplace=inplace
+    )
+    if inplace:
+        assert result is None
+        result = original
+    assert isinstance(result, mdf.MicroSeries)
+    pd.testing.assert_series_equal(pd.Series(result), expected)
+    pd.testing.assert_series_equal(
+        result.weights, pd.Series([2.0, 3.0, 5.0], index=expected.index)
+    )
+    assert result.sum() == 230
+    assert result.weights is not source_weights
+    result.weights.iloc[0] = 99
+    assert source_weights.iloc[0] == 2
+    source_weights.iloc[1] = 88
+    assert result.weights.iloc[1] == 3
+
+
+def test_series_reset_index_preserves_dataframe_and_invalid_inplace_behavior():
+    original = mdf.MicroSeries(
+        [10, 20], index=pd.Index(["a", "b"], name="row"), name="income", weights=[2, 3]
+    )
+    result = original.reset_index(name="amount")
+    assert isinstance(result, mdf.MicroDataFrame)
+    pd.testing.assert_frame_equal(
+        pd.DataFrame(result), pd.Series(original).reset_index(name="amount")
+    )
+    np.testing.assert_array_equal(result.weights, [2, 3])
+    result.weights.iloc[0] = 99
+    assert original.weights.iloc[0] == 2
+    with pytest.raises(TypeError, match="inplace"):
+        original.reset_index(inplace=True)
+
+
+@pytest.mark.parametrize(
+    "select,positions",
+    [
+        (lambda frame: frame.iloc[1:], [1, 2]),
+        (lambda frame: frame[["income"]], [0, 1, 2]),
+        (lambda frame: frame.iloc[:, :1], [0, 1, 2]),
+        (lambda frame: frame.loc[:, ["income"]], [0, 1, 2]),
+        (lambda frame: frame.iloc[[2, 0, 2]], [2, 0, 2]),
+        (lambda frame: frame.reindex(columns=["income"]), [0, 1, 2]),
+    ],
+    ids=["row-slice", "columns", "iloc-columns", "loc-columns", "repeated", "reindex"],
+)
+def test_selected_dataframe_weights_are_independently_mutable(select, positions):
+    source = mdf.MicroDataFrame(
+        {"income": [10.0, 20.0, 30.0], "other": [1, 2, 3]},
+        index=[7, 7, 3],
+        weights=[2, 3, 5],
+    )
+    selected = select(source)
+    expected_weights = np.array([2.0, 3.0, 5.0])[positions]
+    np.testing.assert_array_equal(selected.weights, expected_weights)
+    assert selected.weights.index.equals(selected.index)
+
+    selected.weights.iloc[0] = 100
+    expected_weights[0] = 100
+    np.testing.assert_array_equal(source.weights, [2, 3, 5])
+    assert source.income.sum() == 230  # 10 * 2 + 20 * 3 + 30 * 5.
+    assert selected.income.sum() == np.dot(
+        np.array([10.0, 20.0, 30.0])[positions], expected_weights
+    )
+
+    source.weights.iloc[-1] = 200
+    np.testing.assert_array_equal(selected.weights, expected_weights)
+    assert selected.income.sum() == np.dot(
+        np.array([10.0, 20.0, 30.0])[positions], expected_weights
+    )
+
+
+@pytest.mark.parametrize("series_first", [False, True])
+@pytest.mark.parametrize("options", [{}, {"ignore_index": True}, {"keys": ["a", "b"]}])
+@pytest.mark.parametrize("series_name", ["income", None])
+def test_concat_mixed_dimensions_preserves_series_weights(
+    series_first, options, series_name
+):
+    frame = mdf.MicroDataFrame({"income": [10.0, 20.0]}, index=[7, 7], weights=[2, 3])
+    series = mdf.MicroSeries(
+        [30.0, 40.0], index=[7, 3], name=series_name, weights=[5, 7]
+    )
+    parts = [series, frame] if series_first else [frame, series]
+    plain_parts = [
+        pd.Series(part) if part.ndim == 1 else pd.DataFrame(part) for part in parts
+    ]
+    expected = pd.concat(plain_parts, **options)
+    expected_weights = [5, 7, 2, 3] if series_first else [2, 3, 5, 7]
+
+    result = pd.concat(parts, **options)
+    assert isinstance(result, mdf.MicroDataFrame)
+    pd.testing.assert_frame_equal(pd.DataFrame(result), expected)
+    np.testing.assert_array_equal(result.weights, expected_weights)
+    assert result.weights.index.equals(result.index)
+    for column in expected:
+        assert result[column].sum() == np.nansum(
+            expected[column].to_numpy() * expected_weights
+        )
+
+    result.weights.iloc[0] = 100
+    np.testing.assert_array_equal(frame.weights, [2, 3])
+    np.testing.assert_array_equal(series.weights, [5, 7])
+    series.weights.iloc[-1] = 200
+    series_last_position = 1 if series_first else 3
+    assert result.weights.iloc[series_last_position] == 7
+
+
+@pytest.mark.parametrize("series_first", [False, True])
+@pytest.mark.parametrize("conflicting", [False, True])
+def test_concat_mixed_dimensions_columns_aligns_or_rejects_weights(
+    series_first, conflicting
+):
+    frame = mdf.MicroDataFrame({"income": [10.0, 20.0]}, index=[7, 3], weights=[2, 5])
+    series = mdf.MicroSeries(
+        [30.0, 40.0],
+        index=[3, 7],
+        name="other",
+        weights=[50, 2] if conflicting else [5, 2],
+    )
+    parts = [series, frame] if series_first else [frame, series]
+    if conflicting:
+        with pytest.raises(ValueError, match="weights"):
+            pd.concat(parts, axis=1)
+    else:
+        result = pd.concat(parts, axis=1)
+        np.testing.assert_array_equal(
+            result.weights, [5, 2] if series_first else [2, 5]
+        )
+        assert result.income.sum() == 120  # 10 * 2 + 20 * 5.
+        assert result.other.sum() == 230  # 30 * 5 + 40 * 2.
+
+
+@pytest.mark.parametrize("mapping", [False, True])
+def test_series_to_dataframe_weights_follow_aligned_rows_and_explicit_override(mapping):
+    series = mdf.MicroSeries([10.0, 20.0], index=[7, 3], name="income", weights=[2, 5])
+    data = {"income": series} if mapping else series
+    result = mdf.MicroDataFrame(data, index=[3, 7])
+    np.testing.assert_array_equal(result.weights, [5, 2])
+    assert result.income.sum() == 120  # 20 * 5 + 10 * 2.
+    result.weights.iloc[0] = 100
+    np.testing.assert_array_equal(series.weights, [2, 5])
+
+    explicit = mdf.MicroDataFrame(data, index=[3, 7], weights=[11, 13])
+    np.testing.assert_array_equal(explicit.weights, [11, 13])
+    assert explicit.income.sum() == 350  # 20 * 11 + 10 * 13.
+    with pytest.raises(ValueError, match="weights"):
+        mdf.MicroDataFrame(data, index=[3, 99])
+
+
+@pytest.mark.parametrize("method", ["cov", "corr"])
+@pytest.mark.parametrize("coincident_labels", [False, True])
+def test_dataframe_matrix_summaries_are_plain_and_unweighted(method, coincident_labels):
+    if coincident_labels:
+        frame = mdf.MicroDataFrame(
+            {"x": [10.0, 20.0], "y": [4.0, 8.0]},
+            index=["x", "y"],
+            weights=[2, 3],
+        )
+        # Sample covariance divides centered cross-products by n - 1.
+        covariance = [[50.0, 20.0], [20.0, 8.0]]
+        correlation = [[1.0, 1.0], [1.0, 1.0]]
+    else:
+        frame = mdf.MicroDataFrame(
+            {"x": [10.0, 20.0, 30.0], "y": [4.0, 8.0, 6.0]},
+            weights=[2, 3, 5],
+        )
+        # Centered x = [-10, 0, 10], y = [-2, 2, 0]; n - 1 = 2.
+        covariance = [[100.0, 10.0], [10.0, 4.0]]
+        correlation = [[1.0, 0.5], [0.5, 1.0]]
+    expected = pd.DataFrame(
+        covariance if method == "cov" else correlation,
+        index=frame.columns,
+        columns=frame.columns,
+    )
+
+    result = getattr(frame, method)()
+
+    assert type(result) is pd.DataFrame
+    pd.testing.assert_frame_equal(result, expected)
+    # Chaining a sum must not apply observation weights to column summaries.
+    pd.testing.assert_series_equal(result.sum(), expected.sum())
+    assert not hasattr(result, "weights")
+
+
+@pytest.mark.parametrize(
+    "method,args,kwargs",
+    [
+        ("cov", (), {}),
+        ("cov", (2, 0), {}),
+        ("cov", (), {"min_periods": 4, "ddof": 2}),
+        ("cov", (), {"min_periods": 2, "ddof": 0, "numeric_only": True}),
+        ("corr", (), {}),
+        ("corr", ("pearson", 2, True), {}),
+        ("corr", (), {"method": "spearman", "min_periods": 2}),
+        ("corr", (), {"min_periods": 4}),
+        ("corr", (), {"method": lambda x, y: np.dot(x, y), "min_periods": 2}),
+    ],
+)
+@pytest.mark.parametrize("missing", [False, True])
+def test_dataframe_matrix_summaries_preserve_pandas_arguments(
+    method, args, kwargs, missing
+):
+    data = {
+        "x": [10.0, 20.0, 30.0, 40.0],
+        "y": [4.0, 8.0, np.nan if missing else 6.0, 9.0],
+        "flag": [True, False, True, True],
+    }
+    frame = mdf.MicroDataFrame(data, index=[7, 7, 3, 9], weights=[2, 3, 5, 7])
+    expected = getattr(pd.DataFrame(data, index=frame.index), method)(*args, **kwargs)
+
+    result = getattr(frame, method)(*args, **kwargs)
+
+    assert type(result) is pd.DataFrame
+    pd.testing.assert_frame_equal(result, expected)
+    pd.testing.assert_series_equal(result.sum(), expected.sum())
+
+
+@pytest.mark.parametrize("method", ["cov", "corr"])
+def test_dataframe_matrix_summaries_preserve_numeric_only_and_errors(method):
+    data = {"x": [10.0, 20.0, 30.0], "y": [4.0, 8.0, 6.0], "label": ["a", "b", "c"]}
+    frame = mdf.MicroDataFrame(data, weights=[2, 3, 5])
+    plain = pd.DataFrame(data)
+    expected = getattr(plain, method)(numeric_only=True)
+
+    result = getattr(frame, method)(numeric_only=True)
+
+    assert type(result) is pd.DataFrame
+    pd.testing.assert_frame_equal(result, expected)
+    for kwargs in [{}, {"numeric_only": False}]:
+        with pytest.raises((TypeError, ValueError)) as pandas_error:
+            getattr(plain, method)(**kwargs)
+        with pytest.raises(type(pandas_error.value)) as microdf_error:
+            getattr(frame, method)(**kwargs)
+        assert str(microdf_error.value) == str(pandas_error.value)
+
+
+def test_dataframe_correlation_preserves_optional_kendall_support():
+    data = {"x": [10.0, 20.0, 30.0], "y": [4.0, 8.0, 6.0]}
+    frame = mdf.MicroDataFrame(data, weights=[2, 3, 5])
+    try:
+        expected = pd.DataFrame(data).corr(method="kendall")
+    except ImportError as pandas_error:
+        # Kendall requires scipy; delegation preserves pandas' dependency error.
+        with pytest.raises(type(pandas_error)) as microdf_error:
+            frame.corr(method="kendall")
+        assert str(microdf_error.value) == str(pandas_error)
+    else:
+        result = frame.corr(method="kendall")
+        assert type(result) is pd.DataFrame
+        pd.testing.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("method", ["cov", "corr"])
+def test_dataframe_matrix_summaries_preserve_pandas_metadata(method):
+    data = {"x": [10.0, 20.0, 30.0], "y": [4.0, 8.0, 6.0]}
+    frame = mdf.MicroDataFrame(data, weights=[2, 3, 5])
+    plain = pd.DataFrame(data)
+    for source in [frame, plain]:
+        source.attrs = {"survey": {"year": 2026}}
+        source.flags.allows_duplicate_labels = False
+        source.columns.name = "measure"
+    expected = getattr(plain, method)()
+
+    result = getattr(frame, method)()
+
+    assert type(result) is pd.DataFrame
+    pd.testing.assert_frame_equal(result, expected)
+    assert result.attrs == expected.attrs
+    result.attrs["survey"]["year"] = 2025
+    assert frame.attrs["survey"]["year"] == 2026

{microdf_python-1.4.0 → microdf_python-1.4.1/microdf_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: microdf-python
-Version: 1.4.0
+Version: 1.4.1
 Summary: Weighted pandas DataFrames and Series for survey microdata
 Author-email: Max Ghenis <max@policyengine.org>
 License: MIT

{microdf_python-1.4.0 → microdf_python-1.4.1}/microdf_python.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ LICENSE
 README.md
 pyproject.toml
 microdf/__init__.py
+microdf/_weights.py
 microdf/microdataframe.py
 microdf/microseries.py
 microdf/tests/conftest.py
@@ -14,6 +15,7 @@ microdf/tests/test_quantile_missing_values.py
 microdf/tests/test_serialization.py
 microdf/tests/test_sum_axes.py
 microdf/tests/test_version_metadata.py
+microdf/tests/test_weight_propagation.py
 microdf/tests/test_weighted_cov_corr.py
 microdf_python.egg-info/PKG-INFO
 microdf_python.egg-info/SOURCES.txt

{microdf_python-1.4.0 → microdf_python-1.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "microdf-python"
-version = "1.4.0"
+version = "1.4.1"
 description = "Weighted pandas DataFrames and Series for survey microdata"
 readme = "README.md"
 authors = [