microdf-python 1.2.3__py3-none-any.whl → 1.3.2__py3-none-any.whl

microdf/microdataframe.py

@@ -96,8 +96,10 @@ class _MicroILocIndexer:
 
 class MicroDataFrame(pd.DataFrame):
     def __init__(self, *args, weights=None, **kwargs):
-        """A DataFrame-inheriting class for weighted microdata. Weights can be
-        provided at initialisation, or using set_weights or set_weight_col.
+        """A DataFrame-inheriting class for weighted microdata.
+
+        Weights can be provided at initialisation, or using set_weights or
+        set_weight_col.
 
         :param weights: Array of weights.
         :type weights: np.array
@@ -231,8 +233,9 @@ class MicroDataFrame(pd.DataFrame):
         return fn
 
     def get_args_as_micro_series(*kwarg_names: tuple) -> Callable:
-        """Decorator for auto-parsing column names into MicroSeries objects. If
-        given, kwarg_names limits arguments checked to keyword arguments
+        """Decorator for auto-parsing column names into MicroSeries objects.
+
+        If given, kwarg_names limits arguments checked to keyword arguments
         specified.
 
         :param arg_names: argument names to restrict to.
@@ -292,8 +295,10 @@ class MicroDataFrame(pd.DataFrame):
         weights: Union[np.ndarray, str],
         preserve_old: Optional[bool] = False,
     ) -> None:
-        """Sets the weights for the MicroDataFrame. If a string is received, it
-        will be assumed to be the column name of the weight column.
+        """Sets the weights for the MicroDataFrame.
+
+        If a string is received, it will be assumed to be the column name of
+        the weight column.
 
         :param weights: Array of weights.
         :param preserve_old: If True, keeps the old weights as a column when
@@ -305,7 +310,11 @@ class MicroDataFrame(pd.DataFrame):
 
         if isinstance(weights, str):
             self.weights_col = weights
-            self.weights = pd.Series(self[weights], dtype=float)
+            self.weights = pd.Series(
+                np.asarray(self[weights]),
+                index=self.index,
+                dtype=float,
+            )
             self._link_all_weights()
         elif weights is not None:
             if len(weights) != len(self):
@@ -314,9 +323,18 @@ class MicroDataFrame(pd.DataFrame):
                     f"length of DataFrame ({len(self)})."
                 )
             self.weights_col = None
+            # Align weights to self.index. Without this, weighted ops
+            # (self[col].multiply(self.weights) in .sum()) align on
+            # label, so any non-default index silently produces all-NaN
+            # and aggregations collapse to 0. If a Series is passed in,
+            # strip its index so we position-align to self.index.
+            if isinstance(weights, pd.Series):
+                weights = weights.values
             with warnings.catch_warnings():
                 warnings.filterwarnings("ignore", category=UserWarning)
-                self.weights = pd.Series(weights, dtype=float)
+                self.weights = pd.Series(
+                    np.asarray(weights), index=self.index, dtype=float
+                )
             self._link_all_weights()
 
     def set_weight_col(self, column: str, preserve_old: Optional[bool] = False) -> None:
@@ -416,8 +434,9 @@ class MicroDataFrame(pd.DataFrame):
         :return: MicroDataFrame with reset index or None if inplace=True.
         """
         if inplace:
-            weights_backup = self.weights.copy()
-            # Perform in-place reset on the parent DataFrame
+            # Snapshot weight *values* positionally — the index is about
+            # to change and reset_index preserves row order.
+            weight_values = np.asarray(self.weights.values, dtype=float)
             super().reset_index(
                 level=level,
                 drop=drop,
@@ -427,7 +446,7 @@ class MicroDataFrame(pd.DataFrame):
                 allow_duplicates=allow_duplicates,
                 names=names,
             )
-            self.weights = weights_backup
+            self.weights = pd.Series(weight_values, index=self.index, dtype=float)
             self._link_all_weights()
             return None
         else:
@@ -440,13 +459,22 @@ class MicroDataFrame(pd.DataFrame):
                 allow_duplicates=allow_duplicates,
                 names=names,
             )
-            return MicroDataFrame(res, weights=self.weights)
+            out = MicroDataFrame(res, weights=self.weights.values)
+            # Ensure weights align to res.index (reset_index changes the
+            # index but preserves row order, so pass values positionally).
+            out.weights = pd.Series(
+                np.asarray(self.weights.values, dtype=float),
+                index=out.index,
+                dtype=float,
+            )
+            return out
 
     def copy(self, deep: Optional[bool] = True) -> "MicroDataFrame":
         res = super().copy(deep)
-        # This changes the original columns to Series. Undo it:
-        for col in self.columns:
-            self[col] = MicroSeries(self[col])
+        # super().copy() corrupts self's column types to plain Series.
+        # Restore them in O(N) instead of O(N²) by calling
+        # _link_all_weights once rather than per-column __setitem__.
+        self._link_all_weights()
         res = MicroDataFrame(res, weights=self.weights.copy(deep))
         return res
 
@@ -480,8 +508,11 @@ class MicroDataFrame(pd.DataFrame):
             dropped.
         :return: MicroDataFrame or None if inplace=True.
         """
+        row_drop = axis in (0, "index") or index is not None
         if inplace:
-            weights_backup = self.weights.copy()
+            # Snapshot the pre-drop weights keyed by the pre-drop index so
+            # we can reindex to the surviving rows after the drop.
+            pre_drop_weights = pd.Series(self.weights.values, index=self.index.copy())
             # Perform in-place drop on the parent DataFrame
             super().drop(
                 labels=labels,
@@ -492,7 +523,15 @@ class MicroDataFrame(pd.DataFrame):
                 inplace=True,
                 errors=errors,
             )
-            self.weights = weights_backup
+            if row_drop:
+                surviving = pre_drop_weights.reindex(self.index)
+                self.weights = pd.Series(
+                    surviving.values, index=self.index, dtype=float
+                )
+            else:
+                self.weights = pd.Series(
+                    pre_drop_weights.values, index=self.index, dtype=float
+                )
             self._link_all_weights()
             return None
         else:
@@ -505,7 +544,19 @@ class MicroDataFrame(pd.DataFrame):
                 inplace=False,
                 errors=errors,
             )
-            return MicroDataFrame(res, weights=self.weights)
+            if row_drop:
+                # Row drop: keep only the weights for surviving rows,
+                # in the order of the resulting DataFrame.
+                pre_drop_weights = pd.Series(self.weights.values, index=self.index)
+                new_weights = pre_drop_weights.reindex(res.index).values
+            else:
+                new_weights = self.weights.values
+            out = MicroDataFrame(res, weights=new_weights)
+            # Guard against the set_weights path building weights with a
+            # default RangeIndex, which would misalign against res.index
+            # and silently zero weighted aggregations.
+            out.weights = pd.Series(new_weights, index=out.index, dtype=float)
+            return out
 
     def merge(
         self,
@@ -548,7 +599,17 @@ class MicroDataFrame(pd.DataFrame):
         :param validate: If specified, checks if merge is of specified type.
         :return: MicroDataFrame with merged data.
         """
-        res = super().merge(
+        # Attach the left weights as a temporary column so pandas' merge
+        # propagates them onto every surviving output row (including
+        # many-to-many row duplications, inner-join filtering, and
+        # left-with-missing NaNs). We then strip the column back off.
+        tmp = "__microdf_weights__"
+        # Avoid clobbering if this exact name is already used.
+        while tmp in self.columns or tmp in right.columns:
+            tmp += "_"
+        left_df = pd.DataFrame(self).copy()
+        left_df[tmp] = np.asarray(self.weights.values, dtype=float)
+        res = left_df.merge(
             right,
             how=how,
             on=on,
@@ -562,11 +623,17 @@ class MicroDataFrame(pd.DataFrame):
             indicator=indicator,
             validate=validate,
         )
-
-        # For inner join, both dataframes must have the same weights on
-        # matching rows. For now, we'll use the left dataframe's weights.
-        # This is a simplification and may need more sophisticated handling
-        return MicroDataFrame(res, weights=self.weights)
+        # Pull out the propagated weights. Rows with no left match in a
+        # right/outer join get NaN weight — fill with 0 so they don't
+        # poison later aggregations (a user who needs a different
+        # convention can override afterwards).
+        merged_weights = res[tmp].fillna(0).to_numpy(dtype=float)
+        res = res.drop(columns=[tmp])
+        out = MicroDataFrame(res, weights=merged_weights)
+        # Ensure the weights Series aligns with res.index regardless of
+        # the default-RangeIndex behavior of set_weights.
+        out.weights = pd.Series(merged_weights, index=out.index, dtype=float)
+        return out
 
     def __getattr__(self, name):
         """Allow accessing columns as attributes (e.g., df.column_name).
@@ -598,10 +665,16 @@ class MicroDataFrame(pd.DataFrame):
         return: DataFrameGroupBy object with columns using weights
         rtype: DataFrameGroupBy
         """
-        self["__tmp_weights"] = self.weights
-        gb = super().groupby(by, *args, **kwargs)
+        # Build the groupby on a *copy* that carries a ``__tmp_weights``
+        # column. We used to set this column on ``self`` directly, which
+        # permanently leaked the weight column onto the caller's
+        # DataFrame — any later ``df.sum()`` or ``list(df.columns)``
+        # would then include it.
+        staged = pd.DataFrame(self).copy()
+        staged["__tmp_weights"] = np.asarray(self.weights.values, dtype=float)
+        gb = staged.groupby(by, *args, **kwargs)
         weights = copy.deepcopy(gb["__tmp_weights"])
-        for col in self.columns:  # df.groupby(...)[col]s use weights
+        for col in staged.columns:  # df.groupby(...)[col]s use weights
             res = gb[col]
             res.__class__ = MicroSeriesGroupBy
             res._init()

microdf/microseries.py

@@ -9,10 +9,51 @@ import pandas as pd
 logger = logging.getLogger(__name__)
 
 
+def _weighted_top_share(
+    values: np.ndarray, weights: np.ndarray, top_x_pct: float
+) -> float:
+    """Share of the sum held by the top ``top_x_pct`` of weight.
+
+    Sort by value ascending, cumulate the weight, pick the slice from the top
+    that covers exactly ``top_x_pct`` of total weight, and distribute the tied-
+    at-cutoff row proportionally so constant values return exactly
+    ``top_x_pct`` rather than 1.0.
+    """
+    if top_x_pct <= 0:
+        return 0.0
+    if top_x_pct >= 1:
+        return 1.0
+    total_weight = weights.sum()
+    total_sum = float((values * weights).sum())
+    if total_weight == 0 or total_sum == 0:
+        return np.nan
+    # Ascending sort; the "top" cutoff is the final ``top_x_pct`` of
+    # cumulative weight.
+    order = np.argsort(values, kind="mergesort")
+    v = values[order]
+    w = weights[order]
+    # Cumulative weight from the bottom up.
+    cum_w = np.cumsum(w)
+    target_bottom_weight = total_weight * (1.0 - top_x_pct)
+    # searchsorted(cum_w, target, side="right") gives the first index
+    # whose cumulative weight exceeds the bottom cutoff.
+    k = int(np.searchsorted(cum_w, target_bottom_weight, side="right"))
+    # Rows strictly above the cutoff contribute all of their weight.
+    if k >= len(v):
+        return 0.0
+    top_sum = float((v[k + 1 :] * w[k + 1 :]).sum())
+    # Row k straddles the cutoff; include the fraction of its weight
+    # that lies above the cutoff so ties don't double-count.
+    partial_weight = cum_w[k] - target_bottom_weight
+    top_sum += float(v[k] * partial_weight)
+    return top_sum / total_sum
+
+
 class MicroSeries(pd.Series):
     def __init__(self, *args, weights: np.array = None, **kwargs):
-        """A Series-inheriting class for weighted microdata. Weights can be
-        provided at initialisation, or using set_weights.
+        """A Series-inheriting class for weighted microdata.
+
+        Weights can be provided at initialisation, or using set_weights.
 
         :param weights: Array of weights.
         :type weights: np.array
@@ -64,23 +105,13 @@ class MicroSeries(pd.Series):
         )
         return super().to_numpy(*args, **kwargs)
 
-    def weighted_function(fn: Callable) -> Callable:
-        @wraps(fn)
-        def safe_fn(*args, **kwargs):
-            try:
-                return fn(*args, **kwargs)
-            except ZeroDivisionError:
-                return np.NaN
-
-        return safe_fn
-
-    @weighted_function
     def scalar_function(fn: Callable) -> Callable:
+        """Decorator marking ``fn`` as returning a scalar (float)."""
         fn._rtype = float
         return fn
 
-    @weighted_function
     def vector_function(fn: Callable) -> Callable:
+        """Decorator marking ``fn`` as returning a pandas Series."""
         fn._rtype = pd.Series
         return fn
 
@@ -96,7 +127,11 @@ class MicroSeries(pd.Series):
         """
         if weights is None:
             if len(self) > 0:
-                self.weights = pd.Series(np.ones_like(self._values), dtype=float)
+                self.weights = pd.Series(
+                    np.ones_like(self._values),
+                    index=self.index,
+                    dtype=float,
+                )
         else:
             if len(weights) != len(self):
                 raise ValueError(
@@ -107,7 +142,14 @@ class MicroSeries(pd.Series):
             if preserve_old and self.weights is not None:
                 self["old_weights"] = self.weights
 
-            self.weights = pd.Series(weights, dtype=float)
+            # Align weights to self.index so element-wise operations such
+            # as self.multiply(self.weights) (used by .sum(), .weight())
+            # don't silently produce all-NaN when the caller uses a
+            # non-default index. If a pandas Series is passed in, strip
+            # its index first so we position-align rather than label-align.
+            if isinstance(weights, pd.Series):
+                weights = weights.values
+            self.weights = pd.Series(np.asarray(weights), index=self.index, dtype=float)
 
     def nullify_weights(self) -> None:
         """Set all weights to 1, effectively making the Series unweighted.
@@ -136,12 +178,22 @@ class MicroSeries(pd.Series):
         return self.multiply(self.weights).sum()
 
     @scalar_function
-    def count(self) -> float:
+    def count(self, skipna: bool = True) -> float:
         """Calculates the weighted count of the MicroSeries.
 
+        By default skips NaN values (matching pandas ``Series.count``).
+
+        :param skipna: Exclude NaN values (default True). If False, the
+            weighted count of every row is returned.
+        :type skipna: bool
         :returns: The weighted count.
+        :rtype: float
         """
-        return self.weights.sum()
+        weights = np.asarray(self.weights.values, dtype=float)
+        if not skipna:
+            return float(weights.sum())
+        mask = ~pd.isna(self._values)
+        return float(weights[mask].sum())
 
     @scalar_function
     def mean(self, skipna: bool = True) -> float:
@@ -171,6 +223,89 @@ class MicroSeries(pd.Series):
 
         return np.average(values, weights=weights)
 
+    def _weighted_variance(self, ddof: int = 1, skipna: bool = True) -> float:
+        """Frequency-weighted variance.
+
+        Uses ``sum(w * (x - wmean)**2) / (sum(w) - ddof)``. With
+        ``ddof=0`` this is the population variance; with ``ddof=1`` it
+        is Bessel-corrected assuming the weights are frequency counts —
+        matching ``np.var(..., ddof=ddof)`` on a replicated sample.
+        """
+        values = np.asarray(self._values, dtype=float)
+        weights = np.asarray(self.weights.values, dtype=float)
+        if skipna:
+            mask = ~np.isnan(values)
+            values = values[mask]
+            weights = weights[mask]
+        elif np.isnan(values).any():
+            return np.nan
+        total_w = weights.sum()
+        if total_w == 0 or total_w - ddof <= 0:
+            return np.nan
+        mean = np.average(values, weights=weights)
+        return float((weights * (values - mean) ** 2).sum() / (total_w - ddof))
+
+    @scalar_function
+    def var(self, ddof: int = 1, skipna: bool = True) -> float:
+        """Calculates the weighted variance of the MicroSeries.
+
+        Treats weights as frequency counts (``sum(w) - ddof`` in the
+        denominator) so that with integer weights the result matches
+        ``np.var`` on the replicated sample.
+
+        :param ddof: Delta degrees of freedom (default 1).
+        :param skipna: Exclude NaN values (default True).
+        :returns: The weighted variance.
+        :rtype: float
+        """
+        return self._weighted_variance(ddof=ddof, skipna=skipna)
+
+    @scalar_function
+    def std(self, ddof: int = 1, skipna: bool = True) -> float:
+        """Calculates the weighted standard deviation of the MicroSeries.
+
+        :param ddof: Delta degrees of freedom (default 1).
+        :param skipna: Exclude NaN values (default True).
+        :returns: The weighted standard deviation.
+        :rtype: float
+        """
+        v = self._weighted_variance(ddof=ddof, skipna=skipna)
+        return float(np.sqrt(v)) if np.isfinite(v) else v
+
+    def cov(self, other, *args, **kwargs):
+        """Pandas ``cov`` — **unweighted**.
+
+        MicroSeries does not yet compute weighted covariance. Emits a
+        ``UserWarning`` so callers aren't silently given an unweighted number
+        after ``.sum()`` and ``.mean()`` worked as expected. See issue tracker
+        for a weighted implementation.
+        """
+        warnings.warn(
+            "MicroSeries.cov() falls through to pandas and is "
+            "unweighted. Use MicroSeries.var()/std() for weighted "
+            "second moments, or compute covariance manually with the "
+            "weights.",
+            UserWarning,
+            stacklevel=2,
+        )
+        return super().cov(other, *args, **kwargs)
+
+    def corr(self, other, *args, **kwargs):
+        """Pandas ``corr`` — **unweighted**.
+
+        MicroSeries does not yet compute weighted correlation. Emits a
+        ``UserWarning`` so callers aren't silently given an unweighted number.
+        See issue tracker for a weighted implementation.
+        """
+        warnings.warn(
+            "MicroSeries.corr() falls through to pandas and is "
+            "unweighted. Compute correlation manually with the weights "
+            "if you need the survey-weighted value.",
+            UserWarning,
+            stacklevel=2,
+        )
+        return super().corr(other, *args, **kwargs)
+
     def quantile(self, q: np.array) -> pd.Series:
         """Calculates weighted quantiles of the MicroSeries.
 
@@ -190,6 +325,20 @@ class MicroSeries(pd.Series):
         assert np.all(quantiles >= 0) and np.all(quantiles <= 1), (
             "quantiles should be in [0, 1]"
         )
+        # Drop zero-weight rows before sorting. Without this, q=0 (and
+        # internal plateaus of zero weight) picked a value with 0 weight
+        # that should have been skipped by the inverse CDF. E.g.
+        # MicroSeries([10, 20, 30], weights=[0, 1, 1]).quantile(0)
+        # returned 10 instead of 20.
+        nonzero = sample_weight > 0
+        if not nonzero.any():
+            return (
+                np.nan
+                if np.array(q).shape == ()
+                else pd.Series(np.full(len(quantiles), np.nan), index=quantiles)
+            )
+        values = values[nonzero]
+        sample_weight = sample_weight[nonzero]
         sorter = np.argsort(values)
         values = values[sorter]
         sample_weight = sample_weight[sorter]
@@ -218,58 +367,83 @@ class MicroSeries(pd.Series):
     def gini(self, negatives: Optional[str] = None) -> float:
         """Calculates Gini index.
 
-        :param negatives: An optional string indicating how to treat negative
-            values of x:
+        :param negatives: An optional string indicating how to treat
+            negative values of x:
             'zero' replaces negative values with zeroes.
             'shift' subtracts the minimum value from all values of x,
             when this minimum is negative. That is, it adds the absolute
             minimum value.
             Defaults to None, which leaves negative values as they are.
-        :type q: str
+        :type negatives: str
         :returns: Gini index.
         :rtype: float
         """
         x = np.array(self).astype("float")
+        w = np.asarray(self.weights.values, dtype=float)
         if negatives == "zero":
-            x[x < 0] = 0
-        if negatives == "shift" and np.amin(x) < 0:
-            x -= np.amin(x)
-        if (self.weights != np.ones(len(self))).any():  # Varying weights.
-            sorted_indices = np.argsort(self)
-            sorted_x = np.array(self[sorted_indices])
-            sorted_w = np.array(self.weights[sorted_indices])
-            cumw = np.cumsum(sorted_w)
-            cumxw = np.cumsum(sorted_x * sorted_w)
-            return np.sum(cumxw[1:] * cumw[:-1] - cumxw[:-1] * cumw[1:]) / (
-                cumxw[-1] * cumw[-1]
+            x = np.where(x < 0, 0.0, x)
+        elif negatives == "shift" and len(x) > 0 and np.amin(x) < 0:
+            x = x - np.amin(x)
+        elif negatives is not None:
+            raise ValueError(
+                f"Unknown negatives option {negatives!r}; expected "
+                "'zero', 'shift', or None."
             )
-        else:
-            sorted_x = np.sort(self)
-            n = len(x)
-            cumxw = np.cumsum(sorted_x)
-            # The above formula, with all weights equal to 1 simplifies to:
-            return (n + 1 - 2 * np.sum(cumxw) / cumxw[-1]) / n
+
+        if len(x) == 0:
+            return np.nan
+        if np.any(x < 0):
+            # The Lorenz-based formula assumes non-negative values; with
+            # negatives it can return values outside [0, 1].
+            warnings.warn(
+                "gini() called on data containing negative values; the "
+                "result is not guaranteed to lie in [0, 1]. Pass "
+                "negatives='zero' or negatives='shift' to handle them.",
+                UserWarning,
+                stacklevel=2,
+            )
+
+        # Short-circuit degenerate cases so we don't divide by zero.
+        total = float((x * w).sum())
+        if total == 0:
+            return 0.0
+
+        sorter = np.argsort(x, kind="mergesort")
+        sorted_x = x[sorter]
+        sorted_w = w[sorter]
+        cumw = np.cumsum(sorted_w)
+        cumxw = np.cumsum(sorted_x * sorted_w)
+        # Trapezoidal approximation of the area under the Lorenz curve.
+        return float(
+            np.sum(cumxw[1:] * cumw[:-1] - cumxw[:-1] * cumw[1:])
+            / (cumxw[-1] * cumw[-1])
+        )
 
     @scalar_function
     def top_x_pct_share(self, top_x_pct: float) -> float:
         """Calculates top x% share.
 
+        Uses a cumulative-weight sort so that rows tied at the cutoff
+        contribute proportionally rather than all-or-nothing. With
+        constant values this correctly returns ``top_x_pct`` itself.
+
         :param top_x_pct: Decimal between 0 and 1 of the top %, e.g. 0.1,
             0.001.
         :type top_x_pct: float
         :returns: The weighted share held by the top x%.
         :rtype: float
         """
-        threshold = self.quantile(1 - top_x_pct)
-        top_x_pct_sum = self[self >= threshold].sum()
-        total_sum = self.sum()
-        return top_x_pct_sum / total_sum
+        return _weighted_top_share(
+            np.asarray(self._values, dtype=float),
+            np.asarray(self.weights.values, dtype=float),
+            float(top_x_pct),
+        )
 
     @scalar_function
     def bottom_x_pct_share(self, bottom_x_pct: float) -> float:
         """Calculates bottom x% share.
 
-        :param bottom_x_pct: Decimal between 0 and 1 of the top %, e.g. 0.1,
+        :param bottom_x_pct: Decimal between 0 and 1 of the bottom %, e.g. 0.1,
             0.001.
         :type bottom_x_pct: float
         :returns: The weighted share held by the bottom x%.
@@ -344,19 +518,44 @@ class MicroSeries(pd.Series):
 
     @vector_function
     def rank(self, pct: Optional[bool] = False) -> pd.Series:
-        weights_sum = self.weights.values.sum()
+        """Weighted rank of each element.
+
+        Each element's rank is the cumulative weight of all values that are
+        less than or equal to it. Tied values therefore share the same rank, so
+        downstream bucketing (``decile_rank``, ``quintile_rank``, etc.) lands
+        tied rows in the same bucket.
+
+        :param pct: If True, divide ranks by the total weight so they lie in
+            ``(0, 1]``.
+        :type pct: bool
+        :returns: MicroSeries of ranks aligned to ``self``.
+        :rtype: MicroSeries
+        """
+        weights_sum = np.asarray(self.weights.values, dtype=float).sum()
         if weights_sum == 0:
             raise ZeroDivisionError(
                 "Cannot calculate rank with zero total weight. "
-                "All weights in the MicroSeries are zero, which would result "
-                "in division by zero."
+                "All weights in the MicroSeries are zero, which would "
+                "result in division by zero."
             )
 
-        order = np.argsort(self._values)
-        inverse_order = np.argsort(order)
-        ranks = np.array(self.weights.values)[order].cumsum()[inverse_order]
+        values = np.asarray(self._values)
+        weights = np.asarray(self.weights.values, dtype=float)
+        order = np.argsort(values, kind="mergesort")
+        sorted_values = values[order]
+        sorted_weights = weights[order]
+        cum_w = np.cumsum(sorted_weights)
+        # Max rank semantics: every tied group gets the cumulative
+        # weight at the *end* of the group, so ties share one rank.
+        # searchsorted(side='right') on the sorted values finds the
+        # index just past each tied block in sort order.
+        group_end = np.searchsorted(sorted_values, sorted_values, side="right") - 1
+        sorted_ranks = cum_w[group_end]
+        # Invert the sort to put ranks back into the caller's order.
+        inverse_order = np.argsort(order, kind="mergesort")
+        ranks = sorted_ranks[inverse_order]
         if pct:
-            ranks /= weights_sum
+            ranks = ranks / weights_sum
             ranks = np.where(ranks > 1.0, 1.0, ranks)
         return MicroSeries(ranks, index=self.index, weights=self.weights)
 

microdf/tests/test_microseries_dataframe.py

@@ -2,6 +2,7 @@ import warnings
 
 import numpy as np
 import pandas as pd
+import pytest
 
 import microdf as mdf
 from microdf.microdataframe import MicroDataFrame
@@ -445,6 +446,43 @@ def test_mean_no_warning() -> None:
         assert len(user_warnings) == 0
 
 
+def test_sum_with_non_default_index() -> None:
+    """Weighted sum must not silently return 0 with a non-default index.
+
+    Regression test for the bug where ``set_weights`` stored the weights Series
+    with a default ``RangeIndex`` regardless of ``self.index``. Element-wise
+    ops like ``self.multiply(self.weights)`` then aligned on label, producing
+    all-NaN and a silent ``0.0`` from ``.sum()`` while ``.mean()`` (which uses
+    a positional ndarray) stayed correct.
+    """
+    # MicroSeries with custom integer index.
+    s = mdf.MicroSeries([1, 2, 3], index=[100, 200, 300], weights=[10, 20, 30])
+    assert s.sum() == 140.0
+    assert s.weights.index.tolist() == [100, 200, 300]
+
+    # MicroDataFrame with custom integer index.
+    df = mdf.MicroDataFrame(
+        {"x": [1, 2, 3]}, index=[100, 200, 300], weights=[10, 20, 30]
+    )
+    assert df.x.sum() == 140.0
+    assert df.weights.index.tolist() == [100, 200, 300]
+
+    # MicroDataFrame with string index + set_weights via column name.
+    df2 = mdf.MicroDataFrame(
+        {"x": [1, 2, 3], "w": [10, 20, 30]},
+        index=["a", "b", "c"],
+    )
+    df2.set_weights("w")
+    assert df2.x.sum() == 140.0
+    assert df2.weights.index.tolist() == ["a", "b", "c"]
+
+    # Passing a Series with its own index should position-align, not
+    # label-align, so sum does not depend on accidental index alignment.
+    df3 = mdf.MicroDataFrame({"x": [1, 2, 3]}, index=[100, 200, 300])
+    df3.set_weights(pd.Series([10, 20, 30], index=[0, 1, 2]))
+    assert df3.x.sum() == 140.0
+
+
 def test_repr_no_warning() -> None:
     """Internal .values usage in __repr__ should NOT emit a warning."""
     ms = mdf.MicroSeries([1, 2, 3], weights=[4, 5, 6])
@@ -453,3 +491,327 @@ def test_repr_no_warning() -> None:
         _ = repr(ms)
         user_warnings = [x for x in w if issubclass(x.category, UserWarning)]
         assert len(user_warnings) == 0
+
+
+def test_drop_inplace_aligns_weights() -> None:
+    """Regression: ``drop(inplace=True)`` must keep weights in sync.
+
+    Previously, ``weights_backup = self.weights.copy()`` was taken *before*
+    the drop, then reassigned back afterwards — so the weights vector
+    kept its original length and any subsequent weighted op either
+    raised a length-mismatch ValueError or silently returned 0.
+    """
+    # Row drop inplace.
+    df = mdf.MicroDataFrame({"x": [1, 2, 3, 4]}, weights=[10, 20, 30, 40])
+    df.drop(index=[0, 1], inplace=True)
+    assert len(df) == len(df.weights) == 2
+    assert df.x.sum() == 3 * 30 + 4 * 40  # 250
+
+    # Row drop non-inplace.
+    df = mdf.MicroDataFrame({"x": [1, 2, 3, 4]}, weights=[10, 20, 30, 40])
+    df2 = df.drop(index=[0, 1])
+    assert len(df2) == len(df2.weights) == 2
+    assert df2.x.sum() == 250
+    # Original untouched.
+    assert len(df) == 4
+    assert df.x.sum() == 1 * 10 + 2 * 20 + 3 * 30 + 4 * 40
+
+    # Column drop (weights length unchanged).
+    df = mdf.MicroDataFrame({"x": [1, 2, 3], "y": [4, 5, 6]}, weights=[10, 20, 30])
+    df.drop(columns=["y"], inplace=True)
+    assert list(df.columns) == ["x"]
+    assert df.x.sum() == 1 * 10 + 2 * 20 + 3 * 30
+
+    # String index row drop.
+    df = mdf.MicroDataFrame(
+        {"x": [1, 2, 3, 4]},
+        index=["a", "b", "c", "d"],
+        weights=[10, 20, 30, 40],
+    )
+    df.drop(index=["a", "b"], inplace=True)
+    assert df.x.sum() == 250
+
+    # labels= with default axis=0.
+    df = mdf.MicroDataFrame({"x": [1, 2, 3, 4]}, weights=[10, 20, 30, 40])
+    df.drop(labels=[0, 1], inplace=True)
+    assert df.x.sum() == 250
+
+
+def test_merge_preserves_weights_per_surviving_row() -> None:
+    """Regression: merge must propagate weights onto the merged rows.
+
+    Previously the implementation passed ``self.weights`` straight to the
+    MicroDataFrame constructor, so any merge that changed row count (inner
+    filtering, left-with-missing, many-to-many, outer) raised ``ValueError:
+    Length of weights (N) does not match length of DataFrame (M)``.
+    """
+    # Inner join filters rows.
+    left = mdf.MicroDataFrame(
+        {"k": [1, 2, 3, 4], "v": [10, 20, 30, 40]}, weights=[1, 2, 3, 4]
+    )
+    right = pd.DataFrame({"k": [2, 4], "w": [20, 40]})
+    res = left.merge(right, on="k")
+    assert len(res) == 2
+    # k=2 carries weight 2; k=4 carries weight 4.
+    np.testing.assert_array_equal(sorted(res.weights.values), [2.0, 4.0])
+    assert res.v.sum() == 2 * 20 + 4 * 40
+
+    # Left join with missing from right.
+    left = mdf.MicroDataFrame(
+        {"k": [1, 2, 3, 4], "v": [10, 20, 30, 40]}, weights=[1, 2, 3, 4]
+    )
+    right = pd.DataFrame({"k": [2, 4], "w": [100, 200]})
+    res = left.merge(right, on="k", how="left")
+    assert len(res) == 4
+    assert res.v.sum() == 300  # 1*10 + 2*20 + 3*30 + 4*40
+
+    # Many-to-many duplicates left rows; the same weight should ride
+    # along on each duplicate.
+    left = mdf.MicroDataFrame({"k": [1, 2], "v": [10, 20]}, weights=[5, 7])
+    right = pd.DataFrame({"k": [1, 1, 2], "w": [100, 200, 300]})
+    res = left.merge(right, on="k")
+    assert len(res) == 3
+    # v=10 weighted 5 appears twice, v=20 weighted 7 appears once.
+    assert res.v.sum() == 10 * 5 + 10 * 5 + 20 * 7
+
+    # Outer join: right-only rows have no left weight. We default to 0
+    # so they don't silently poison downstream aggregations.
+    left = mdf.MicroDataFrame({"k": [1, 2], "v": [10, 20]}, weights=[1, 2])
+    right = pd.DataFrame({"k": [2, 3], "w": [20, 30]})
+    res = left.merge(right, on="k", how="outer")
+    # k=1 -> weight 1, k=2 -> weight 2, k=3 -> weight 0 (right-only).
+    assert sorted(res.weights.values) == [0.0, 1.0, 2.0]
+
+
+def test_groupby_does_not_leak_tmp_weights_column() -> None:
+    """Regression: groupby used to mutate self by adding __tmp_weights.
+
+    Previously, ``MicroDataFrame.groupby`` set ``self["__tmp_weights"]`` and
+    never cleaned it up, so ``df.columns`` afterwards included the weight
+    column and any later ``df.sum()`` or iteration over columns picked it up as
+    data.
+    """
+    df = mdf.MicroDataFrame({"g": ["a", "a", "b"], "v": [1, 2, 3]}, weights=[1, 2, 3])
+    original_cols = list(df.columns)
+    _ = df.groupby("g").sum()
+    assert list(df.columns) == original_cols
+    assert "__tmp_weights" not in df.columns
+
+    # Groupby by a list of columns should also not leak.
+    df2 = mdf.MicroDataFrame(
+        {"g1": ["a", "a", "b"], "g2": [1, 1, 2], "v": [1, 2, 3]},
+        weights=[1, 2, 3],
+    )
+    orig2 = list(df2.columns)
+    _ = df2.groupby(["g1", "g2"]).v.sum()
+    assert list(df2.columns) == orig2
+
+    # Weighted aggregation is still correct after the fix.
+    result = df.groupby("g").v.sum()
+    assert result["a"] == 1 * 1 + 2 * 2
+    assert result["b"] == 3 * 3
+
+
+def test_quantile_skips_zero_weight_rows() -> None:
+    """Regression: quantile(0) shouldn't pick a zero-weight element.
+
+    Previously, ``np.searchsorted(cumsum_norm, 0, side='left')`` returned 0
+    even when that first sorted element had zero weight, so ``MicroSeries([10,
+    20, 30], weights=[0, 1, 1]).quantile(0)`` returned 10 instead of 20. The
+    fix drops zero-weight rows before computing the CDF.
+    """
+    s = mdf.MicroSeries([10, 20, 30], weights=[0, 1, 1])
+    assert s.quantile(0.0) == 20
+    assert s.quantile(0.5) == 20
+    assert s.quantile(1.0) == 30
+
+    # Internal plateau of zero weight.
+    s = mdf.MicroSeries([10, 20, 30, 40], weights=[1, 0, 1, 1])
+    assert s.quantile(0.0) == 10
+    # Post-filter values [10, 30, 40] with equal weights -> cum=[.33,.67,1].
+    # 0.4 -> smallest cum >= 0.4 is index 1 -> value 30.
+    assert s.quantile(0.4) == 30
+    # The zero-weight value (20) should never be selected.
+    for q in np.linspace(0, 1, 21):
+        assert s.quantile(q) != 20
+
+    # All zero weights -> NaN (defined behaviour).
+    s = mdf.MicroSeries([10, 20, 30], weights=[0, 0, 0])
+    assert np.isnan(s.quantile(0.5))
+
+
+def test_top_x_pct_share_handles_ties_and_edges() -> None:
+    """Regression: top_x_pct_share double-counted threshold ties.
+
+    Old implementation: ``self[self >= threshold].sum() / self.sum()``.
+    With constant values every call returned 1.0 regardless of the
+    requested top percent; ``top_x_pct_share(0)`` returned the share of
+    the max bucket instead of 0.
+    """
+    # Constant values: the top p% should hold exactly p% of the total.
+    for p in [0.0, 0.01, 0.1, 0.5, 1.0]:
+        got = mdf.MicroSeries([5] * 10, weights=[1] * 10).top_x_pct_share(p)
+        assert np.isclose(got, p), f"top={p}, got {got}"
+
+    # Non-constant, equal weights.
+    s = mdf.MicroSeries(list(range(1, 11)), weights=[1] * 10)
+    # Sum 1..10 = 55. Top 10% = top 1 row = 10 -> 10/55.
+    assert np.isclose(s.top_x_pct_share(0.1), 10 / 55)
+    # Top 50% = rows 6..10 -> 40/55.
+    assert np.isclose(s.top_x_pct_share(0.5), 40 / 55)
+    # Top 0% = 0, top 100% = 1.
+    assert s.top_x_pct_share(0.0) == 0.0
+    assert s.top_x_pct_share(1.0) == 1.0
+
+    # Bottom share complements the top share.
+    assert np.isclose(s.bottom_x_pct_share(0.1), 1 - s.top_x_pct_share(0.9))
+
+    # Ties with unequal totals.
+    s_ties = mdf.MicroSeries([1, 1, 10, 10], weights=[1, 1, 1, 1])
+    # Top 50% = the two 10s -> 20/22.
+    assert np.isclose(s_ties.top_x_pct_share(0.5), 20 / 22)
+
+    # Downstream helpers still work.
+    assert np.isclose(s_ties.top_10_pct_share(), s_ties.top_x_pct_share(0.1))
+    assert np.isclose(s_ties.top_50_pct_share(), s_ties.top_x_pct_share(0.5))
+
+
+def test_gini_negatives_option_applied() -> None:
+    """Regression: gini(negatives=...) was silently ignored.
+
+    Both branches of the old implementation sorted ``self`` directly rather
+    than the local ``x`` that was mutated by the ``negatives`` option, so
+    ``negatives='zero'`` and ``negatives='shift'`` did nothing.
+    """
+    s = mdf.MicroSeries([-5, 0, 10], weights=[1, 1, 1])
+
+    # Leaving negatives in place now warns.
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
+        _ = s.gini()
+        user_warnings = [x for x in w if issubclass(x.category, UserWarning)]
+        assert len(user_warnings) == 1
+        assert "negative" in str(user_warnings[0].message).lower()
+
+    # 'zero' clamps negatives. Values become [0, 0, 10] with equal
+    # weights; closed-form gini = 2/3.
+    assert np.isclose(s.gini(negatives="zero"), 2 / 3)
+
+    # 'shift' adds |min|. Values become [0, 5, 15]; Gini in [0, 1].
+    shifted = s.gini(negatives="shift")
+    assert 0 <= shifted <= 1
+
+    # All-zero short-circuits to 0 instead of nan/RuntimeWarning.
+    assert mdf.MicroSeries([0, 0, 0], weights=[1, 2, 3]).gini() == 0.0
+
+    # Invalid negatives arg raises.
+    with pytest.raises(ValueError):
+        mdf.MicroSeries([1, 2, 3], weights=[1, 1, 1]).gini(negatives="bogus")
+
+
+def test_std_var_are_weighted() -> None:
+    """Regression: std/var used to silently fall through to pandas.
+
+    The old implementation had no override, so a MicroSeries with very uneven
+    weights returned the unweighted 1.0. Now std and var treat the weights as
+    frequency counts, matching numpy on the replicated sample.
+    """
+    s = mdf.MicroSeries([1, 2, 3], weights=[100, 1, 1])
+    # Unweighted would be 1.0. Weighted std pulls toward the heavy row.
+    assert s.std() < 1.0
+    assert s.var() < 1.0
+
+    # Integer-replication equivalence.
+    s = mdf.MicroSeries([1, 2, 3], weights=[2, 3, 1])
+    rep = np.array([1, 1, 2, 2, 2, 3])
+    assert np.isclose(s.std(), np.std(rep, ddof=1))
+    assert np.isclose(s.var(), np.var(rep, ddof=1))
+    assert np.isclose(s.var(ddof=0), np.var(rep, ddof=0))
+
+    # NaN handling.
+    s = mdf.MicroSeries([1.0, np.nan, 3.0], weights=[2, 3, 1])
+    assert not np.isnan(s.std())
+    assert np.isnan(s.std(skipna=False))
+
+    # DataFrame dispatch: df.std() / df.var() now return weighted stats.
+    df = mdf.MicroDataFrame({"x": [1, 2, 3], "y": [10, 20, 30]}, weights=[2, 3, 1])
+    np.testing.assert_allclose(
+        df.std().values,
+        [
+            np.std(rep, ddof=1),
+            np.std(np.array([10, 10, 20, 20, 20, 30]), ddof=1),
+        ],
+    )
+
+
+def test_cov_corr_warn_when_fallthrough() -> None:
+    """Regression: cov/corr silently returned unweighted pandas values.
+
+    They still fall through to pandas (a weighted impl is a separate issue) but
+    now emit a UserWarning so callers aren't misled.
+    """
+    s1 = mdf.MicroSeries([1, 2, 3], weights=[1, 1, 1])
+    s2 = mdf.MicroSeries([2, 4, 6], weights=[1, 1, 1])
+
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
+        _ = s1.cov(s2)
+        msgs = [str(x.message) for x in w if issubclass(x.category, UserWarning)]
+        assert any("unweighted" in m.lower() for m in msgs)
+
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
+        _ = s1.corr(s2)
+        msgs = [str(x.message) for x in w if issubclass(x.category, UserWarning)]
+        assert any("unweighted" in m.lower() for m in msgs)
+
+
+def test_count_skips_nan_by_default() -> None:
+    """Regression: ``count()`` included NaN-row weight, contrary to pandas.
+
+    Pandas ``Series.count`` skips NaN; MicroSeries returned the full weight sum
+    regardless. The fix matches pandas semantics and adds a ``skipna`` kwarg so
+    callers can opt out.
+    """
+    s = mdf.MicroSeries([1.0, np.nan, 3.0], weights=[10, 20, 30])
+    assert s.count() == 40.0
+    assert s.count(skipna=True) == 40.0
+    assert s.count(skipna=False) == 60.0
+
+    # No NaN: skipna is a no-op.
+    assert mdf.MicroSeries([1, 2, 3], weights=[2, 3, 4]).count() == 9.0
+
+    # All NaN: count skips everything.
+    all_nan = mdf.MicroSeries([np.nan] * 3, weights=[1, 2, 3])
+    assert all_nan.count() == 0.0
+    assert all_nan.count(skipna=False) == 6.0
+
+
+def test_rank_ties_share_bucket() -> None:
+    """Regression: rank used to assign ties to different ranks/buckets.
+
+    Previously ``rank`` returned the running cumulative weight in sort order,
+    so every row — tied or not — got a distinct value. As a result
+    ``MicroSeries([5]*5, weights=[1]*5).decile_rank()`` returned ``[2, 4, 6, 8,
+    10]`` rather than all 10. With max-rank semantics, tied values share the
+    cumulative weight at the end of their tie group, so bucketing is stable
+    under ties.
+    """
+    # All tied: every element lands in the top decile.
+    s = mdf.MicroSeries([5] * 5, weights=[1] * 5)
+    np.testing.assert_array_equal(s.rank().values, [5, 5, 5, 5, 5])
+    np.testing.assert_array_equal(s.decile_rank().values, [10] * 5)
+    np.testing.assert_array_equal(s.quintile_rank().values, [5] * 5)
+
+    # Partial ties.
+    s = mdf.MicroSeries([1, 2, 2, 3], weights=[1, 1, 1, 1])
+    np.testing.assert_array_equal(s.rank().values, [1, 3, 3, 4])
+
+    # pct=True normalizes to (0, 1] and still shares ranks on ties.
+    s = mdf.MicroSeries([5] * 4, weights=[1] * 4)
+    np.testing.assert_allclose(s.rank(pct=True).values, [1.0, 1.0, 1.0, 1.0])
+
+    # Non-ties still match the old cumulative-weight behaviour, so the
+    # existing ``test_rank`` expectations hold.
+    s = mdf.MicroSeries([1, 2, 3], weights=[4, 5, 6])
+    np.testing.assert_array_equal(s.rank().values, [4, 9, 15])

{microdf_python-1.2.3.dist-info → microdf_python-1.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: microdf-python
-Version: 1.2.3
+Version: 1.3.2
 Summary: Weighted pandas DataFrames and Series for survey microdata
 Author-email: Max Ghenis <max@policyengine.org>
 License: MIT
@@ -50,13 +50,12 @@ import microdf as mdf
 import pandas as pd
 
 # Create sample data with weights
-df = pd.DataFrame({
-    'income': [10_000, 20_000, 30_000, 40_000, 50_000],
-    'weights': [1, 2, 3, 2, 1]
-})
+df = pd.DataFrame(
+    {"income": [10_000, 20_000, 30_000, 40_000, 50_000], "weights": [1, 2, 3, 2, 1]}
+)
 
 # Create a MicroDataFrame
-mdf_df = mdf.MicroDataFrame(df, weights='weights')
+mdf_df = mdf.MicroDataFrame(df, weights="weights")
 
 # All operations are weight-aware
 print(mdf_df.income.mean())  # Weighted mean

microdf_python-1.3.2.dist-info/RECORD

@@ -0,0 +1,11 @@
+microdf/__init__.py,sha256=ldRvhE8t4cD7qlxQ5yLndMnWEUYpDw4m-jCDV_psXkY,319
+microdf/microdataframe.py,sha256=X6OB4Cy8_ENyr1tzplYMdbtVY33zvHmrHX1KUpAEiLE,37284
+microdf/microseries.py,sha256=9ScONBPMH03KtkIWxWYqcwsZ7c761FhWCr8DSnJkZQU,33394
+microdf/tests/conftest.py,sha256=u-EMyX1-u_nM-YO0RJYCzYHQDXxUI2WQE6GkyJlErqg,150
+microdf/tests/test_microseries_dataframe.py,sha256=r9S9z2RFgnFZFMsaT72N-zClT6jEL8YxQ_MUPAr3xi8,30969
+microdf/tests/test_pandas3_compatibility.py,sha256=A34Ni_WQ303sSNv-sqv5CGAQp54zj-ZSGAPEBHZslNI,8573
+microdf_python-1.3.2.dist-info/licenses/LICENSE,sha256=uPs-ASYnzlldpf2z8jeRgQFeEH3FLhSuX0rw0OKWoDU,1067
+microdf_python-1.3.2.dist-info/METADATA,sha256=Mq_ElGrPKz7E3wjCXGBWnuwt40SrDoLSh9hTg8tqSto,2307
+microdf_python-1.3.2.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+microdf_python-1.3.2.dist-info/top_level.txt,sha256=T2WFPTygQQMdS3GF8YpZ12DKfMGrspbZ3r7z-e3KfiM,8
+microdf_python-1.3.2.dist-info/RECORD,,

{microdf_python-1.2.3.dist-info → microdf_python-1.3.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (82.0.0)
+Generator: setuptools (84.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

microdf_python-1.2.3.dist-info/RECORD

@@ -1,11 +0,0 @@
-microdf/__init__.py,sha256=ldRvhE8t4cD7qlxQ5yLndMnWEUYpDw4m-jCDV_psXkY,319
-microdf/microdataframe.py,sha256=kjID2Zn_8zEp5IG7zc81B9M9QoAb2smMhczL3m8-n1k,33506
-microdf/microseries.py,sha256=fl5GHRRmeehxA-701bbTbz93T7a_BOpjEDmS2BvXSfQ,24775
-microdf/tests/conftest.py,sha256=u-EMyX1-u_nM-YO0RJYCzYHQDXxUI2WQE6GkyJlErqg,150
-microdf/tests/test_microseries_dataframe.py,sha256=WhwxyoFNPQLCvtsIPfAic0Zk71jWedZ6MT0ZaYBPEXk,16186
-microdf/tests/test_pandas3_compatibility.py,sha256=A34Ni_WQ303sSNv-sqv5CGAQp54zj-ZSGAPEBHZslNI,8573
-microdf_python-1.2.3.dist-info/licenses/LICENSE,sha256=uPs-ASYnzlldpf2z8jeRgQFeEH3FLhSuX0rw0OKWoDU,1067
-microdf_python-1.2.3.dist-info/METADATA,sha256=bbTmyHLxi1ThumndS_GMYNlb93WH-PjFSS7h6KJ5j2A,2311
-microdf_python-1.2.3.dist-info/WHEEL,sha256=YCfwYGOYMi5Jhw2fU4yNgwErybb2IX5PEwBKV4ZbdBo,91
-microdf_python-1.2.3.dist-info/top_level.txt,sha256=T2WFPTygQQMdS3GF8YpZ12DKfMGrspbZ3r7z-e3KfiM,8
-microdf_python-1.2.3.dist-info/RECORD,,