microdf-python 1.3.9__py3-none-any.whl → 1.4.0__py3-none-any.whl

microdf/microdataframe.py

@@ -161,13 +161,60 @@ class MicroDataFrame(pd.DataFrame):
     def override_df_functions(self) -> None:
         """Override DataFrame functions to work with weighted operations."""
         for name in MicroSeries.FUNCTIONS:
-            if name in MicroSeries.SCALAR_FUNCTIONS:
+            if name == "sum":
+                # Sum has its own axis-aware signature and result types.
+                continue
+            elif name in MicroSeries.SCALAR_FUNCTIONS:
                 setattr(self, name, self._create_scalar_function(name))
             elif name in MicroSeries.VECTOR_FUNCTIONS:
                 setattr(self, name, self._create_vector_function(name))
             elif name in MicroSeries.AGNOSTIC_FUNCTIONS:
                 setattr(self, name, self._create_agnostic_function(name))
 
+    def sum(
+        self,
+        axis: Optional[Union[int, str]] = 0,
+        skipna: bool = True,
+        numeric_only: bool = False,
+        min_count: int = 0,
+        **kwargs,
+    ) -> Union[pd.Series, MicroSeries, float]:
+        """Sum numeric columns, weighting reductions across observations.
+
+        Column sums (axis=0 or 'index') apply observation weights and return a
+        plain Series. Row sums (axis=1 or 'columns') do not multiply row values
+        by weights; they return a MicroSeries with an independent copy of the
+        original weights for subsequent weighted aggregation.
+
+        Non-numeric columns are excluded, matching other MicroDataFrame
+        aggregations. skipna and min_count follow pandas sum semantics.
+        Explicit axis=None follows the installed pandas version: column sums in
+        pandas 2, and a weighted total over both axes in pandas 3.
+        """
+        axis_number = None if axis is None else self._get_axis_number(axis)
+        values = pd.DataFrame(self)
+        numeric_columns = [
+            pd.api.types.is_numeric_dtype(dtype) for dtype in values.dtypes
+        ]
+        values = values.iloc[:, numeric_columns]
+        if axis_number != 1 and self.weights is not None:
+            values = values.mul(self.weights, axis=0)
+        result = values.sum(
+            axis=axis,
+            skipna=skipna,
+            numeric_only=numeric_only,
+            min_count=min_count,
+            **kwargs,
+        )
+        if axis_number == 1:
+            weights = (
+                self.weights.copy()
+                if self.weights is not None
+                else pd.Series(1.0, index=self.index)
+            )
+            return MicroSeries(result, weights=weights)
+        return result
+
     def _create_scalar_function(self, name: str) -> Callable:
         """Create a scalar function that returns a Series of results.
 

microdf/microseries.py

@@ -9,6 +9,41 @@ import pandas as pd
 logger = logging.getLogger(__name__)
 
 
+def _weighted_centered_vector(
+    values: np.ndarray, weights: np.ndarray
+) -> tuple[np.ndarray, int]:
+    """Return scaled sqrt-weighted deviations and their power-of-two
+    exponent."""
+    # Center relative to a maximum-weight observation: shifting by a low-weight
+    # extreme could erase differences among the influential observations.
+    # A relative mean also preserves nearby values at a large common offset.
+    reference = values[np.argmax(weights)]
+    with np.errstate(over="ignore"):
+        shifted = values - reference
+    exponent = 0
+    if np.isinf(shifted).any():
+        # Opposite finite extremes can overflow their difference. Halving is
+        # exact for those values; restore that factor in the final exponent.
+        shifted = values / 2 - reference / 2
+        exponent = 1
+    magnitude = np.max(np.abs(shifted))
+    if magnitude == 0:
+        return shifted, 0
+    _, shift = np.frexp(magnitude)
+    shifted = np.ldexp(shifted, -shift)
+    # Raise tiny mean weights by an exact common power of two so products
+    # with the scaled deviations do not underflow. Never scale down: that
+    # could discard small weights when frequencies span a wide range.
+    _, mean_weight_exponent = np.frexp(np.max(weights))
+    mean_weights = np.ldexp(weights, -min(int(mean_weight_exponent), 0))
+    shifted -= np.average(shifted, weights=mean_weights)
+    # Weight each vector before taking products, then scale again so squared
+    # deviations never accumulate raw frequencies at the original value scale.
+    shifted *= np.sqrt(weights)
+    _, weight_shift = np.frexp(np.max(np.abs(shifted)))
+    return np.ldexp(shifted, -weight_shift), exponent + int(shift) + int(weight_shift)
+
+
 def _weighted_top_share(
     values: np.ndarray, weights: np.ndarray, top_x_pct: float
 ) -> float:
@@ -187,16 +222,38 @@ class MicroSeries(pd.Series):
         :returns: A Series multiplying the MicroSeries by its weight.
         :rtype: pd.Series
         """
-        return self.multiply(self.weights)
+        return pd.Series(self, copy=False).multiply(self.weights)
 
     @scalar_function
-    def sum(self) -> float:
+    def sum(
+        self,
+        axis: Optional[Union[int, str]] = 0,
+        skipna: bool = True,
+        numeric_only: bool = False,
+        min_count: int = 0,
+        **kwargs,
+    ) -> float:
         """Calculates the weighted sum of the MicroSeries.
 
+        axis may be 0, 'index' or None, as for pandas Series.sum. skipna,
+        numeric_only and min_count are applied to the weighted values;
+        min_count counts valid observations, not the sum of their weights.
+
         :returns: The weighted sum.
         :rtype: float
         """
-        return self.multiply(self.weights).sum()
+        # Keep the intermediate unweighted so subclass constructors cannot
+        # apply observation weights a second time during the final reduction.
+        values = pd.Series(self)
+        if not self.empty:
+            values = values.multiply(self.weights)
+        return values.sum(
+            axis=axis,
+            skipna=skipna,
+            numeric_only=numeric_only,
+            min_count=min_count,
+            **kwargs,
+        )
 
     @scalar_function
     def count(self, skipna: bool = True) -> float:
@@ -293,39 +350,152 @@ class MicroSeries(pd.Series):
         v = self._weighted_variance(ddof=ddof, skipna=skipna)
         return float(np.sqrt(v)) if np.isfinite(v) else v
 
-    def cov(self, other, *args, **kwargs):
-        """Pandas ``cov`` — **unweighted**.
+    def _weighted_pair(
+        self,
+        other: pd.Series,
+        min_periods: Optional[int],
+        ddof: int,
+        skipna: bool,
+    ) -> Optional[tuple[np.ndarray, np.ndarray, np.ndarray, float]]:
+        """Align usable paired observations with their left weights."""
+        if not isinstance(other, pd.Series):
+            raise TypeError("other must be a pandas Series or MicroSeries")
+        if not isinstance(ddof, (int, np.integer)):
+            raise TypeError("ddof must be an integer")
+        if min_periods is None:
+            min_periods = 1
+        if not isinstance(min_periods, (int, np.integer)) or min_periods < 0:
+            raise ValueError("min_periods must be a nonnegative integer")
+        if len(self) == 0 or len(other) == 0:
+            return None
+
+        # Align left row positions so values and weights undergo exactly the
+        # same join, including pandas' expansion of duplicate index labels.
+        positions = pd.Series(np.arange(len(self)), index=self.index)
+        positions, right = positions.align(pd.Series(other), join="inner")
+        positions = positions.to_numpy(dtype=int)
+        x = (
+            pd.Series(self._values)
+            .iloc[positions]
+            .to_numpy(dtype=float, na_value=np.nan)
+        )
+        y = right.to_numpy(dtype=float, na_value=np.nan)
+        weights = np.asarray(self.weights, dtype=float)[positions]
+        if not np.isfinite(weights).all() or (weights < 0).any():
+            raise ValueError("frequency weights must be finite and nonnegative")
+
+        # Zero frequency means the row is absent, including for skipna=False.
+        positive = weights > 0
+        x, y, weights = x[positive], y[positive], weights[positive]
+        missing = np.isnan(x) | np.isnan(y)
+        if not skipna and missing.any():
+            return None
+        x, y, weights = x[~missing], y[~missing], weights[~missing]
+        total_weight = weights.sum()
+        if not np.isfinite(total_weight):
+            raise ValueError("the sum of frequency weights must be finite")
+        if len(x) < min_periods or total_weight == 0 or total_weight <= ddof:
+            return None
+        return (
+            x,
+            y,
+            weights,
+            float(total_weight - ddof),
+        )
 
-        MicroSeries does not yet compute weighted covariance. Emits a
-        ``UserWarning`` so callers aren't silently given an unweighted number
-        after ``.sum()`` and ``.mean()`` worked as expected. See issue tracker
-        for a weighted implementation.
+    def cov(
+        self,
+        other: pd.Series,
+        min_periods: Optional[int] = None,
+        ddof: int = 1,
+        *,
+        skipna: bool = True,
+    ) -> float:
+        """Calculate frequency-weighted covariance with another Series.
+
+        Observations align by index as in pandas, including its duplicate-
+        label join behavior. Only this Series' weights are used; weights on
+        another MicroSeries are ignored. Each aligned left weight must be
+        finite and nonnegative. Zero-weight rows are omitted.
+
+        Uses ``sum(w * (x - xmean) * (y - ymean)) / (sum(w) - ddof)``.
+        Integer weights therefore match covariance on the replicated sample.
+        Missing values are removed pairwise before computing both means.
+
+        :param other: A pandas Series or MicroSeries to align by index.
+        :param min_periods: Minimum usable aligned row pairs, not the sum of
+            frequency weights. Defaults to 1.
+        :param ddof: Degrees of freedom subtracted from the weight total.
+        :param skipna: Drop pairs with a missing value. If False, any missing
+            value in a positive-weight aligned pair produces NaN.
+        :returns: Weighted covariance, or NaN for an empty or insufficient
+            sample (including a weight total no greater than ddof).
         """
-        warnings.warn(
-            "MicroSeries.cov() falls through to pandas and is "
-            "unweighted. Use MicroSeries.var()/std() for weighted "
-            "second moments, or compute covariance manually with the "
-            "weights.",
-            UserWarning,
-            stacklevel=2,
+        pair = self._weighted_pair(other, min_periods, ddof, skipna)
+        if pair is None:
+            return np.nan
+        x, y, weights, denominator = pair
+        if not np.isfinite(x).all() or not np.isfinite(y).all():
+            return np.nan
+        x, x_exponent = _weighted_centered_vector(x, weights)
+        y, y_exponent = _weighted_centered_vector(y, weights)
+        # Combine exponents only after dividing out sum(weights) - ddof.
+        # Neither the original squared scale nor raw weighted sum need fit.
+        denominator, denominator_exponent = np.frexp(denominator)
+        return float(
+            np.ldexp(
+                np.sum(x * y) / denominator,
+                x_exponent + y_exponent - int(denominator_exponent),
+            )
         )
-        return super().cov(other, *args, **kwargs)
 
-    def corr(self, other, *args, **kwargs):
-        """Pandas ``corr`` — **unweighted**.
-
-        MicroSeries does not yet compute weighted correlation. Emits a
-        ``UserWarning`` so callers aren't silently given an unweighted number.
-        See issue tracker for a weighted implementation.
+    def corr(
+        self,
+        other: pd.Series,
+        method: str = "pearson",
+        min_periods: Optional[int] = None,
+        *,
+        ddof: int = 1,
+        skipna: bool = True,
+    ) -> float:
+        """Calculate frequency-weighted Pearson correlation.
+
+        Uses the same aligned pairs and left Series weights for covariance and
+        both variances. Weights on another MicroSeries are ignored. Weights
+        must be finite and nonnegative; zero-weight rows are omitted. Other
+        correlation methods, including callables, are unsupported.
+
+        :param other: A pandas Series or MicroSeries to align by index.
+        :param method: Only "pearson" is supported.
+        :param min_periods: Minimum usable aligned row pairs, not frequency
+            weight total. Defaults to 1.
+        :param ddof: Degrees of freedom for all three moments. It cancels from
+            the correlation but the weight total must exceed it.
+        :param skipna: Drop pairs with a missing value. If False, any missing
+            value in a positive-weight aligned pair produces NaN.
+        :returns: Weighted correlation, or NaN for an empty, insufficient, or
+            constant sample.
         """
-        warnings.warn(
-            "MicroSeries.corr() falls through to pandas and is "
-            "unweighted. Compute correlation manually with the weights "
-            "if you need the survey-weighted value.",
-            UserWarning,
-            stacklevel=2,
-        )
-        return super().corr(other, *args, **kwargs)
+        if method != "pearson":
+            raise ValueError("weighted correlation only supports method='pearson'")
+        pair = self._weighted_pair(other, min_periods, ddof, skipna)
+        if pair is None:
+            return np.nan
+        x, y, weights, _ = pair
+        if not np.isfinite(x).all() or not np.isfinite(y).all():
+            return np.nan
+        # A weighted mean can round away from identical decimal inputs.
+        # Check the retained observations exactly before subtracting it.
+        if (x == x[0]).all() or (y == y[0]).all():
+            return np.nan
+        x, _ = _weighted_centered_vector(x, weights)
+        y, _ = _weighted_centered_vector(y, weights)
+        x_ss = np.sum(x * x)
+        y_ss = np.sum(y * y)
+        if x_ss == 0 or y_ss == 0:
+            return np.nan
+        result = np.sum(x * y) / (np.sqrt(x_ss) * np.sqrt(y_ss))
+        return float(np.clip(result, -1.0, 1.0))
 
     def quantile(self, q: np.array, skipna: bool = True) -> pd.Series:
         """Calculates weighted quantiles of the MicroSeries.

microdf/tests/test_microseries_dataframe.py

@@ -744,28 +744,6 @@ def test_std_var_are_weighted() -> None:
     )
 
 
-def test_cov_corr_warn_when_fallthrough() -> None:
-    """Regression: cov/corr silently returned unweighted pandas values.
-
-    They still fall through to pandas (a weighted impl is a separate issue) but
-    now emit a UserWarning so callers aren't misled.
-    """
-    s1 = mdf.MicroSeries([1, 2, 3], weights=[1, 1, 1])
-    s2 = mdf.MicroSeries([2, 4, 6], weights=[1, 1, 1])
-
-    with warnings.catch_warnings(record=True) as w:
-        warnings.simplefilter("always")
-        _ = s1.cov(s2)
-        msgs = [str(x.message) for x in w if issubclass(x.category, UserWarning)]
-        assert any("unweighted" in m.lower() for m in msgs)
-
-    with warnings.catch_warnings(record=True) as w:
-        warnings.simplefilter("always")
-        _ = s1.corr(s2)
-        msgs = [str(x.message) for x in w if issubclass(x.category, UserWarning)]
-        assert any("unweighted" in m.lower() for m in msgs)
-
-
 def test_count_skips_nan_by_default() -> None:
     """Regression: ``count()`` included NaN-row weight, contrary to pandas.
 

microdf/tests/test_sum_axes.py

@@ -0,0 +1,216 @@
+import warnings
+
+import numpy as np
+import pandas as pd
+import pytest
+
+import microdf as mdf
+
+
+def test_sum_axis_1() -> None:
+    # Test basic row-wise sum
+    df = mdf.MicroDataFrame(
+        {"A": [1, 2, 3], "B": [4, 5, 6], "C": [7, 8, 9]},
+        weights=[0.5, 1.0, 2.0],
+    )
+
+    # Row-wise sum (axis=1) should not use weights
+    row_sums = df.sum(axis=1)
+    expected = pd.Series([12, 15, 18], index=df.index)  # 1+4+7, 2+5+8, 3+6+9
+    pd.testing.assert_series_equal(pd.Series(row_sums), expected)
+
+    # Column-wise sum (axis=0) should use weights
+    col_sums = df.sum(axis=0)
+    expected_weighted = pd.Series(
+        {
+            "A": 1 * 0.5 + 2 * 1.0 + 3 * 2.0,  # 8.5
+            "B": 4 * 0.5 + 5 * 1.0 + 6 * 2.0,  # 19.0
+            "C": 7 * 0.5 + 8 * 1.0 + 9 * 2.0,  # 29.5
+        }
+    )
+    pd.testing.assert_series_equal(col_sums, expected_weighted)
+
+    # Test with mixed types (non-numeric columns should be ignored)
+    df_mixed = mdf.MicroDataFrame(
+        {"A": [1, 2, 3], "B": [4, 5, 6], "text": ["a", "b", "c"]},
+        weights=[1, 1, 1],
+    )
+
+    row_sums_mixed = df_mixed.sum(axis=1)
+    expected_mixed = pd.Series([5, 7, 9], index=df_mixed.index)  # Only A+B
+    pd.testing.assert_series_equal(pd.Series(row_sums_mixed), expected_mixed)
+
+    # Test with axis='columns' (string form)
+    row_sums_str = df.sum(axis="columns")
+    pd.testing.assert_series_equal(pd.Series(row_sums_str), expected)
+
+    # Test with additional parameters
+    df_with_nan = mdf.MicroDataFrame(
+        {"A": [1, np.nan, 3], "B": [4, 5, 6], "C": [7, 8, np.nan]},
+        weights=[1, 1, 1],
+    )
+
+    # skipna=True (default)
+    row_sums_skipna = df_with_nan.sum(axis=1)
+    expected_skipna = pd.Series([12.0, 13.0, 9.0])  # NaN values skipped
+    pd.testing.assert_series_equal(pd.Series(row_sums_skipna), expected_skipna)
+
+    # skipna=False
+    row_sums_no_skipna = df_with_nan.sum(axis=1, skipna=False)
+    expected_no_skipna = pd.Series([12.0, np.nan, np.nan])  # NaN propagates
+    pd.testing.assert_series_equal(pd.Series(row_sums_no_skipna), expected_no_skipna)
+
+    # Test min_count parameter
+    row_sums_min_count = df_with_nan.sum(axis=1, min_count=3)
+    expected_min_count = pd.Series(
+        [12.0, np.nan, np.nan]
+    )  # Row 1 and 2 have < 3 non-NA values
+    pd.testing.assert_series_equal(pd.Series(row_sums_min_count), expected_min_count)
+
+
+@pytest.mark.parametrize("axis", [0, "index", 1, "columns"])
+@pytest.mark.parametrize("positional", [False, True])
+def test_sum_binds_positional_and_keyword_axes(axis, positional):
+    frame = mdf.MicroDataFrame(
+        {"a": [1, 2, 3], "b": [4, 5, 6]}, index=[7, 8, 9], weights=[1, 2, 3]
+    )
+    result = frame.sum(axis) if positional else frame.sum(axis=axis)
+    if axis in (0, "index"):
+        assert type(result) is pd.Series
+        pd.testing.assert_series_equal(result, pd.Series({"a": 14.0, "b": 32.0}))
+    else:
+        assert isinstance(result, mdf.MicroSeries)
+        pd.testing.assert_series_equal(
+            pd.Series(result), pd.Series([5, 7, 9], index=frame.index)
+        )
+        pd.testing.assert_series_equal(result.weights, frame.weights)
+        # Row values are not weighted yet; subsequent aggregation is weighted.
+        assert result.sum() == 5 * 1 + 7 * 2 + 9 * 3
+        result.weights.iloc[0] = 100
+        assert frame.weights.iloc[0] == 1
+
+
+@pytest.mark.parametrize("skipna,min_count", [(True, 0), (False, 0), (True, 3)])
+@pytest.mark.parametrize("axis", [0, "index", None])
+def test_weighted_column_sum_options(axis, skipna, min_count):
+    raw = pd.DataFrame({"a": [1.0, np.nan, 3.0], "b": [4.0, 5.0, 6.0]})
+    weights = pd.Series([1.0, 2.0, 3.0])
+    frame = mdf.MicroDataFrame(raw, weights=weights)
+    # Native sum defines version-specific axis=None and missing-value behavior.
+    # The independently weighted entries are [1, NaN, 9] and [4, 10, 18].
+    expected_data = pd.DataFrame({"a": [1.0, np.nan, 9.0], "b": [4.0, 10.0, 18.0]})
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", FutureWarning)
+        expected = expected_data.sum(axis=axis, skipna=skipna, min_count=min_count)
+        actual = frame.sum(axis=axis, skipna=skipna, min_count=min_count)
+    if isinstance(expected, pd.Series):
+        assert type(actual) is pd.Series
+        pd.testing.assert_series_equal(actual, expected)
+    else:
+        np.testing.assert_allclose(actual, expected, equal_nan=True)
+
+
+@pytest.mark.parametrize("axis", [None, 0, "index"])
+@pytest.mark.parametrize("skipna,min_count", [(True, 0), (False, 0), (True, 3)])
+def test_microseries_sum_options(axis, skipna, min_count):
+    series = mdf.MicroSeries([1.0, np.nan, 3.0], index=[7, 8, 9], weights=[1, 2, 3])
+    expected = pd.Series([1.0, np.nan, 9.0]).sum(
+        axis=axis, skipna=skipna, min_count=min_count
+    )
+    np.testing.assert_allclose(
+        series.sum(axis, skipna=skipna, min_count=min_count), expected, equal_nan=True
+    )
+
+
+@pytest.mark.parametrize("min_count,expected", [(0, 0.0), (1, np.nan)])
+def test_empty_numeric_row_sum_identity(min_count, expected):
+    frame = mdf.MicroDataFrame({"text": ["a", "b"]}, index=[7, 8], weights=[2, 3])
+    actual = frame.sum(axis=1, min_count=min_count)
+    assert isinstance(actual, mdf.MicroSeries)
+    pd.testing.assert_series_equal(
+        pd.Series(actual), pd.Series([expected, expected], index=frame.index)
+    )
+    pd.testing.assert_series_equal(actual.weights, frame.weights)
+
+
+def test_sum_rejects_invalid_arguments():
+    frame = mdf.MicroDataFrame({"a": [1, 2]}, weights=[1, 2])
+    with pytest.raises(TypeError):
+        frame.sum(1, axis=0)
+    with pytest.raises(TypeError):
+        frame.sum(bogus=True)
+    with pytest.raises(ValueError):
+        frame.sum(axis=2)
+    with pytest.raises(ValueError):
+        frame["a"].sum(axis=1)
+
+
+def test_sum_handles_boolean_and_nullable_numeric_columns():
+    raw = pd.DataFrame(
+        {
+            "count": pd.Series([1, None, 3], dtype="Int64"),
+            "flag": pd.Series([True, False, True], dtype="boolean"),
+            "text": ["a", "b", "c"],
+        }
+    )
+    frame = mdf.MicroDataFrame(raw, weights=[1, 2, 3])
+    expected = raw[["count", "flag"]].sum(axis=1)
+    pd.testing.assert_series_equal(pd.Series(frame.sum(1)), expected)
+    totals = frame.sum(numeric_only=True)
+    assert list(totals.index) == ["count", "flag"]
+    assert totals["count"] == 10
+    assert totals["flag"] == 4
+
+
+def test_sum_preserves_other_scalar_positional_arguments():
+    frame = mdf.MicroDataFrame({"a": [-1.0, 2.0, 3.0]}, weights=[1, 2, 3])
+    for method, argument in [
+        ("gini", "shift"),
+        ("top_x_pct_share", 0.25),
+        ("mean", False),
+        ("var", 0),
+    ]:
+        actual = getattr(frame, method)(argument)["a"]
+        expected = getattr(frame["a"], method)(argument)
+        assert actual == expected
+
+
+@pytest.mark.parametrize("min_count,expected", [(0, 0.0), (1, np.nan)])
+def test_sum_of_empty_inputs(min_count, expected):
+    frame = mdf.MicroDataFrame(pd.DataFrame({"a": pd.Series([], dtype=float)}))
+    row_sums = frame.sum(axis=1, min_count=min_count)
+    assert isinstance(row_sums, mdf.MicroSeries)
+    assert row_sums.empty
+    pd.testing.assert_series_equal(row_sums.weights, pd.Series([], dtype=float))
+    pd.testing.assert_series_equal(
+        frame.sum(min_count=min_count), pd.Series({"a": expected})
+    )
+    series = mdf.MicroSeries([], dtype=float)
+    np.testing.assert_allclose(
+        series.sum(min_count=min_count), expected, equal_nan=True
+    )
+    with pytest.raises(ValueError):
+        series.sum(axis=1)
+
+
+@pytest.mark.parametrize("axis", [0, 1])
+@pytest.mark.parametrize("mixed_dtypes", [False, True])
+def test_sum_preserves_duplicate_numeric_column_labels(axis, mixed_dtypes):
+    if mixed_dtypes:
+        raw = pd.DataFrame([[1.0, "x", 4.0], [2.0, "y", 5.0]], columns=["a", "a", "a"])
+    else:
+        raw = pd.DataFrame([[1.0, 4.0], [2.0, 5.0]], columns=["a", "a"])
+    frame = mdf.MicroDataFrame(raw, weights=[2, 3])
+
+    result = frame.sum(axis)
+
+    if axis == 0:
+        assert type(result) is pd.Series
+        expected = pd.Series([8.0, 23.0], index=["a", "a"])
+        pd.testing.assert_series_equal(result, expected)
+    else:
+        assert isinstance(result, mdf.MicroSeries)
+        pd.testing.assert_series_equal(pd.Series(result), pd.Series([5.0, 7.0]))
+        pd.testing.assert_series_equal(result.weights, frame.weights)
+        assert result.sum() == 31.0
+    pd.testing.assert_frame_equal(pd.DataFrame(frame), raw)

microdf/tests/test_weighted_cov_corr.py

@@ -0,0 +1,343 @@
+import warnings
+from decimal import Decimal, localcontext
+from fractions import Fraction
+from itertools import permutations
+
+import numpy as np
+import pandas as pd
+import pytest
+
+import microdf as mdf
+
+
+def replicated_moments(x, y, weights, ddof=1):
+    """Independent frequency-weight oracle: expand to an ordinary sample."""
+    repeated_x = np.repeat(np.asarray(x, dtype=float), weights)
+    repeated_y = np.repeat(np.asarray(y, dtype=float), weights)
+    return (
+        np.cov(repeated_x, repeated_y, ddof=ddof)[0, 1],
+        np.corrcoef(repeated_x, repeated_y)[0, 1],
+    )
+
+
+@pytest.mark.parametrize("ddof", [0, 1, 2])
+def test_cov_corr_match_replicated_frequency_sample(ddof):
+    x, y, weights = [1, 4, 8], [5, 2, 9], [1, 3, 2]
+    left = mdf.MicroSeries(x, weights=weights)
+    right = pd.Series(y)
+    expected_cov, expected_corr = replicated_moments(x, y, weights, ddof)
+    with warnings.catch_warnings(record=True) as caught:
+        warnings.simplefilter("always")
+        assert left.cov(right, ddof=ddof) == pytest.approx(expected_cov)
+        assert left.corr(right, ddof=ddof) == pytest.approx(expected_corr)
+    assert not any("unweighted" in str(item.message).lower() for item in caught)
+
+
+def test_cov_corr_align_indices_and_use_only_left_weights():
+    left = mdf.MicroSeries([1, 4, 8], index=["a", "b", "c"], weights=[1, 3, 2])
+    right = mdf.MicroSeries(
+        [9, 5, 2, 100], index=["c", "a", "b", "d"], weights=[99, 1, 1, 9]
+    )
+    expected_cov, expected_corr = replicated_moments([1, 4, 8], [5, 2, 9], [1, 3, 2])
+    assert left.cov(right) == pytest.approx(expected_cov)
+    assert left.corr(right) == pytest.approx(expected_corr)
+    assert right.weights.tolist() == [99, 1, 1, 9]
+
+
+def test_cov_corr_use_same_pairwise_nonmissing_sample():
+    left = mdf.MicroSeries(
+        [1, np.nan, 5, 9, 20], index=list("abcde"), weights=[1, 9, 2, 3, 7]
+    )
+    right = pd.Series([2, 4, np.nan, 8, 30], index=list("abcdf"))
+    expected_cov, expected_corr = replicated_moments([1, 9], [2, 8], [1, 3])
+    assert left.cov(right) == pytest.approx(expected_cov)
+    assert left.corr(right) == pytest.approx(expected_corr)
+    assert np.isnan(left.cov(right, skipna=False))
+    assert np.isnan(left.corr(right, skipna=False))
+
+
+@pytest.mark.parametrize("same_index", [True, False])
+def test_cov_corr_follow_pandas_duplicate_index_alignment(same_index):
+    left = mdf.MicroSeries([1, 4, 7], index=["a", "a", "b"], weights=[1, 3, 2])
+    if same_index:
+        right = pd.Series([2, 3, 8], index=["a", "a", "b"])
+        x, y, weights = [1, 4, 7], [2, 3, 8], [1, 3, 2]
+    else:
+        right = pd.Series([2, 5, 8], index=["a", "b", "b"])
+        # The shared a/b labels join, repeating the left row weight per pair.
+        x, y, weights = [1, 4, 7, 7], [2, 2, 5, 8], [1, 3, 2, 2]
+    expected_cov, expected_corr = replicated_moments(x, y, weights)
+    assert left.cov(right) == pytest.approx(expected_cov)
+    assert left.corr(right) == pytest.approx(expected_corr)
+
+
+def test_zero_weight_rows_do_not_enter_pairwise_sample():
+    left = mdf.MicroSeries([1, np.nan, 5, 1000], weights=[2, 0, 1, 0])
+    right = pd.Series([3, np.nan, 9, -1000])
+    expected_cov, expected_corr = replicated_moments([1, 5], [3, 9], [2, 1])
+    assert left.cov(right, skipna=False) == pytest.approx(expected_cov)
+    assert left.corr(right, skipna=False) == pytest.approx(expected_corr)
+    assert np.isnan(left.cov(right, min_periods=3))
+    assert np.isnan(left.corr(right, min_periods=3))
+
+
+def test_min_periods_counts_usable_rows_separately_from_frequency_weight():
+    left = mdf.MicroSeries([1, 5], weights=[10, 20])
+    right = pd.Series([3, 9])
+    expected_cov, expected_corr = replicated_moments([1, 5], [3, 9], [10, 20], ddof=0)
+    # Existing pandas positional arguments retain their order.
+    assert left.cov(right, 2, 0) == pytest.approx(expected_cov)
+    assert left.corr(right, "pearson", 2, ddof=0) == pytest.approx(expected_corr)
+    assert np.isnan(left.cov(right, min_periods=3))
+    assert np.isnan(left.corr(right, min_periods=3))
+
+
+@pytest.mark.parametrize(
+    "values,other,weights",
+    [([], [], []), ([np.nan], [1], [3]), ([1], [2], [0]), ([1], [2], [1])],
+)
+def test_cov_corr_return_nan_when_sample_is_insufficient(values, other, weights):
+    left = mdf.MicroSeries(values, weights=weights, dtype=float)
+    right = pd.Series(other, dtype=float)
+    assert np.isnan(left.cov(right))
+    assert np.isnan(left.corr(right))
+
+
+def test_cov_corr_no_index_overlap():
+    left = mdf.MicroSeries([1, 2], index=["a", "b"], weights=[1, 2])
+    right = pd.Series([3, 4], index=["c", "d"])
+    assert np.isnan(left.cov(right))
+    assert np.isnan(left.corr(right))
+
+
+def test_cov_corr_constant_and_frequency_singleton():
+    left = mdf.MicroSeries([4, 4], weights=[2, 3])
+    right = pd.Series([1, 5])
+    assert left.cov(right) == 0
+    assert np.isnan(left.corr(right))
+    singleton = mdf.MicroSeries([4], weights=[3])
+    assert singleton.cov(pd.Series([2])) == 0
+    assert np.isnan(singleton.corr(pd.Series([2])))
+    assert np.isnan(singleton.cov(pd.Series([2]), ddof=3))
+    assert np.isnan(singleton.corr(pd.Series([2]), ddof=3))
+
+
+def test_cov_corr_nullable_numeric_data():
+    left = mdf.MicroSeries(pd.Series([1, pd.NA, 4], dtype="Int64"), weights=[2, 9, 3])
+    right = pd.Series([3, 8, 7], dtype="Float64")
+    expected_cov, expected_corr = replicated_moments([1, 4], [3, 7], [2, 3])
+    assert left.cov(right) == pytest.approx(expected_cov)
+    assert left.corr(right) == pytest.approx(expected_corr)
+
+
+@pytest.mark.parametrize("method", ["spearman", "kendall", lambda x, y: 1.0])
+def test_non_pearson_methods_are_explicitly_unsupported(method):
+    left = mdf.MicroSeries([1, 2, 3], weights=[1, 2, 3])
+    with pytest.raises(ValueError, match="pearson"):
+        left.corr(pd.Series([4, 2, 5]), method=method)
+
+
+@pytest.mark.parametrize("weights", [[1, -1], [1, np.nan], [1, np.inf]])
+def test_cov_corr_reject_invalid_frequency_weights(weights):
+    left = mdf.MicroSeries([1, 2], weights=weights)
+    for method in (left.cov, left.corr):
+        with pytest.raises(ValueError, match="weights"):
+            method(pd.Series([2, 4]))
+
+
+def test_binary_statistics_not_in_dataframe_aggregation_factories():
+    assert "cov" not in mdf.MicroSeries.FUNCTIONS
+    assert "corr" not in mdf.MicroSeries.FUNCTIONS
+
+
+@pytest.mark.parametrize(
+    "x,y",
+    [([0.1, 0.1], [0, 1]), ([0, 1], [0.1, 0.1]), ([0.1, 0.1], [0.1, 0.1])],
+)
+@pytest.mark.parametrize("with_filtered_rows", [False, True])
+def test_corr_exact_decimal_constants_return_nan(x, y, with_filtered_rows):
+    # Unequal weights can round the mean away from the identical 0.1 values.
+    # Constant detection must inspect usable observations before centering.
+    weights = [1, 2]
+    if with_filtered_rows:
+        x = x + [9, np.nan]
+        y = y + [7, 4]
+        weights = weights + [0, 3]
+    left = mdf.MicroSeries(x, weights=weights)
+    assert np.isnan(left.corr(pd.Series(y)))
+
+
+def test_corr_does_not_treat_nearby_distinct_values_as_constant():
+    x = [0.1, np.nextafter(0.1, np.inf)]
+    left = mdf.MicroSeries(x, weights=[1, 2])
+    assert np.isfinite(left.corr(pd.Series([0, 1])))
+    assert np.isfinite(mdf.MicroSeries([0, 1], weights=[1, 2]).corr(pd.Series(x)))
+
+
+def exact_weighted_moments(x, y, weights, ddof=1):
+    """Compute moments of the actual input floats with exact rational
+    arithmetic."""
+    x, y, weights = [
+        [Fraction(float(value)) for value in values] for values in (x, y, weights)
+    ]
+    total = sum(weights)
+    xmean = sum(w * value for w, value in zip(weights, x)) / total
+    ymean = sum(w * value for w, value in zip(weights, y)) / total
+    xy = sum(w * (a - xmean) * (b - ymean) for a, b, w in zip(x, y, weights))
+    xx = sum(w * (value - xmean) ** 2 for value, w in zip(x, weights))
+    yy = sum(w * (value - ymean) ** 2 for value, w in zip(y, weights))
+    with localcontext() as context:
+        context.prec = 100
+        product = xx * yy
+        correlation = (Decimal(xy.numerator) / Decimal(xy.denominator)) / (
+            Decimal(product.numerator) / Decimal(product.denominator)
+        ).sqrt()
+    return float(xy / (total - ddof)), float(correlation)
+
+
+@pytest.mark.parametrize("ddof", [0, 1, 2])
+@pytest.mark.parametrize("swap", [False, True])
+def test_cov_corr_preserve_small_differences_at_large_offsets(ddof, swap):
+    # Two distinct points are perfectly linear even two float steps apart.
+    x = np.array([1e12 - 2**-13, 1e12 + 2**-13])
+    y = np.array([0.0, 1.0])
+    if swap:
+        x, y = y, x
+    weights = [1, 2]
+    expected = exact_weighted_moments(x, y, weights, ddof)
+    assert expected[1] == 1.0
+    for shifted_x, shifted_y in [(x, y), (x - x[0], y - y[0])]:
+        left = mdf.MicroSeries(shifted_x, weights=weights)
+        right = pd.Series(shifted_y)
+        np.testing.assert_allclose(
+            [left.cov(right, ddof=ddof), left.corr(right, ddof=ddof)],
+            expected,
+            rtol=2e-15,
+            atol=0,
+        )
+
+
+@pytest.mark.parametrize("frequency", [1, 1_000_000])
+@pytest.mark.parametrize("ddof", [0, 1, 2])
+def test_cov_corr_large_finite_values_do_not_overflow_raw_frequencies(frequency, ddof):
+    x = np.array([1.0, 2.0, 3.0]) * 1e153
+    weights = [frequency] * 3
+    expected = exact_weighted_moments(x, x, weights, ddof)
+    assert np.isfinite(expected).all()
+    assert expected[1] == 1.0
+    left = mdf.MicroSeries(x, weights=weights)
+    with np.errstate(over="raise", invalid="raise"):
+        actual = [left.cov(pd.Series(x), ddof=ddof), left.corr(pd.Series(x), ddof=ddof)]
+    np.testing.assert_allclose(actual, expected, rtol=2e-15, atol=0)
+
+
+@pytest.mark.parametrize("frequency", [0.5, 1, 1_000_000])
+@pytest.mark.parametrize("ddof", [0, 1, 2])
+@pytest.mark.parametrize("scales", [(1.0, 1.0), (1e153, -1e153), (1e200, 1e-200)])
+def test_cov_corr_preserve_frequency_correction_across_value_scales(
+    frequency, ddof, scales
+):
+    x = np.array([1.0, 4.0, 8.0]) * scales[0]
+    y = np.array([5.0, 2.0, 9.0]) * scales[1]
+    weights = np.array([1, 3, 2]) * frequency
+    expected = exact_weighted_moments(x, y, weights, ddof)
+    left = mdf.MicroSeries(x, weights=weights)
+    with np.errstate(over="raise", invalid="raise"):
+        actual = [left.cov(pd.Series(y), ddof=ddof), left.corr(pd.Series(y), ddof=ddof)]
+    np.testing.assert_allclose(actual, expected, rtol=3e-15, atol=0)
+
+
+@pytest.mark.parametrize("huge", [1e16, 1e20])
+@pytest.mark.parametrize("order", list(permutations(range(3))))
+def test_cov_corr_low_weight_extreme_does_not_make_result_depend_on_row_order(
+    huge, order
+):
+    # The large observation contributes to covariance, but using it as the
+    # centering origin must not erase the difference between 1 and 2.
+    x = np.array([huge, 1.0, 2.0])
+    y = np.array([0.0, 1.0, 2.0])
+    weights = np.array([1 / huge, 1.0, 1.0])
+    expected = exact_weighted_moments(x, y, weights, ddof=0)
+    order = list(order)
+    left = mdf.MicroSeries(x[order], weights=weights[order])
+    right = pd.Series(y[order])
+    np.testing.assert_allclose(
+        [left.cov(right, ddof=0), left.corr(right, ddof=0)],
+        expected,
+        rtol=3e-15,
+        atol=0,
+    )
+
+
+def test_cov_corr_smallest_common_positive_weight_cancels_from_population_moments():
+    # The common positive weight cancels: xy = 1, xx = yy = 2 for
+    # centered observations [-1, 0, 1] and [-1, 1, 0].
+    x, y = [1, 2, 3], [3, 5, 4]
+    weights = [np.nextafter(0.0, 1.0)] * 3
+    expected = exact_weighted_moments(x, y, weights, ddof=0)
+    assert expected == (1 / 3, 0.5)
+    left = mdf.MicroSeries(x, weights=weights)
+    right = pd.Series(y)
+    np.testing.assert_allclose(
+        [left.cov(right, ddof=0), left.corr(right, ddof=0)],
+        expected,
+        rtol=3e-15,
+        atol=0,
+    )
+
+
+@pytest.mark.parametrize(
+    "frequency", [np.nextafter(0.0, 1.0), np.finfo(float).tiny, 1.0]
+)
+@pytest.mark.parametrize("multipliers", [[1, 2, 3], [1, 1, 2]])
+@pytest.mark.parametrize("order", list(permutations(range(3))))
+def test_cov_corr_unequal_tiny_and_normal_weights_match_exact_moments(
+    frequency, multipliers, order
+):
+    x, y = np.array([1.0, 2.0, 3.0]), np.array([3.0, 5.0, 4.0])
+    weights = np.array(multipliers) * frequency
+    expected = exact_weighted_moments(x, y, weights, ddof=0)
+    order = list(order)
+    left = mdf.MicroSeries(x[order], weights=weights[order])
+    right = pd.Series(y[order])
+    np.testing.assert_allclose(
+        [left.cov(right, ddof=0), left.corr(right, ddof=0)],
+        expected,
+        rtol=3e-15,
+        atol=0,
+    )
+
+
+@pytest.mark.parametrize(
+    "x,y",
+    [
+        (np.array([1, 2, 3]) * 1e153, np.array([3, 5, 4]) * -1e153),
+        (np.array([1, 2, 3]) * 1e200, np.array([3, 5, 4]) * 1e-200),
+        ([1e12 - 2**-13, 1e12, 1e12 + 2**-13], [3, 5, 4]),
+    ],
+)
+def test_cov_corr_subnormal_weights_preserve_extreme_value_scales(x, y):
+    weights = np.array([1, 2, 3]) * np.nextafter(0.0, 1.0)
+    expected = exact_weighted_moments(x, y, weights, ddof=0)
+    left = mdf.MicroSeries(x, weights=weights)
+    right = pd.Series(y)
+    with np.errstate(over="raise", invalid="raise"):
+        actual = [left.cov(right, ddof=0), left.corr(right, ddof=0)]
+    np.testing.assert_allclose(actual, expected, rtol=3e-15, atol=0)
+
+
+@pytest.mark.parametrize("frequency", [np.nextafter(0.0, 1.0), 0.5])
+@pytest.mark.parametrize("ddof", [0, 1, 2])
+def test_cov_corr_center_weight_scaling_preserves_original_frequency_ddof(
+    frequency, ddof
+):
+    x, y = [1, 2, 3], [3, 5, 4]
+    weights = np.array([1, 2, 3]) * frequency
+    left = mdf.MicroSeries(x, weights=weights)
+    right = pd.Series(y)
+    actual = [left.cov(right, ddof=ddof), left.corr(right, ddof=ddof)]
+    if weights.sum() <= ddof:
+        assert np.isnan(actual).all()
+    else:
+        expected = exact_weighted_moments(x, y, weights, ddof=ddof)
+        np.testing.assert_allclose(actual, expected, rtol=3e-15, atol=0)

{microdf_python-1.3.9.dist-info → microdf_python-1.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: microdf-python
-Version: 1.3.9
+Version: 1.4.0
 Summary: Weighted pandas DataFrames and Series for survey microdata
 Author-email: Max Ghenis <max@policyengine.org>
 License: MIT

{microdf_python-1.3.9.dist-info → microdf_python-1.4.0.dist-info}/RECORD

@@ -1,17 +1,19 @@
 microdf/__init__.py,sha256=sddmTcTZFSb1fjZkoLUR5TK7r5PEld4v2F92xBI5Sxs,641
-microdf/microdataframe.py,sha256=mmWlwiXOfBLikCNeMTqXPkPlGu_l0cx1fGNwvLjnflU,42528
-microdf/microseries.py,sha256=W_6FY6iTfdsyfVwE9KreTE-38SSm6ADmeA7YTAKzD2Q,37332
+microdf/microdataframe.py,sha256=yqLUC43WBDT8Z-AQ7OfjKUoizehcUGBHgt7wg_BVlFI,44416
+microdf/microseries.py,sha256=Y2k80dUOy8XxDZ6xN-YiUX04NXajILaLEc2isMahBCA,45004
 microdf/tests/conftest.py,sha256=u-EMyX1-u_nM-YO0RJYCzYHQDXxUI2WQE6GkyJlErqg,150
 microdf/tests/test_aggregation_errors.py,sha256=9jJDiEyxMb2z1Zmj-o8AHDNp8LOputbEkAMefifnWaE,2013
 microdf/tests/test_dataframe_weight_storage.py,sha256=ngIsWa_QcBnnaLpAyIZhTgMxjv_7cK8Nbf7f4n29R4Q,1975
-microdf/tests/test_microseries_dataframe.py,sha256=r9S9z2RFgnFZFMsaT72N-zClT6jEL8YxQ_MUPAr3xi8,30969
+microdf/tests/test_microseries_dataframe.py,sha256=vL0fg_NydU6a5myVVtXyHMr8eQOB_yyAkZYwLmoEnOA,30075
 microdf/tests/test_nullify_weights_index.py,sha256=kZgzMaZEa_PXbsor2S4E-6VRid3C3rcC9ufk0qa7mgY,341
 microdf/tests/test_pandas3_compatibility.py,sha256=A34Ni_WQ303sSNv-sqv5CGAQp54zj-ZSGAPEBHZslNI,8573
 microdf/tests/test_quantile_missing_values.py,sha256=lfntDvV2q7KH_CPVrXFJSQFoaGlc_OkRGhKwpxtDQtY,5327
 microdf/tests/test_serialization.py,sha256=a7pHL2hNiG5iJjRtfx3C1BCmgOZouAekiOwUxouAPfo,5083
+microdf/tests/test_sum_axes.py,sha256=N05ocwI5lLv2OgoaovRIqFIae-70356kZemRRet0ac8,8521
 microdf/tests/test_version_metadata.py,sha256=M1EabzHLKZZw3Djd6Zu2UuMQtDLV6rZ1zDrOU7W_jf0,227
-microdf_python-1.3.9.dist-info/licenses/LICENSE,sha256=uPs-ASYnzlldpf2z8jeRgQFeEH3FLhSuX0rw0OKWoDU,1067
-microdf_python-1.3.9.dist-info/METADATA,sha256=fqiyOns5TGLBphD_6Ydj5zmvu4UxUIQWGW5TiXyDTxA,2305
-microdf_python-1.3.9.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-microdf_python-1.3.9.dist-info/top_level.txt,sha256=T2WFPTygQQMdS3GF8YpZ12DKfMGrspbZ3r7z-e3KfiM,8
-microdf_python-1.3.9.dist-info/RECORD,,
+microdf/tests/test_weighted_cov_corr.py,sha256=LTnFhMWnb28f7L_9kOhPiV5UlLMAUbC9OlLIaZ1XgLs,13954
+microdf_python-1.4.0.dist-info/licenses/LICENSE,sha256=uPs-ASYnzlldpf2z8jeRgQFeEH3FLhSuX0rw0OKWoDU,1067
+microdf_python-1.4.0.dist-info/METADATA,sha256=NjHgMRpJzMIc2G3IOjOOaY3YaLc-gBMStki-4LuY9d4,2305
+microdf_python-1.4.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+microdf_python-1.4.0.dist-info/top_level.txt,sha256=T2WFPTygQQMdS3GF8YpZ12DKfMGrspbZ3r7z-e3KfiM,8
+microdf_python-1.4.0.dist-info/RECORD,,