microdf-python 1.3.10__tar.gz → 1.4.0__tar.gz

{microdf_python-1.3.10/microdf_python.egg-info → microdf_python-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: microdf-python
-Version: 1.3.10
+Version: 1.4.0
 Summary: Weighted pandas DataFrames and Series for survey microdata
 Author-email: Max Ghenis <max@policyengine.org>
 License: MIT

{microdf_python-1.3.10 → microdf_python-1.4.0}/microdf/microseries.py

@@ -9,6 +9,41 @@ import pandas as pd
 logger = logging.getLogger(__name__)
 
 
+def _weighted_centered_vector(
+    values: np.ndarray, weights: np.ndarray
+) -> tuple[np.ndarray, int]:
+    """Return scaled sqrt-weighted deviations and their power-of-two
+    exponent."""
+    # Center relative to a maximum-weight observation: shifting by a low-weight
+    # extreme could erase differences among the influential observations.
+    # A relative mean also preserves nearby values at a large common offset.
+    reference = values[np.argmax(weights)]
+    with np.errstate(over="ignore"):
+        shifted = values - reference
+    exponent = 0
+    if np.isinf(shifted).any():
+        # Opposite finite extremes can overflow their difference. Halving is
+        # exact for those values; restore that factor in the final exponent.
+        shifted = values / 2 - reference / 2
+        exponent = 1
+    magnitude = np.max(np.abs(shifted))
+    if magnitude == 0:
+        return shifted, 0
+    _, shift = np.frexp(magnitude)
+    shifted = np.ldexp(shifted, -shift)
+    # Raise tiny mean weights by an exact common power of two so products
+    # with the scaled deviations do not underflow. Never scale down: that
+    # could discard small weights when frequencies span a wide range.
+    _, mean_weight_exponent = np.frexp(np.max(weights))
+    mean_weights = np.ldexp(weights, -min(int(mean_weight_exponent), 0))
+    shifted -= np.average(shifted, weights=mean_weights)
+    # Weight each vector before taking products, then scale again so squared
+    # deviations never accumulate raw frequencies at the original value scale.
+    shifted *= np.sqrt(weights)
+    _, weight_shift = np.frexp(np.max(np.abs(shifted)))
+    return np.ldexp(shifted, -weight_shift), exponent + int(shift) + int(weight_shift)
+
+
 def _weighted_top_share(
     values: np.ndarray, weights: np.ndarray, top_x_pct: float
 ) -> float:
@@ -315,39 +350,152 @@ class MicroSeries(pd.Series):
         v = self._weighted_variance(ddof=ddof, skipna=skipna)
         return float(np.sqrt(v)) if np.isfinite(v) else v
 
-    def cov(self, other, *args, **kwargs):
-        """Pandas ``cov`` — **unweighted**.
+    def _weighted_pair(
+        self,
+        other: pd.Series,
+        min_periods: Optional[int],
+        ddof: int,
+        skipna: bool,
+    ) -> Optional[tuple[np.ndarray, np.ndarray, np.ndarray, float]]:
+        """Align usable paired observations with their left weights."""
+        if not isinstance(other, pd.Series):
+            raise TypeError("other must be a pandas Series or MicroSeries")
+        if not isinstance(ddof, (int, np.integer)):
+            raise TypeError("ddof must be an integer")
+        if min_periods is None:
+            min_periods = 1
+        if not isinstance(min_periods, (int, np.integer)) or min_periods < 0:
+            raise ValueError("min_periods must be a nonnegative integer")
+        if len(self) == 0 or len(other) == 0:
+            return None
+
+        # Align left row positions so values and weights undergo exactly the
+        # same join, including pandas' expansion of duplicate index labels.
+        positions = pd.Series(np.arange(len(self)), index=self.index)
+        positions, right = positions.align(pd.Series(other), join="inner")
+        positions = positions.to_numpy(dtype=int)
+        x = (
+            pd.Series(self._values)
+            .iloc[positions]
+            .to_numpy(dtype=float, na_value=np.nan)
+        )
+        y = right.to_numpy(dtype=float, na_value=np.nan)
+        weights = np.asarray(self.weights, dtype=float)[positions]
+        if not np.isfinite(weights).all() or (weights < 0).any():
+            raise ValueError("frequency weights must be finite and nonnegative")
+
+        # Zero frequency means the row is absent, including for skipna=False.
+        positive = weights > 0
+        x, y, weights = x[positive], y[positive], weights[positive]
+        missing = np.isnan(x) | np.isnan(y)
+        if not skipna and missing.any():
+            return None
+        x, y, weights = x[~missing], y[~missing], weights[~missing]
+        total_weight = weights.sum()
+        if not np.isfinite(total_weight):
+            raise ValueError("the sum of frequency weights must be finite")
+        if len(x) < min_periods or total_weight == 0 or total_weight <= ddof:
+            return None
+        return (
+            x,
+            y,
+            weights,
+            float(total_weight - ddof),
+        )
 
-        MicroSeries does not yet compute weighted covariance. Emits a
-        ``UserWarning`` so callers aren't silently given an unweighted number
-        after ``.sum()`` and ``.mean()`` worked as expected. See issue tracker
-        for a weighted implementation.
+    def cov(
+        self,
+        other: pd.Series,
+        min_periods: Optional[int] = None,
+        ddof: int = 1,
+        *,
+        skipna: bool = True,
+    ) -> float:
+        """Calculate frequency-weighted covariance with another Series.
+
+        Observations align by index as in pandas, including its duplicate-
+        label join behavior. Only this Series' weights are used; weights on
+        another MicroSeries are ignored. Each aligned left weight must be
+        finite and nonnegative. Zero-weight rows are omitted.
+
+        Uses ``sum(w * (x - xmean) * (y - ymean)) / (sum(w) - ddof)``.
+        Integer weights therefore match covariance on the replicated sample.
+        Missing values are removed pairwise before computing both means.
+
+        :param other: A pandas Series or MicroSeries to align by index.
+        :param min_periods: Minimum usable aligned row pairs, not the sum of
+            frequency weights. Defaults to 1.
+        :param ddof: Degrees of freedom subtracted from the weight total.
+        :param skipna: Drop pairs with a missing value. If False, any missing
+            value in a positive-weight aligned pair produces NaN.
+        :returns: Weighted covariance, or NaN for an empty or insufficient
+            sample (including a weight total no greater than ddof).
         """
-        warnings.warn(
-            "MicroSeries.cov() falls through to pandas and is "
-            "unweighted. Use MicroSeries.var()/std() for weighted "
-            "second moments, or compute covariance manually with the "
-            "weights.",
-            UserWarning,
-            stacklevel=2,
+        pair = self._weighted_pair(other, min_periods, ddof, skipna)
+        if pair is None:
+            return np.nan
+        x, y, weights, denominator = pair
+        if not np.isfinite(x).all() or not np.isfinite(y).all():
+            return np.nan
+        x, x_exponent = _weighted_centered_vector(x, weights)
+        y, y_exponent = _weighted_centered_vector(y, weights)
+        # Combine exponents only after dividing out sum(weights) - ddof.
+        # Neither the original squared scale nor raw weighted sum need fit.
+        denominator, denominator_exponent = np.frexp(denominator)
+        return float(
+            np.ldexp(
+                np.sum(x * y) / denominator,
+                x_exponent + y_exponent - int(denominator_exponent),
+            )
         )
-        return super().cov(other, *args, **kwargs)
-
-    def corr(self, other, *args, **kwargs):
-        """Pandas ``corr`` — **unweighted**.
 
-        MicroSeries does not yet compute weighted correlation. Emits a
-        ``UserWarning`` so callers aren't silently given an unweighted number.
-        See issue tracker for a weighted implementation.
+    def corr(
+        self,
+        other: pd.Series,
+        method: str = "pearson",
+        min_periods: Optional[int] = None,
+        *,
+        ddof: int = 1,
+        skipna: bool = True,
+    ) -> float:
+        """Calculate frequency-weighted Pearson correlation.
+
+        Uses the same aligned pairs and left Series weights for covariance and
+        both variances. Weights on another MicroSeries are ignored. Weights
+        must be finite and nonnegative; zero-weight rows are omitted. Other
+        correlation methods, including callables, are unsupported.
+
+        :param other: A pandas Series or MicroSeries to align by index.
+        :param method: Only "pearson" is supported.
+        :param min_periods: Minimum usable aligned row pairs, not frequency
+            weight total. Defaults to 1.
+        :param ddof: Degrees of freedom for all three moments. It cancels from
+            the correlation but the weight total must exceed it.
+        :param skipna: Drop pairs with a missing value. If False, any missing
+            value in a positive-weight aligned pair produces NaN.
+        :returns: Weighted correlation, or NaN for an empty, insufficient, or
+            constant sample.
         """
-        warnings.warn(
-            "MicroSeries.corr() falls through to pandas and is "
-            "unweighted. Compute correlation manually with the weights "
-            "if you need the survey-weighted value.",
-            UserWarning,
-            stacklevel=2,
-        )
-        return super().corr(other, *args, **kwargs)
+        if method != "pearson":
+            raise ValueError("weighted correlation only supports method='pearson'")
+        pair = self._weighted_pair(other, min_periods, ddof, skipna)
+        if pair is None:
+            return np.nan
+        x, y, weights, _ = pair
+        if not np.isfinite(x).all() or not np.isfinite(y).all():
+            return np.nan
+        # A weighted mean can round away from identical decimal inputs.
+        # Check the retained observations exactly before subtracting it.
+        if (x == x[0]).all() or (y == y[0]).all():
+            return np.nan
+        x, _ = _weighted_centered_vector(x, weights)
+        y, _ = _weighted_centered_vector(y, weights)
+        x_ss = np.sum(x * x)
+        y_ss = np.sum(y * y)
+        if x_ss == 0 or y_ss == 0:
+            return np.nan
+        result = np.sum(x * y) / (np.sqrt(x_ss) * np.sqrt(y_ss))
+        return float(np.clip(result, -1.0, 1.0))
 
     def quantile(self, q: np.array, skipna: bool = True) -> pd.Series:
         """Calculates weighted quantiles of the MicroSeries.

{microdf_python-1.3.10 → microdf_python-1.4.0}/microdf/tests/test_microseries_dataframe.py

@@ -744,28 +744,6 @@ def test_std_var_are_weighted() -> None:
     )
 
 
-def test_cov_corr_warn_when_fallthrough() -> None:
-    """Regression: cov/corr silently returned unweighted pandas values.
-
-    They still fall through to pandas (a weighted impl is a separate issue) but
-    now emit a UserWarning so callers aren't misled.
-    """
-    s1 = mdf.MicroSeries([1, 2, 3], weights=[1, 1, 1])
-    s2 = mdf.MicroSeries([2, 4, 6], weights=[1, 1, 1])
-
-    with warnings.catch_warnings(record=True) as w:
-        warnings.simplefilter("always")
-        _ = s1.cov(s2)
-        msgs = [str(x.message) for x in w if issubclass(x.category, UserWarning)]
-        assert any("unweighted" in m.lower() for m in msgs)
-
-    with warnings.catch_warnings(record=True) as w:
-        warnings.simplefilter("always")
-        _ = s1.corr(s2)
-        msgs = [str(x.message) for x in w if issubclass(x.category, UserWarning)]
-        assert any("unweighted" in m.lower() for m in msgs)
-
-
 def test_count_skips_nan_by_default() -> None:
     """Regression: ``count()`` included NaN-row weight, contrary to pandas.
 

microdf_python-1.4.0/microdf/tests/test_weighted_cov_corr.py

@@ -0,0 +1,343 @@
+import warnings
+from decimal import Decimal, localcontext
+from fractions import Fraction
+from itertools import permutations
+
+import numpy as np
+import pandas as pd
+import pytest
+
+import microdf as mdf
+
+
+def replicated_moments(x, y, weights, ddof=1):
+    """Independent frequency-weight oracle: expand to an ordinary sample."""
+    repeated_x = np.repeat(np.asarray(x, dtype=float), weights)
+    repeated_y = np.repeat(np.asarray(y, dtype=float), weights)
+    return (
+        np.cov(repeated_x, repeated_y, ddof=ddof)[0, 1],
+        np.corrcoef(repeated_x, repeated_y)[0, 1],
+    )
+
+
+@pytest.mark.parametrize("ddof", [0, 1, 2])
+def test_cov_corr_match_replicated_frequency_sample(ddof):
+    x, y, weights = [1, 4, 8], [5, 2, 9], [1, 3, 2]
+    left = mdf.MicroSeries(x, weights=weights)
+    right = pd.Series(y)
+    expected_cov, expected_corr = replicated_moments(x, y, weights, ddof)
+    with warnings.catch_warnings(record=True) as caught:
+        warnings.simplefilter("always")
+        assert left.cov(right, ddof=ddof) == pytest.approx(expected_cov)
+        assert left.corr(right, ddof=ddof) == pytest.approx(expected_corr)
+    assert not any("unweighted" in str(item.message).lower() for item in caught)
+
+
+def test_cov_corr_align_indices_and_use_only_left_weights():
+    left = mdf.MicroSeries([1, 4, 8], index=["a", "b", "c"], weights=[1, 3, 2])
+    right = mdf.MicroSeries(
+        [9, 5, 2, 100], index=["c", "a", "b", "d"], weights=[99, 1, 1, 9]
+    )
+    expected_cov, expected_corr = replicated_moments([1, 4, 8], [5, 2, 9], [1, 3, 2])
+    assert left.cov(right) == pytest.approx(expected_cov)
+    assert left.corr(right) == pytest.approx(expected_corr)
+    assert right.weights.tolist() == [99, 1, 1, 9]
+
+
+def test_cov_corr_use_same_pairwise_nonmissing_sample():
+    left = mdf.MicroSeries(
+        [1, np.nan, 5, 9, 20], index=list("abcde"), weights=[1, 9, 2, 3, 7]
+    )
+    right = pd.Series([2, 4, np.nan, 8, 30], index=list("abcdf"))
+    expected_cov, expected_corr = replicated_moments([1, 9], [2, 8], [1, 3])
+    assert left.cov(right) == pytest.approx(expected_cov)
+    assert left.corr(right) == pytest.approx(expected_corr)
+    assert np.isnan(left.cov(right, skipna=False))
+    assert np.isnan(left.corr(right, skipna=False))
+
+
+@pytest.mark.parametrize("same_index", [True, False])
+def test_cov_corr_follow_pandas_duplicate_index_alignment(same_index):
+    left = mdf.MicroSeries([1, 4, 7], index=["a", "a", "b"], weights=[1, 3, 2])
+    if same_index:
+        right = pd.Series([2, 3, 8], index=["a", "a", "b"])
+        x, y, weights = [1, 4, 7], [2, 3, 8], [1, 3, 2]
+    else:
+        right = pd.Series([2, 5, 8], index=["a", "b", "b"])
+        # The shared a/b labels join, repeating the left row weight per pair.
+        x, y, weights = [1, 4, 7, 7], [2, 2, 5, 8], [1, 3, 2, 2]
+    expected_cov, expected_corr = replicated_moments(x, y, weights)
+    assert left.cov(right) == pytest.approx(expected_cov)
+    assert left.corr(right) == pytest.approx(expected_corr)
+
+
+def test_zero_weight_rows_do_not_enter_pairwise_sample():
+    left = mdf.MicroSeries([1, np.nan, 5, 1000], weights=[2, 0, 1, 0])
+    right = pd.Series([3, np.nan, 9, -1000])
+    expected_cov, expected_corr = replicated_moments([1, 5], [3, 9], [2, 1])
+    assert left.cov(right, skipna=False) == pytest.approx(expected_cov)
+    assert left.corr(right, skipna=False) == pytest.approx(expected_corr)
+    assert np.isnan(left.cov(right, min_periods=3))
+    assert np.isnan(left.corr(right, min_periods=3))
+
+
+def test_min_periods_counts_usable_rows_separately_from_frequency_weight():
+    left = mdf.MicroSeries([1, 5], weights=[10, 20])
+    right = pd.Series([3, 9])
+    expected_cov, expected_corr = replicated_moments([1, 5], [3, 9], [10, 20], ddof=0)
+    # Existing pandas positional arguments retain their order.
+    assert left.cov(right, 2, 0) == pytest.approx(expected_cov)
+    assert left.corr(right, "pearson", 2, ddof=0) == pytest.approx(expected_corr)
+    assert np.isnan(left.cov(right, min_periods=3))
+    assert np.isnan(left.corr(right, min_periods=3))
+
+
+@pytest.mark.parametrize(
+    "values,other,weights",
+    [([], [], []), ([np.nan], [1], [3]), ([1], [2], [0]), ([1], [2], [1])],
+)
+def test_cov_corr_return_nan_when_sample_is_insufficient(values, other, weights):
+    left = mdf.MicroSeries(values, weights=weights, dtype=float)
+    right = pd.Series(other, dtype=float)
+    assert np.isnan(left.cov(right))
+    assert np.isnan(left.corr(right))
+
+
+def test_cov_corr_no_index_overlap():
+    left = mdf.MicroSeries([1, 2], index=["a", "b"], weights=[1, 2])
+    right = pd.Series([3, 4], index=["c", "d"])
+    assert np.isnan(left.cov(right))
+    assert np.isnan(left.corr(right))
+
+
+def test_cov_corr_constant_and_frequency_singleton():
+    left = mdf.MicroSeries([4, 4], weights=[2, 3])
+    right = pd.Series([1, 5])
+    assert left.cov(right) == 0
+    assert np.isnan(left.corr(right))
+    singleton = mdf.MicroSeries([4], weights=[3])
+    assert singleton.cov(pd.Series([2])) == 0
+    assert np.isnan(singleton.corr(pd.Series([2])))
+    assert np.isnan(singleton.cov(pd.Series([2]), ddof=3))
+    assert np.isnan(singleton.corr(pd.Series([2]), ddof=3))
+
+
+def test_cov_corr_nullable_numeric_data():
+    left = mdf.MicroSeries(pd.Series([1, pd.NA, 4], dtype="Int64"), weights=[2, 9, 3])
+    right = pd.Series([3, 8, 7], dtype="Float64")
+    expected_cov, expected_corr = replicated_moments([1, 4], [3, 7], [2, 3])
+    assert left.cov(right) == pytest.approx(expected_cov)
+    assert left.corr(right) == pytest.approx(expected_corr)
+
+
+@pytest.mark.parametrize("method", ["spearman", "kendall", lambda x, y: 1.0])
+def test_non_pearson_methods_are_explicitly_unsupported(method):
+    left = mdf.MicroSeries([1, 2, 3], weights=[1, 2, 3])
+    with pytest.raises(ValueError, match="pearson"):
+        left.corr(pd.Series([4, 2, 5]), method=method)
+
+
+@pytest.mark.parametrize("weights", [[1, -1], [1, np.nan], [1, np.inf]])
+def test_cov_corr_reject_invalid_frequency_weights(weights):
+    left = mdf.MicroSeries([1, 2], weights=weights)
+    for method in (left.cov, left.corr):
+        with pytest.raises(ValueError, match="weights"):
+            method(pd.Series([2, 4]))
+
+
+def test_binary_statistics_not_in_dataframe_aggregation_factories():
+    assert "cov" not in mdf.MicroSeries.FUNCTIONS
+    assert "corr" not in mdf.MicroSeries.FUNCTIONS
+
+
+@pytest.mark.parametrize(
+    "x,y",
+    [([0.1, 0.1], [0, 1]), ([0, 1], [0.1, 0.1]), ([0.1, 0.1], [0.1, 0.1])],
+)
+@pytest.mark.parametrize("with_filtered_rows", [False, True])
+def test_corr_exact_decimal_constants_return_nan(x, y, with_filtered_rows):
+    # Unequal weights can round the mean away from the identical 0.1 values.
+    # Constant detection must inspect usable observations before centering.
+    weights = [1, 2]
+    if with_filtered_rows:
+        x = x + [9, np.nan]
+        y = y + [7, 4]
+        weights = weights + [0, 3]
+    left = mdf.MicroSeries(x, weights=weights)
+    assert np.isnan(left.corr(pd.Series(y)))
+
+
+def test_corr_does_not_treat_nearby_distinct_values_as_constant():
+    x = [0.1, np.nextafter(0.1, np.inf)]
+    left = mdf.MicroSeries(x, weights=[1, 2])
+    assert np.isfinite(left.corr(pd.Series([0, 1])))
+    assert np.isfinite(mdf.MicroSeries([0, 1], weights=[1, 2]).corr(pd.Series(x)))
+
+
+def exact_weighted_moments(x, y, weights, ddof=1):
+    """Compute moments of the actual input floats with exact rational
+    arithmetic."""
+    x, y, weights = [
+        [Fraction(float(value)) for value in values] for values in (x, y, weights)
+    ]
+    total = sum(weights)
+    xmean = sum(w * value for w, value in zip(weights, x)) / total
+    ymean = sum(w * value for w, value in zip(weights, y)) / total
+    xy = sum(w * (a - xmean) * (b - ymean) for a, b, w in zip(x, y, weights))
+    xx = sum(w * (value - xmean) ** 2 for value, w in zip(x, weights))
+    yy = sum(w * (value - ymean) ** 2 for value, w in zip(y, weights))
+    with localcontext() as context:
+        context.prec = 100
+        product = xx * yy
+        correlation = (Decimal(xy.numerator) / Decimal(xy.denominator)) / (
+            Decimal(product.numerator) / Decimal(product.denominator)
+        ).sqrt()
+    return float(xy / (total - ddof)), float(correlation)
+
+
+@pytest.mark.parametrize("ddof", [0, 1, 2])
+@pytest.mark.parametrize("swap", [False, True])
+def test_cov_corr_preserve_small_differences_at_large_offsets(ddof, swap):
+    # Two distinct points are perfectly linear even two float steps apart.
+    x = np.array([1e12 - 2**-13, 1e12 + 2**-13])
+    y = np.array([0.0, 1.0])
+    if swap:
+        x, y = y, x
+    weights = [1, 2]
+    expected = exact_weighted_moments(x, y, weights, ddof)
+    assert expected[1] == 1.0
+    for shifted_x, shifted_y in [(x, y), (x - x[0], y - y[0])]:
+        left = mdf.MicroSeries(shifted_x, weights=weights)
+        right = pd.Series(shifted_y)
+        np.testing.assert_allclose(
+            [left.cov(right, ddof=ddof), left.corr(right, ddof=ddof)],
+            expected,
+            rtol=2e-15,
+            atol=0,
+        )
+
+
+@pytest.mark.parametrize("frequency", [1, 1_000_000])
+@pytest.mark.parametrize("ddof", [0, 1, 2])
+def test_cov_corr_large_finite_values_do_not_overflow_raw_frequencies(frequency, ddof):
+    x = np.array([1.0, 2.0, 3.0]) * 1e153
+    weights = [frequency] * 3
+    expected = exact_weighted_moments(x, x, weights, ddof)
+    assert np.isfinite(expected).all()
+    assert expected[1] == 1.0
+    left = mdf.MicroSeries(x, weights=weights)
+    with np.errstate(over="raise", invalid="raise"):
+        actual = [left.cov(pd.Series(x), ddof=ddof), left.corr(pd.Series(x), ddof=ddof)]
+    np.testing.assert_allclose(actual, expected, rtol=2e-15, atol=0)
+
+
+@pytest.mark.parametrize("frequency", [0.5, 1, 1_000_000])
+@pytest.mark.parametrize("ddof", [0, 1, 2])
+@pytest.mark.parametrize("scales", [(1.0, 1.0), (1e153, -1e153), (1e200, 1e-200)])
+def test_cov_corr_preserve_frequency_correction_across_value_scales(
+    frequency, ddof, scales
+):
+    x = np.array([1.0, 4.0, 8.0]) * scales[0]
+    y = np.array([5.0, 2.0, 9.0]) * scales[1]
+    weights = np.array([1, 3, 2]) * frequency
+    expected = exact_weighted_moments(x, y, weights, ddof)
+    left = mdf.MicroSeries(x, weights=weights)
+    with np.errstate(over="raise", invalid="raise"):
+        actual = [left.cov(pd.Series(y), ddof=ddof), left.corr(pd.Series(y), ddof=ddof)]
+    np.testing.assert_allclose(actual, expected, rtol=3e-15, atol=0)
+
+
+@pytest.mark.parametrize("huge", [1e16, 1e20])
+@pytest.mark.parametrize("order", list(permutations(range(3))))
+def test_cov_corr_low_weight_extreme_does_not_make_result_depend_on_row_order(
+    huge, order
+):
+    # The large observation contributes to covariance, but using it as the
+    # centering origin must not erase the difference between 1 and 2.
+    x = np.array([huge, 1.0, 2.0])
+    y = np.array([0.0, 1.0, 2.0])
+    weights = np.array([1 / huge, 1.0, 1.0])
+    expected = exact_weighted_moments(x, y, weights, ddof=0)
+    order = list(order)
+    left = mdf.MicroSeries(x[order], weights=weights[order])
+    right = pd.Series(y[order])
+    np.testing.assert_allclose(
+        [left.cov(right, ddof=0), left.corr(right, ddof=0)],
+        expected,
+        rtol=3e-15,
+        atol=0,
+    )
+
+
+def test_cov_corr_smallest_common_positive_weight_cancels_from_population_moments():
+    # The common positive weight cancels: xy = 1, xx = yy = 2 for
+    # centered observations [-1, 0, 1] and [-1, 1, 0].
+    x, y = [1, 2, 3], [3, 5, 4]
+    weights = [np.nextafter(0.0, 1.0)] * 3
+    expected = exact_weighted_moments(x, y, weights, ddof=0)
+    assert expected == (1 / 3, 0.5)
+    left = mdf.MicroSeries(x, weights=weights)
+    right = pd.Series(y)
+    np.testing.assert_allclose(
+        [left.cov(right, ddof=0), left.corr(right, ddof=0)],
+        expected,
+        rtol=3e-15,
+        atol=0,
+    )
+
+
+@pytest.mark.parametrize(
+    "frequency", [np.nextafter(0.0, 1.0), np.finfo(float).tiny, 1.0]
+)
+@pytest.mark.parametrize("multipliers", [[1, 2, 3], [1, 1, 2]])
+@pytest.mark.parametrize("order", list(permutations(range(3))))
+def test_cov_corr_unequal_tiny_and_normal_weights_match_exact_moments(
+    frequency, multipliers, order
+):
+    x, y = np.array([1.0, 2.0, 3.0]), np.array([3.0, 5.0, 4.0])
+    weights = np.array(multipliers) * frequency
+    expected = exact_weighted_moments(x, y, weights, ddof=0)
+    order = list(order)
+    left = mdf.MicroSeries(x[order], weights=weights[order])
+    right = pd.Series(y[order])
+    np.testing.assert_allclose(
+        [left.cov(right, ddof=0), left.corr(right, ddof=0)],
+        expected,
+        rtol=3e-15,
+        atol=0,
+    )
+
+
+@pytest.mark.parametrize(
+    "x,y",
+    [
+        (np.array([1, 2, 3]) * 1e153, np.array([3, 5, 4]) * -1e153),
+        (np.array([1, 2, 3]) * 1e200, np.array([3, 5, 4]) * 1e-200),
+        ([1e12 - 2**-13, 1e12, 1e12 + 2**-13], [3, 5, 4]),
+    ],
+)
+def test_cov_corr_subnormal_weights_preserve_extreme_value_scales(x, y):
+    weights = np.array([1, 2, 3]) * np.nextafter(0.0, 1.0)
+    expected = exact_weighted_moments(x, y, weights, ddof=0)
+    left = mdf.MicroSeries(x, weights=weights)
+    right = pd.Series(y)
+    with np.errstate(over="raise", invalid="raise"):
+        actual = [left.cov(right, ddof=0), left.corr(right, ddof=0)]
+    np.testing.assert_allclose(actual, expected, rtol=3e-15, atol=0)
+
+
+@pytest.mark.parametrize("frequency", [np.nextafter(0.0, 1.0), 0.5])
+@pytest.mark.parametrize("ddof", [0, 1, 2])
+def test_cov_corr_center_weight_scaling_preserves_original_frequency_ddof(
+    frequency, ddof
+):
+    x, y = [1, 2, 3], [3, 5, 4]
+    weights = np.array([1, 2, 3]) * frequency
+    left = mdf.MicroSeries(x, weights=weights)
+    right = pd.Series(y)
+    actual = [left.cov(right, ddof=ddof), left.corr(right, ddof=ddof)]
+    if weights.sum() <= ddof:
+        assert np.isnan(actual).all()
+    else:
+        expected = exact_weighted_moments(x, y, weights, ddof=ddof)
+        np.testing.assert_allclose(actual, expected, rtol=3e-15, atol=0)

{microdf_python-1.3.10 → microdf_python-1.4.0/microdf_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: microdf-python
-Version: 1.3.10
+Version: 1.4.0
 Summary: Weighted pandas DataFrames and Series for survey microdata
 Author-email: Max Ghenis <max@policyengine.org>
 License: MIT

{microdf_python-1.3.10 → microdf_python-1.4.0}/microdf_python.egg-info/SOURCES.txt

@@ -14,6 +14,7 @@ microdf/tests/test_quantile_missing_values.py
 microdf/tests/test_serialization.py
 microdf/tests/test_sum_axes.py
 microdf/tests/test_version_metadata.py
+microdf/tests/test_weighted_cov_corr.py
 microdf_python.egg-info/PKG-INFO
 microdf_python.egg-info/SOURCES.txt
 microdf_python.egg-info/dependency_links.txt

{microdf_python-1.3.10 → microdf_python-1.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "microdf-python"
-version = "1.3.10"
+version = "1.4.0"
 description = "Weighted pandas DataFrames and Series for survey microdata"
 readme = "README.md"
 authors = [