microdf-python 1.3.4__tar.gz → 1.3.5__tar.gz

{microdf_python-1.3.4 → microdf_python-1.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: microdf-python
-Version: 1.3.4
+Version: 1.3.5
 Summary: Weighted pandas DataFrames and Series for survey microdata
 Author-email: Max Ghenis <max@policyengine.org>
 License: MIT

{microdf_python-1.3.4 → microdf_python-1.3.5}/microdf/microseries.py

@@ -310,7 +310,7 @@ class MicroSeries(pd.Series):
         )
         return super().corr(other, *args, **kwargs)
 
-    def quantile(self, q: np.array) -> pd.Series:
+    def quantile(self, q: np.array, skipna: bool = True) -> pd.Series:
         """Calculates weighted quantiles of the MicroSeries.
 
         Uses the inverse CDF method: the q-th quantile is the smallest
@@ -319,6 +319,11 @@ class MicroSeries(pd.Series):
 
         :param q: Quantile(s) to calculate, must be in [0, 1].
         :type q: float or np.array
+        :param skipna: Exclude NaN values (default True). NaN sorts to the
+            end of the array, so leaving NaN rows in would let their weight
+            inflate the cumulative distribution and push the cutoff upward.
+            If False, NaN is returned whenever any value is NaN.
+        :type skipna: bool
 
         :return: Weighted quantile value(s).
         :rtype: float or pd.Series
@@ -329,12 +334,22 @@ class MicroSeries(pd.Series):
         assert np.all(quantiles >= 0) and np.all(quantiles <= 1), (
             "quantiles should be in [0, 1]"
         )
+        na_mask = pd.isna(values)
+        if not skipna and na_mask.any():
+            return (
+                np.nan
+                if np.array(q).shape == ()
+                else pd.Series(np.full(len(quantiles), np.nan), index=quantiles)
+            )
         # Drop zero-weight rows before sorting. Without this, q=0 (and
         # internal plateaus of zero weight) picked a value with 0 weight
         # that should have been skipped by the inverse CDF. E.g.
         # MicroSeries([10, 20, 30], weights=[0, 1, 1]).quantile(0)
         # returned 10 instead of 20.
-        nonzero = sample_weight > 0
+        # Drop NaN rows for the same reason: NaN sorts last, so its weight
+        # would inflate the cumulative distribution and push the cutoff up
+        # (median of [1, nan, 3] returned 3.0 instead of 1.0).
+        nonzero = (sample_weight > 0) & ~na_mask
         if not nonzero.any():
             return (
                 np.nan
@@ -359,13 +374,15 @@ class MicroSeries(pd.Series):
         return pd.Series(result, index=quantiles)
 
     @scalar_function
-    def median(self) -> float:
+    def median(self, skipna: bool = True) -> float:
         """Calculates the weighted median of the MicroSeries.
 
+        :param skipna: Exclude NaN values (default True).
+        :type skipna: bool
         :returns: The weighted median of a DataFrame's column.
         :rtype: float
         """
-        return self.quantile(0.5)
+        return self.quantile(0.5, skipna=skipna)
 
     @scalar_function
     def gini(self, negatives: Optional[str] = None) -> float:
@@ -873,6 +890,37 @@ class MicroSeriesGroupBy(pd.core.groupby.generic.SeriesGroupBy):
                     or name in MicroSeries.AGNOSTIC_FUNCTIONS
                     and is_array
                 ):
+                    if name in MicroSeries.AGNOSTIC_FUNCTIONS and not df.empty:
+                        # Concatenate values without keys: concat rejects missing
+                        # MultiIndex keys even when groupby(dropna=False) retains
+                        # them. Reuse the grouping levels and codes so missing
+                        # labels keep the same representation as scalar results.
+                        results = [
+                            via_micro_series(row, *args, **kwargs)
+                            for _, row in df.iterrows()
+                        ]
+                        result = pd.concat(results)
+                        group_index = (
+                            df.index
+                            if isinstance(df.index, pd.MultiIndex)
+                            else pd.MultiIndex.from_arrays([df.index])
+                        )
+                        quantile_codes, quantile_levels = result.index.factorize(
+                            sort=False
+                        )
+                        result.index = pd.MultiIndex(
+                            levels=[*group_index.levels, quantile_levels],
+                            codes=[
+                                codes.repeat(len(results[0]))
+                                for codes in group_index.codes
+                            ]
+                            + [quantile_codes],
+                            names=[*df.index.names, result.index.name],
+                            # Existing group codes are valid; checking would
+                            # rewrite their retained missing labels to -1.
+                            verify_integrity=False,
+                        )
+                        return result
                     result = df.apply(
                         lambda row: via_micro_series(row, *args, **kwargs),
                         axis=1,

microdf_python-1.3.5/microdf/tests/test_quantile_missing_values.py

@@ -0,0 +1,134 @@
+import microdf as mdf
+import numpy as np
+import pandas as pd
+import pytest
+
+
+def test_quantile_skips_nan():
+    """NaN weight must not inflate the cumulative distribution.
+
+    Dropping a NaN row should give the same answer as never having had
+    it: the inverse-CDF quantile of [1, nan, 3] equals that of [1, 3].
+    """
+    with_nan = mdf.MicroSeries([1.0, np.nan, 3.0], weights=[1, 1, 1])
+    without_nan = mdf.MicroSeries([1.0, 3.0], weights=[1, 1])
+    assert with_nan.median() == without_nan.median()
+    assert with_nan.quantile(0.5) == without_nan.quantile(0.5)
+
+    q = [0.25, 0.5, 0.75]
+    np.testing.assert_array_equal(
+        mdf.MicroSeries([1.0, np.nan, 3.0, 5.0], weights=[1, 1, 1, 1]).quantile(q),
+        mdf.MicroSeries([1.0, 3.0, 5.0], weights=[1, 1, 1]).quantile(q),
+    )
+
+
+def test_quantile_skipna_false_propagates_nan():
+    """Skipna=False returns NaN when any value is NaN, like mean/var."""
+    s = mdf.MicroSeries([1.0, np.nan, 3.0], weights=[1, 1, 1])
+    assert np.isnan(s.quantile(0.5, skipna=False))
+    assert np.isnan(s.median(skipna=False))
+    assert s.quantile([0.25, 0.75], skipna=False).isna().all()
+
+
+def test_quantile_all_nan_returns_nan():
+    s = mdf.MicroSeries([np.nan, np.nan], weights=[1, 1])
+    assert np.isnan(s.median())
+
+
+@pytest.mark.parametrize("skipna", [True, False])
+@pytest.mark.parametrize("q", [-0.1, 1.1, [0.5, 1.1]])
+def test_quantile_validates_bounds_with_missing_values(q, skipna):
+    series = mdf.MicroSeries([1.0, np.nan, 3.0], weights=[1, 7, 1])
+    with pytest.raises(AssertionError, match="quantiles should be in"):
+        series.quantile(q, skipna=skipna)
+
+
+@pytest.mark.parametrize("skipna", [True, False])
+@pytest.mark.parametrize("multiple_keys", [False, True])
+def test_grouped_quantiles_preserve_missing_groups(skipna, multiple_keys):
+    series = mdf.MicroSeries(
+        [1.0, np.nan, 3.0, 5.0, np.nan, np.nan], weights=[1, 1, 1, 1, 1, 1]
+    )
+    groups = ["a", "a", "b", "b", "c", "c"]
+    keys = [groups, [1, 1, 2, 2, 3, 3]] if multiple_keys else groups
+    grouped = series.groupby(keys)
+    quantiles = [0.25, 0.75]
+    result = grouped.quantile(quantiles, skipna=skipna)
+    # Scalar calls retain every group. Vector calls must retain the same
+    # groups, including the partial-NaN and all-NaN groups.
+    for quantile in quantiles:
+        pd.testing.assert_series_equal(
+            result.xs(quantile, level=-1),
+            grouped.quantile(quantile, skipna=skipna),
+        )
+    first_group = 1.0 if skipna else np.nan
+    np.testing.assert_allclose(
+        result.to_numpy(),
+        [first_group, first_group, 3.0, 5.0, np.nan, np.nan],
+        equal_nan=True,
+    )
+
+
+def test_grouped_quantiles_preserve_repeated_requests():
+    series = mdf.MicroSeries([1.0, np.nan, 3.0, 5.0], weights=[1, 1, 1, 1])
+    result = series.groupby(["a", "a", "b", "b"]).quantile([0.5, 0.5], skipna=False)
+    assert result.index.tolist() == [("a", 0.5), ("a", 0.5), ("b", 0.5), ("b", 0.5)]
+    np.testing.assert_allclose(
+        result.to_numpy(), [np.nan, np.nan, 3.0, 3.0], equal_nan=True
+    )
+
+
+@pytest.mark.parametrize("quantiles", [[0.75, 0.25], [0.5, 0.5], []])
+@pytest.mark.parametrize("skipna", [True, False])
+@pytest.mark.parametrize("sort", [True, False])
+def test_grouped_quantiles_preserve_missing_multiple_keys(quantiles, skipna, sort):
+    """Missing group keys survive alongside missing values and repeated q."""
+    frame = mdf.MicroDataFrame(
+        {
+            "region": ["north", "north", None, "south", "south"],
+            "year": [2024, 2024, 2024, np.nan, 2025],
+            "income": [10.0, np.nan, 20.0, 30.0, 40.0],
+        },
+        weights=[1, 4, 2, 3, 1],
+    )
+    grouped = frame.groupby(["region", "year"], dropna=False, sort=sort)["income"]
+    result = grouped.quantile(quantiles, skipna=skipna)
+
+    # Each retained group has one nonmissing value. With skipna=False,
+    # the north group is NaN because it also contains a missing value.
+    north = 10.0 if skipna else np.nan
+    groups = [("north", 2024.0, north)]
+    if sort:
+        groups += [
+            ("south", 2025.0, 40.0),
+            ("south", np.nan, 30.0),
+            (np.nan, 2024.0, 20.0),
+        ]
+    else:
+        groups += [
+            (np.nan, 2024.0, 20.0),
+            ("south", np.nan, 30.0),
+            ("south", 2025.0, 40.0),
+        ]
+    expected_index = pd.MultiIndex.from_tuples(
+        [(region, year, q) for region, year, _ in groups for q in quantiles],
+        names=["region", "year", None],
+    )
+    expected_values = [value for _, _, value in groups for _ in quantiles]
+    assert result.index.names == expected_index.names
+    if quantiles:
+        for level in range(3):
+            pd.testing.assert_index_equal(
+                result.index.get_level_values(level),
+                expected_index.get_level_values(level),
+            )
+    assert result.index.nlevels == 3
+    np.testing.assert_allclose(result.to_numpy(), expected_values, equal_nan=True)
+    for q in set(quantiles):
+        if quantiles.count(q) == 1:
+            selected = result.xs(q, level=-1)
+            scalar = grouped.quantile(q, skipna=skipna)
+            assert selected.index.equals(scalar.index)
+            np.testing.assert_allclose(
+                selected.to_numpy(), scalar.to_numpy(), equal_nan=True
+            )

{microdf_python-1.3.4 → microdf_python-1.3.5}/microdf_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: microdf-python
-Version: 1.3.4
+Version: 1.3.5
 Summary: Weighted pandas DataFrames and Series for survey microdata
 Author-email: Max Ghenis <max@policyengine.org>
 License: MIT

{microdf_python-1.3.4 → microdf_python-1.3.5}/microdf_python.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ microdf/tests/test_dataframe_weight_storage.py
 microdf/tests/test_microseries_dataframe.py
 microdf/tests/test_nullify_weights_index.py
 microdf/tests/test_pandas3_compatibility.py
+microdf/tests/test_quantile_missing_values.py
 microdf_python.egg-info/PKG-INFO
 microdf_python.egg-info/SOURCES.txt
 microdf_python.egg-info/dependency_links.txt

{microdf_python-1.3.4 → microdf_python-1.3.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "microdf-python"
-version = "1.3.4"
+version = "1.3.5"
 description = "Weighted pandas DataFrames and Series for survey microdata"
 readme = "README.md"
 authors = [