microdf-python 1.3.4__tar.gz → 1.3.6__tar.gz

{microdf_python-1.3.4 → microdf_python-1.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: microdf-python
-Version: 1.3.4
+Version: 1.3.6
 Summary: Weighted pandas DataFrames and Series for survey microdata
 Author-email: Max Ghenis <max@policyengine.org>
 License: MIT

{microdf_python-1.3.4 → microdf_python-1.3.6}/microdf/microdataframe.py

@@ -150,9 +150,13 @@ class MicroDataFrame(pd.DataFrame):
                 if pd.api.types.is_numeric_dtype(self[col]):
                     try:
                         results[col] = getattr(self[col], name)(*args, **kwargs)
-                    except Exception:
-                        # Skip columns that can't be aggregated
-                        pass
+                    except TypeError as exc:
+                        # Skip columns whose dtype can't take this aggregation.
+                        # Deliberately narrow: catching every Exception here also
+                        # swallowed real errors (e.g. the ValueError from
+                        # gini(negatives=...)) and returned a silently truncated
+                        # result instead of raising.
+                        logger.debug("skipping column %s in %s: %s", col, name, exc)
             return pd.Series(results)
 
         return fn
@@ -173,9 +177,13 @@ class MicroDataFrame(pd.DataFrame):
                         result = getattr(self[col], name)(*args, **kwargs)
                         results.append(result)
                         columns.append(col)
-                    except Exception:
-                        # Skip columns that can't be aggregated
-                        pass
+                    except TypeError as exc:
+                        # Skip columns whose dtype can't take this aggregation.
+                        # Deliberately narrow: catching every Exception here also
+                        # swallowed real errors (e.g. the ValueError from
+                        # gini(negatives=...)) and returned a silently truncated
+                        # result instead of raising.
+                        logger.debug("skipping column %s in %s: %s", col, name, exc)
 
             if results:
                 df = pd.DataFrame(results)
@@ -208,9 +216,13 @@ class MicroDataFrame(pd.DataFrame):
                             result = getattr(self[col], name)(*args, **kwargs)
                             results.append(result)
                             columns.append(col)
-                        except Exception:
-                            # Skip columns that can't be aggregated
-                            pass
+                        except TypeError as exc:
+                            # Skip columns whose dtype can't take this aggregation.
+                            # Deliberately narrow: catching every Exception here also
+                            # swallowed real errors (e.g. the ValueError from
+                            # gini(negatives=...)) and returned a silently truncated
+                            # result instead of raising.
+                            logger.debug("skipping column %s in %s: %s", col, name, exc)
 
                 if results:
                     df = pd.DataFrame(results)
@@ -225,9 +237,13 @@ class MicroDataFrame(pd.DataFrame):
                     if pd.api.types.is_numeric_dtype(self[col]):
                         try:
                             results[col] = getattr(self[col], name)(*args, **kwargs)
-                        except Exception:
-                            # Skip columns that can't be aggregated
-                            pass
+                        except TypeError as exc:
+                            # Skip columns whose dtype can't take this aggregation.
+                            # Deliberately narrow: catching every Exception here also
+                            # swallowed real errors (e.g. the ValueError from
+                            # gini(negatives=...)) and returned a silently truncated
+                            # result instead of raising.
+                            logger.debug("skipping column %s in %s: %s", col, name, exc)
                 return pd.Series(results)
 
         return fn
@@ -839,9 +855,13 @@ class MicroDataFrameGroupBy(pd.core.groupby.generic.DataFrameGroupBy):
                             results[col] = getattr(getattr(self, col), name)(
                                 *args, **kwargs
                             )
-                        except Exception:
-                            # Skip columns that can't be aggregated
-                            pass
+                        except TypeError as exc:
+                            # Skip columns whose dtype can't take this aggregation.
+                            # Deliberately narrow: catching every Exception here also
+                            # swallowed real errors (e.g. the ValueError from
+                            # gini(negatives=...)) and returned a silently truncated
+                            # result instead of raising.
+                            logger.debug("skipping column %s in %s: %s", col, name, exc)
                     # Return plain DataFrame - aggregated results don't have
                     # per-row weights (weights were already applied)
                     return pd.DataFrame(results) if results else pd.DataFrame()
@@ -859,9 +879,13 @@ class MicroDataFrameGroupBy(pd.core.groupby.generic.DataFrameGroupBy):
                             results[col] = getattr(getattr(self, col), name)(
                                 *args, **kwargs
                             )
-                        except Exception:
-                            # Skip columns that can't be aggregated
-                            pass
+                        except TypeError as exc:
+                            # Skip columns whose dtype can't take this aggregation.
+                            # Deliberately narrow: catching every Exception here also
+                            # swallowed real errors (e.g. the ValueError from
+                            # gini(negatives=...)) and returned a silently truncated
+                            # result instead of raising.
+                            logger.debug("skipping column %s in %s: %s", col, name, exc)
                     # Return plain DataFrame - aggregated results don't have
                     # per-row weights (weights were already applied)
                     return pd.DataFrame(results) if results else pd.DataFrame()
@@ -921,8 +945,15 @@ class MicroDataFrameGroupBy(pd.core.groupby.generic.DataFrameGroupBy):
                                 results[col] = getattr(getattr(res, col), name)(
                                     *args, **kwargs
                                 )
-                            except Exception:
-                                pass
+                            except TypeError as exc:
+                                # Skip columns whose dtype can't take this aggregation.
+                                # Deliberately narrow: catching every Exception here also
+                                # swallowed real errors (e.g. the ValueError from
+                                # gini(negatives=...)) and returned a silently truncated
+                                # result instead of raising.
+                                logger.debug(
+                                    "skipping column %s in %s: %s", col, name, exc
+                                )
                         # Return plain DataFrame - aggregated results don't
                         # have per-row weights (weights were already applied)
                         return pd.DataFrame(results) if results else pd.DataFrame()
@@ -940,8 +971,15 @@ class MicroDataFrameGroupBy(pd.core.groupby.generic.DataFrameGroupBy):
                                 results[col] = getattr(getattr(res, col), name)(
                                     *args, **kwargs
                                 )
-                            except Exception:
-                                pass
+                            except TypeError as exc:
+                                # Skip columns whose dtype can't take this aggregation.
+                                # Deliberately narrow: catching every Exception here also
+                                # swallowed real errors (e.g. the ValueError from
+                                # gini(negatives=...)) and returned a silently truncated
+                                # result instead of raising.
+                                logger.debug(
+                                    "skipping column %s in %s: %s", col, name, exc
+                                )
                         # Return plain DataFrame - aggregated results don't
                         # have per-row weights (weights were already applied)
                         return pd.DataFrame(results) if results else pd.DataFrame()

{microdf_python-1.3.4 → microdf_python-1.3.6}/microdf/microseries.py

@@ -310,7 +310,7 @@ class MicroSeries(pd.Series):
         )
         return super().corr(other, *args, **kwargs)
 
-    def quantile(self, q: np.array) -> pd.Series:
+    def quantile(self, q: np.array, skipna: bool = True) -> pd.Series:
         """Calculates weighted quantiles of the MicroSeries.
 
         Uses the inverse CDF method: the q-th quantile is the smallest
@@ -319,6 +319,11 @@ class MicroSeries(pd.Series):
 
         :param q: Quantile(s) to calculate, must be in [0, 1].
         :type q: float or np.array
+        :param skipna: Exclude NaN values (default True). NaN sorts to the
+            end of the array, so leaving NaN rows in would let their weight
+            inflate the cumulative distribution and push the cutoff upward.
+            If False, NaN is returned whenever any value is NaN.
+        :type skipna: bool
 
         :return: Weighted quantile value(s).
         :rtype: float or pd.Series
@@ -329,12 +334,22 @@ class MicroSeries(pd.Series):
         assert np.all(quantiles >= 0) and np.all(quantiles <= 1), (
             "quantiles should be in [0, 1]"
         )
+        na_mask = pd.isna(values)
+        if not skipna and na_mask.any():
+            return (
+                np.nan
+                if np.array(q).shape == ()
+                else pd.Series(np.full(len(quantiles), np.nan), index=quantiles)
+            )
         # Drop zero-weight rows before sorting. Without this, q=0 (and
         # internal plateaus of zero weight) picked a value with 0 weight
         # that should have been skipped by the inverse CDF. E.g.
         # MicroSeries([10, 20, 30], weights=[0, 1, 1]).quantile(0)
         # returned 10 instead of 20.
-        nonzero = sample_weight > 0
+        # Drop NaN rows for the same reason: NaN sorts last, so its weight
+        # would inflate the cumulative distribution and push the cutoff up
+        # (median of [1, nan, 3] returned 3.0 instead of 1.0).
+        nonzero = (sample_weight > 0) & ~na_mask
         if not nonzero.any():
             return (
                 np.nan
@@ -359,13 +374,15 @@ class MicroSeries(pd.Series):
         return pd.Series(result, index=quantiles)
 
     @scalar_function
-    def median(self) -> float:
+    def median(self, skipna: bool = True) -> float:
         """Calculates the weighted median of the MicroSeries.
 
+        :param skipna: Exclude NaN values (default True).
+        :type skipna: bool
         :returns: The weighted median of a DataFrame's column.
         :rtype: float
         """
-        return self.quantile(0.5)
+        return self.quantile(0.5, skipna=skipna)
 
     @scalar_function
     def gini(self, negatives: Optional[str] = None) -> float:
@@ -386,8 +403,9 @@ class MicroSeries(pd.Series):
         w = np.asarray(self.weights.values, dtype=float)
         if negatives == "zero":
             x = np.where(x < 0, 0.0, x)
-        elif negatives == "shift" and len(x) > 0 and np.amin(x) < 0:
-            x = x - np.amin(x)
+        elif negatives == "shift":
+            if len(x) > 0 and np.amin(x) < 0:
+                x = x - np.amin(x)
         elif negatives is not None:
             raise ValueError(
                 f"Unknown negatives option {negatives!r}; expected "
@@ -873,6 +891,37 @@ class MicroSeriesGroupBy(pd.core.groupby.generic.SeriesGroupBy):
                     or name in MicroSeries.AGNOSTIC_FUNCTIONS
                     and is_array
                 ):
+                    if name in MicroSeries.AGNOSTIC_FUNCTIONS and not df.empty:
+                        # Concatenate values without keys: concat rejects missing
+                        # MultiIndex keys even when groupby(dropna=False) retains
+                        # them. Reuse the grouping levels and codes so missing
+                        # labels keep the same representation as scalar results.
+                        results = [
+                            via_micro_series(row, *args, **kwargs)
+                            for _, row in df.iterrows()
+                        ]
+                        result = pd.concat(results)
+                        group_index = (
+                            df.index
+                            if isinstance(df.index, pd.MultiIndex)
+                            else pd.MultiIndex.from_arrays([df.index])
+                        )
+                        quantile_codes, quantile_levels = result.index.factorize(
+                            sort=False
+                        )
+                        result.index = pd.MultiIndex(
+                            levels=[*group_index.levels, quantile_levels],
+                            codes=[
+                                codes.repeat(len(results[0]))
+                                for codes in group_index.codes
+                            ]
+                            + [quantile_codes],
+                            names=[*df.index.names, result.index.name],
+                            # Existing group codes are valid; checking would
+                            # rewrite their retained missing labels to -1.
+                            verify_integrity=False,
+                        )
+                        return result
                     result = df.apply(
                         lambda row: via_micro_series(row, *args, **kwargs),
                         axis=1,

microdf_python-1.3.6/microdf/tests/test_aggregation_errors.py

@@ -0,0 +1,57 @@
+import microdf as mdf
+import numpy as np
+import pandas as pd
+import pytest
+
+
+def test_aggregation_surfaces_real_errors():
+    """A genuine argument error must raise, not be swallowed per column."""
+    df = mdf.MicroDataFrame(pd.DataFrame({"x": [1, 2, 3]}), weights=[1, 1, 1])
+    with pytest.raises(ValueError, match="Unknown negatives option"):
+        df.gini(negatives="bogus")
+
+
+def test_aggregation_still_skips_non_numeric_columns():
+    """Narrowing the guard must not change which columns aggregate."""
+    df = mdf.MicroDataFrame(
+        pd.DataFrame(
+            {
+                "x": [1, 2, 3],
+                "s": ["a", "b", "c"],
+                "dt": pd.to_datetime(["2020-01-01"] * 3),
+            }
+        ),
+        weights=[1, 2, 3],
+    )
+    assert list(df.sum().index) == ["x"]
+    assert df.sum()["x"] == 14
+    assert list(df.mean().index) == ["x"]
+
+
+def test_gini_shift_accepts_nonnegative_and_negative_columns():
+    frame = mdf.MicroDataFrame(
+        {"positive": [1, 2, 3], "negative": [-1, 1, 3]}, weights=[1, 1, 1]
+    )
+    # Shift leaves [1, 2, 3] unchanged and changes [-1, 1, 3] to [0, 2, 4].
+    # The pairwise-difference Ginis are 2/9 and 4/9, respectively.
+    expected = pd.Series({"positive": 2 / 9, "negative": 4 / 9})
+    pd.testing.assert_series_equal(frame.gini(negatives="shift"), expected)
+
+
+@pytest.mark.parametrize("selected", [False, True])
+def test_grouped_gini_shift_accepts_positive_groups(selected):
+    frame = mdf.MicroDataFrame(
+        {"group": ["a", "a", "a", "b", "b", "b"], "value": [1, 2, 3, -1, 1, 3]},
+        weights=[1, 1, 1, 1, 1, 1],
+    )
+    grouped = frame.groupby("group")
+    if selected:
+        grouped = grouped[["value"]]
+    expected = pd.DataFrame(
+        {"value": [2 / 9, 4 / 9]}, index=pd.Index(["a", "b"], name="group")
+    )
+    pd.testing.assert_frame_equal(grouped.gini(negatives="shift"), expected)
+
+
+def test_gini_shift_accepts_empty_series():
+    assert np.isnan(mdf.MicroSeries([], weights=[]).gini(negatives="shift"))

microdf_python-1.3.6/microdf/tests/test_quantile_missing_values.py

@@ -0,0 +1,134 @@
+import microdf as mdf
+import numpy as np
+import pandas as pd
+import pytest
+
+
+def test_quantile_skips_nan():
+    """NaN weight must not inflate the cumulative distribution.
+
+    Dropping a NaN row should give the same answer as never having had
+    it: the inverse-CDF quantile of [1, nan, 3] equals that of [1, 3].
+    """
+    with_nan = mdf.MicroSeries([1.0, np.nan, 3.0], weights=[1, 1, 1])
+    without_nan = mdf.MicroSeries([1.0, 3.0], weights=[1, 1])
+    assert with_nan.median() == without_nan.median()
+    assert with_nan.quantile(0.5) == without_nan.quantile(0.5)
+
+    q = [0.25, 0.5, 0.75]
+    np.testing.assert_array_equal(
+        mdf.MicroSeries([1.0, np.nan, 3.0, 5.0], weights=[1, 1, 1, 1]).quantile(q),
+        mdf.MicroSeries([1.0, 3.0, 5.0], weights=[1, 1, 1]).quantile(q),
+    )
+
+
+def test_quantile_skipna_false_propagates_nan():
+    """Skipna=False returns NaN when any value is NaN, like mean/var."""
+    s = mdf.MicroSeries([1.0, np.nan, 3.0], weights=[1, 1, 1])
+    assert np.isnan(s.quantile(0.5, skipna=False))
+    assert np.isnan(s.median(skipna=False))
+    assert s.quantile([0.25, 0.75], skipna=False).isna().all()
+
+
+def test_quantile_all_nan_returns_nan():
+    s = mdf.MicroSeries([np.nan, np.nan], weights=[1, 1])
+    assert np.isnan(s.median())
+
+
+@pytest.mark.parametrize("skipna", [True, False])
+@pytest.mark.parametrize("q", [-0.1, 1.1, [0.5, 1.1]])
+def test_quantile_validates_bounds_with_missing_values(q, skipna):
+    series = mdf.MicroSeries([1.0, np.nan, 3.0], weights=[1, 7, 1])
+    with pytest.raises(AssertionError, match="quantiles should be in"):
+        series.quantile(q, skipna=skipna)
+
+
+@pytest.mark.parametrize("skipna", [True, False])
+@pytest.mark.parametrize("multiple_keys", [False, True])
+def test_grouped_quantiles_preserve_missing_groups(skipna, multiple_keys):
+    series = mdf.MicroSeries(
+        [1.0, np.nan, 3.0, 5.0, np.nan, np.nan], weights=[1, 1, 1, 1, 1, 1]
+    )
+    groups = ["a", "a", "b", "b", "c", "c"]
+    keys = [groups, [1, 1, 2, 2, 3, 3]] if multiple_keys else groups
+    grouped = series.groupby(keys)
+    quantiles = [0.25, 0.75]
+    result = grouped.quantile(quantiles, skipna=skipna)
+    # Scalar calls retain every group. Vector calls must retain the same
+    # groups, including the partial-NaN and all-NaN groups.
+    for quantile in quantiles:
+        pd.testing.assert_series_equal(
+            result.xs(quantile, level=-1),
+            grouped.quantile(quantile, skipna=skipna),
+        )
+    first_group = 1.0 if skipna else np.nan
+    np.testing.assert_allclose(
+        result.to_numpy(),
+        [first_group, first_group, 3.0, 5.0, np.nan, np.nan],
+        equal_nan=True,
+    )
+
+
+def test_grouped_quantiles_preserve_repeated_requests():
+    series = mdf.MicroSeries([1.0, np.nan, 3.0, 5.0], weights=[1, 1, 1, 1])
+    result = series.groupby(["a", "a", "b", "b"]).quantile([0.5, 0.5], skipna=False)
+    assert result.index.tolist() == [("a", 0.5), ("a", 0.5), ("b", 0.5), ("b", 0.5)]
+    np.testing.assert_allclose(
+        result.to_numpy(), [np.nan, np.nan, 3.0, 3.0], equal_nan=True
+    )
+
+
+@pytest.mark.parametrize("quantiles", [[0.75, 0.25], [0.5, 0.5], []])
+@pytest.mark.parametrize("skipna", [True, False])
+@pytest.mark.parametrize("sort", [True, False])
+def test_grouped_quantiles_preserve_missing_multiple_keys(quantiles, skipna, sort):
+    """Missing group keys survive alongside missing values and repeated q."""
+    frame = mdf.MicroDataFrame(
+        {
+            "region": ["north", "north", None, "south", "south"],
+            "year": [2024, 2024, 2024, np.nan, 2025],
+            "income": [10.0, np.nan, 20.0, 30.0, 40.0],
+        },
+        weights=[1, 4, 2, 3, 1],
+    )
+    grouped = frame.groupby(["region", "year"], dropna=False, sort=sort)["income"]
+    result = grouped.quantile(quantiles, skipna=skipna)
+
+    # Each retained group has one nonmissing value. With skipna=False,
+    # the north group is NaN because it also contains a missing value.
+    north = 10.0 if skipna else np.nan
+    groups = [("north", 2024.0, north)]
+    if sort:
+        groups += [
+            ("south", 2025.0, 40.0),
+            ("south", np.nan, 30.0),
+            (np.nan, 2024.0, 20.0),
+        ]
+    else:
+        groups += [
+            (np.nan, 2024.0, 20.0),
+            ("south", np.nan, 30.0),
+            ("south", 2025.0, 40.0),
+        ]
+    expected_index = pd.MultiIndex.from_tuples(
+        [(region, year, q) for region, year, _ in groups for q in quantiles],
+        names=["region", "year", None],
+    )
+    expected_values = [value for _, _, value in groups for _ in quantiles]
+    assert result.index.names == expected_index.names
+    if quantiles:
+        for level in range(3):
+            pd.testing.assert_index_equal(
+                result.index.get_level_values(level),
+                expected_index.get_level_values(level),
+            )
+    assert result.index.nlevels == 3
+    np.testing.assert_allclose(result.to_numpy(), expected_values, equal_nan=True)
+    for q in set(quantiles):
+        if quantiles.count(q) == 1:
+            selected = result.xs(q, level=-1)
+            scalar = grouped.quantile(q, skipna=skipna)
+            assert selected.index.equals(scalar.index)
+            np.testing.assert_allclose(
+                selected.to_numpy(), scalar.to_numpy(), equal_nan=True
+            )

{microdf_python-1.3.4 → microdf_python-1.3.6}/microdf_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: microdf-python
-Version: 1.3.4
+Version: 1.3.6
 Summary: Weighted pandas DataFrames and Series for survey microdata
 Author-email: Max Ghenis <max@policyengine.org>
 License: MIT

{microdf_python-1.3.4 → microdf_python-1.3.6}/microdf_python.egg-info/SOURCES.txt

@@ -5,10 +5,12 @@ microdf/__init__.py
 microdf/microdataframe.py
 microdf/microseries.py
 microdf/tests/conftest.py
+microdf/tests/test_aggregation_errors.py
 microdf/tests/test_dataframe_weight_storage.py
 microdf/tests/test_microseries_dataframe.py
 microdf/tests/test_nullify_weights_index.py
 microdf/tests/test_pandas3_compatibility.py
+microdf/tests/test_quantile_missing_values.py
 microdf_python.egg-info/PKG-INFO
 microdf_python.egg-info/SOURCES.txt
 microdf_python.egg-info/dependency_links.txt

{microdf_python-1.3.4 → microdf_python-1.3.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "microdf-python"
-version = "1.3.4"
+version = "1.3.6"
 description = "Weighted pandas DataFrames and Series for survey microdata"
 readme = "README.md"
 authors = [