microdf-python 1.1.0__py3-none-any.whl → 1.1.2__py3-none-any.whl

microdf/microdataframe.py

@@ -44,6 +44,10 @@ class _MicroLocIndexer:
         self._parent_loc[key] = value
         self._mdf._link_all_weights()
 
+    def __getattr__(self, name):
+        """Delegate unknown attributes to the parent loc indexer."""
+        return getattr(self._parent_loc, name)
+
 
 class _MicroILocIndexer:
     """Custom iloc indexer that returns MicroDataFrame with proper weights."""
@@ -85,6 +89,10 @@ class _MicroILocIndexer:
         self._parent_iloc[key] = value
         self._mdf._link_all_weights()
 
+    def __getattr__(self, name):
+        """Delegate unknown attributes to the parent iloc indexer."""
+        return getattr(self._parent_iloc, name)
+
 
 class MicroDataFrame(pd.DataFrame):
     def __init__(self, *args, weights=None, **kwargs):

microdf/microseries.py

@@ -101,28 +101,48 @@ class MicroSeries(pd.Series):
         return self.weights.sum()
 
     @scalar_function
-    def mean(self) -> float:
+    def mean(self, skipna: bool = True) -> float:
         """Calculates the weighted mean of the MicroSeries.
 
+        :param skipna: Exclude NA/null values. If True (default), NaN values
+            are excluded. If False, returns NaN if any value is NaN.
+        :type skipna: bool
         :returns: The weighted mean.
         :rtype: float
         """
-        return np.average(self.values, weights=self.weights)
+        values = self.values
+        weights = self.weights
+
+        if skipna:
+            # Create mask for non-NaN values
+            mask = ~pd.isna(values)
+            if not mask.any():
+                # All values are NaN
+                return np.nan
+            values = values[mask]
+            weights = weights[mask]
+
+        # If skipna=False and there are any NaN values, return NaN
+        if not skipna and pd.isna(values).any():
+            return np.nan
+
+        return np.average(values, weights=weights)
 
     def quantile(self, q: np.array) -> pd.Series:
         """Calculates weighted quantiles of the MicroSeries.
 
-        Doesn't exactly match unweighted quantiles of stacked values.
-        See stackoverflow.com/q/21844024#comment102342137_29677616.
+        Uses the inverse CDF method: the q-th quantile is the smallest
+        value where the cumulative weight proportion >= q. This matches
+        the default behavior of R's survey::svyquantile.
 
-        :param q: Array of quantiles to calculate.
-        :type q: np.array
+        :param q: Quantile(s) to calculate, must be in [0, 1].
+        :type q: float or np.array
 
-        :return: Array of weighted quantiles.
-        :rtype: pd.Series
+        :return: Weighted quantile value(s).
+        :rtype: float or pd.Series
         """
         values = np.array(self.values)
-        quantiles = np.array(q)
+        quantiles = np.atleast_1d(q)
         sample_weight = np.array(self.weights)
         assert np.all(quantiles >= 0) and np.all(
             quantiles <= 1
@@ -130,11 +150,20 @@ class MicroSeries(pd.Series):
         sorter = np.argsort(values)
         values = values[sorter]
         sample_weight = sample_weight[sorter]
-        weighted_quantiles = np.cumsum(sample_weight) - 0.5 * sample_weight
-        weighted_quantiles /= np.sum(sample_weight)
-        result = np.interp(quantiles, weighted_quantiles, values)
-        if quantiles.shape == ():
-            return result
+        cumsum = np.cumsum(sample_weight)
+        cumsum_normalized = cumsum / cumsum[-1]
+        result = np.array(
+            [
+                values[
+                    min(
+                        np.searchsorted(cumsum_normalized, qi), len(values) - 1
+                    )
+                ]
+                for qi in quantiles
+            ]
+        )
+        if np.array(q).shape == ():
+            return result[0]
         return pd.Series(result, index=quantiles)
 
     @scalar_function

microdf/tests/test_microseries_dataframe.py

@@ -94,6 +94,36 @@ def test_mean() -> None:
         pass
 
 
+def test_mean_skipna() -> None:
+    # Test skipna=True (default) - should skip NaN values
+    arr = np.array([3.0, np.nan, 2.0])
+    w = np.array([4.0, 1.0, 1.0])
+    series = mdf.MicroSeries(arr, weights=w)
+
+    # skipna=True should exclude NaN and its weight
+    expected = np.average([3.0, 2.0], weights=[4.0, 1.0])
+    assert series.mean(skipna=True) == expected
+    assert series.mean() == expected  # Default should be skipna=True
+
+    # Test skipna=False - should return NaN if any value is NaN
+    assert np.isnan(series.mean(skipna=False))
+
+    # Test with all NaN values
+    arr_all_nan = np.array([np.nan, np.nan, np.nan])
+    w_all_nan = np.array([1.0, 2.0, 3.0])
+    series_all_nan = mdf.MicroSeries(arr_all_nan, weights=w_all_nan)
+    assert np.isnan(series_all_nan.mean(skipna=True))
+    assert np.isnan(series_all_nan.mean(skipna=False))
+
+    # Test with no NaN values - skipna should not affect result
+    arr_no_nan = np.array([3.0, 5.0, 2.0])
+    w_no_nan = np.array([4.0, 1.0, 1.0])
+    series_no_nan = mdf.MicroSeries(arr_no_nan, weights=w_no_nan)
+    expected_no_nan = np.average(arr_no_nan, weights=w_no_nan)
+    assert series_no_nan.mean(skipna=True) == expected_no_nan
+    assert series_no_nan.mean(skipna=False) == expected_no_nan
+
+
 def test_poverty_count() -> None:
     arr = np.array([10000, 20000, 50000])
     w = np.array([1123, 1144, 2211])
@@ -112,6 +142,55 @@ def test_median() -> None:
     assert series.median() == 4
 
 
+def test_weighted_quantile_skewed() -> None:
+    # 99% of the population has 0 income, 1% has 1M
+    # The median should be 0, not an interpolated value
+    series = mdf.MicroSeries([0, 1_000_000], weights=[99, 1])
+    assert series.median() == 0
+    assert series.quantile(0.5) == 0
+    # 99th percentile is still 0 since exactly 99% have 0
+    assert series.quantile(0.99) == 0
+    # Only quantile > 0.99 gives 1M
+    assert series.quantile(1.0) == 1_000_000
+    # Test multiple quantiles
+    result = series.quantile([0.1, 0.5, 0.99, 1.0])
+    assert result[0.1] == 0
+    assert result[0.5] == 0
+    assert result[0.99] == 0
+    assert result[1.0] == 1_000_000
+
+
+def test_weighted_quantile_boundaries() -> None:
+    # Test q=0 returns minimum, q=1 returns maximum
+    series = mdf.MicroSeries([10, 20, 30], weights=[1, 1, 1])
+    assert series.quantile(0.0) == 10
+    assert series.quantile(1.0) == 30
+
+
+def test_weighted_quantile_equal_weights() -> None:
+    # With equal weights, should match "replicated" interpretation
+    # Values: 1, 2, 3 each with weight 2 -> like [1,1,2,2,3,3]
+    series = mdf.MicroSeries([1, 2, 3], weights=[2, 2, 2])
+    # cumsum_normalized = [2/6, 4/6, 6/6] = [0.333, 0.667, 1.0]
+    # median (0.5): smallest where cumsum >= 0.5 -> index 1 -> value 2
+    assert series.median() == 2
+    # 0.25 quantile: smallest where cumsum >= 0.25 -> index 0 -> value 1
+    assert series.quantile(0.25) == 1
+    # 0.75 quantile: smallest where cumsum >= 0.75 -> index 2 -> value 3
+    assert series.quantile(0.75) == 3
+
+
+def test_weighted_quantile_unsorted_input() -> None:
+    # Ensure sorting works correctly
+    series = mdf.MicroSeries([30, 10, 20], weights=[1, 2, 1])
+    # Sorted: values [10, 20, 30], weights [2, 1, 1]
+    # cumsum_normalized = [0.5, 0.75, 1.0]
+    assert series.quantile(0.0) == 10
+    assert series.quantile(0.5) == 10  # cumsum[0]=0.5 >= 0.5
+    assert series.quantile(0.6) == 20  # cumsum[1]=0.75 >= 0.6
+    assert series.quantile(1.0) == 30
+
+
 def test_unweighted_groupby() -> None:
     df = mdf.MicroDataFrame({"x": [1, 2], "y": [3, 4], "z": [5, 6]})
     assert (df.groupby("x").z.sum().values == np.array([5.0, 6.0])).all()

{microdf_python-1.1.0.dist-info → microdf_python-1.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: microdf-python
-Version: 1.1.0
+Version: 1.1.2
 Summary: Weighted pandas DataFrames and Series for survey microdata
 Author-email: Max Ghenis <max@policyengine.org>
 License: MIT

microdf_python-1.1.2.dist-info/RECORD

@@ -0,0 +1,10 @@
+microdf/__init__.py,sha256=ldRvhE8t4cD7qlxQ5yLndMnWEUYpDw4m-jCDV_psXkY,319
+microdf/microdataframe.py,sha256=h0vIuCACMo1BuV3VKDIEfhc_UwMUV5kUsnlQe3bHfgM,33539
+microdf/microseries.py,sha256=YRnSHVm3mGGd1_-D29BAUDaqWoIiwGjJPKYUrnWbinY,23376
+microdf/tests/conftest.py,sha256=u-EMyX1-u_nM-YO0RJYCzYHQDXxUI2WQE6GkyJlErqg,150
+microdf/tests/test_microseries_dataframe.py,sha256=lG2_3lJqTdGekbbJRFl0Vp0b4DEQwTk6scXr-ncOHgs,14647
+microdf_python-1.1.2.dist-info/licenses/LICENSE,sha256=uPs-ASYnzlldpf2z8jeRgQFeEH3FLhSuX0rw0OKWoDU,1067
+microdf_python-1.1.2.dist-info/METADATA,sha256=MNlbpTktwqMoFlf-CImA-Xsi-ix7Zv1XAYZ09W1YRcY,2420
+microdf_python-1.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+microdf_python-1.1.2.dist-info/top_level.txt,sha256=T2WFPTygQQMdS3GF8YpZ12DKfMGrspbZ3r7z-e3KfiM,8
+microdf_python-1.1.2.dist-info/RECORD,,

microdf_python-1.1.0.dist-info/RECORD

@@ -1,10 +0,0 @@
-microdf/__init__.py,sha256=ldRvhE8t4cD7qlxQ5yLndMnWEUYpDw4m-jCDV_psXkY,319
-microdf/microdataframe.py,sha256=X91fUvDS9xMzj2FFAoa6g9EhR_mJv60DXtvK4iIlMrM,33237
-microdf/microseries.py,sha256=MFBStp1IaNVABaf3_Ap_VwXSC3bA8V7rXD4TEG_091g,22452
-microdf/tests/conftest.py,sha256=u-EMyX1-u_nM-YO0RJYCzYHQDXxUI2WQE6GkyJlErqg,150
-microdf/tests/test_microseries_dataframe.py,sha256=U33uBNW6uiF8c7ygDJjwYX_tJJc_i7OKHi-O746XQLY,11398
-microdf_python-1.1.0.dist-info/licenses/LICENSE,sha256=uPs-ASYnzlldpf2z8jeRgQFeEH3FLhSuX0rw0OKWoDU,1067
-microdf_python-1.1.0.dist-info/METADATA,sha256=vPKXByn0atLs1zxTwc6XqESm7iE1wQ4lNtVIIAPBfQA,2420
-microdf_python-1.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-microdf_python-1.1.0.dist-info/top_level.txt,sha256=T2WFPTygQQMdS3GF8YpZ12DKfMGrspbZ3r7z-e3KfiM,8
-microdf_python-1.1.0.dist-info/RECORD,,