microdf-python 1.3.0__tar.gz → 1.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {microdf_python-1.3.0/microdf_python.egg-info → microdf_python-1.3.2}/PKG-INFO +5 -6
- {microdf_python-1.3.0 → microdf_python-1.3.2}/README.md +4 -5
- {microdf_python-1.3.0 → microdf_python-1.3.2}/microdf/microdataframe.py +11 -6
- {microdf_python-1.3.0 → microdf_python-1.3.2}/microdf/microseries.py +18 -17
- {microdf_python-1.3.0 → microdf_python-1.3.2}/microdf/tests/test_microseries_dataframe.py +33 -37
- {microdf_python-1.3.0 → microdf_python-1.3.2/microdf_python.egg-info}/PKG-INFO +5 -6
- {microdf_python-1.3.0 → microdf_python-1.3.2}/pyproject.toml +1 -1
- {microdf_python-1.3.0 → microdf_python-1.3.2}/LICENSE +0 -0
- {microdf_python-1.3.0 → microdf_python-1.3.2}/microdf/__init__.py +0 -0
- {microdf_python-1.3.0 → microdf_python-1.3.2}/microdf/tests/conftest.py +0 -0
- {microdf_python-1.3.0 → microdf_python-1.3.2}/microdf/tests/test_pandas3_compatibility.py +0 -0
- {microdf_python-1.3.0 → microdf_python-1.3.2}/microdf_python.egg-info/SOURCES.txt +0 -0
- {microdf_python-1.3.0 → microdf_python-1.3.2}/microdf_python.egg-info/dependency_links.txt +0 -0
- {microdf_python-1.3.0 → microdf_python-1.3.2}/microdf_python.egg-info/requires.txt +0 -0
- {microdf_python-1.3.0 → microdf_python-1.3.2}/microdf_python.egg-info/top_level.txt +0 -0
- {microdf_python-1.3.0 → microdf_python-1.3.2}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: microdf-python
|
|
3
|
-
Version: 1.3.
|
|
3
|
+
Version: 1.3.2
|
|
4
4
|
Summary: Weighted pandas DataFrames and Series for survey microdata
|
|
5
5
|
Author-email: Max Ghenis <max@policyengine.org>
|
|
6
6
|
License: MIT
|
|
@@ -50,13 +50,12 @@ import microdf as mdf
|
|
|
50
50
|
import pandas as pd
|
|
51
51
|
|
|
52
52
|
# Create sample data with weights
|
|
53
|
-
df = pd.DataFrame(
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
})
|
|
53
|
+
df = pd.DataFrame(
|
|
54
|
+
{"income": [10_000, 20_000, 30_000, 40_000, 50_000], "weights": [1, 2, 3, 2, 1]}
|
|
55
|
+
)
|
|
57
56
|
|
|
58
57
|
# Create a MicroDataFrame
|
|
59
|
-
mdf_df = mdf.MicroDataFrame(df, weights=
|
|
58
|
+
mdf_df = mdf.MicroDataFrame(df, weights="weights")
|
|
60
59
|
|
|
61
60
|
# All operations are weight-aware
|
|
62
61
|
print(mdf_df.income.mean()) # Weighted mean
|
|
@@ -29,13 +29,12 @@ import microdf as mdf
|
|
|
29
29
|
import pandas as pd
|
|
30
30
|
|
|
31
31
|
# Create sample data with weights
|
|
32
|
-
df = pd.DataFrame(
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
})
|
|
32
|
+
df = pd.DataFrame(
|
|
33
|
+
{"income": [10_000, 20_000, 30_000, 40_000, 50_000], "weights": [1, 2, 3, 2, 1]}
|
|
34
|
+
)
|
|
36
35
|
|
|
37
36
|
# Create a MicroDataFrame
|
|
38
|
-
mdf_df = mdf.MicroDataFrame(df, weights=
|
|
37
|
+
mdf_df = mdf.MicroDataFrame(df, weights="weights")
|
|
39
38
|
|
|
40
39
|
# All operations are weight-aware
|
|
41
40
|
print(mdf_df.income.mean()) # Weighted mean
|
|
@@ -96,8 +96,10 @@ class _MicroILocIndexer:
|
|
|
96
96
|
|
|
97
97
|
class MicroDataFrame(pd.DataFrame):
|
|
98
98
|
def __init__(self, *args, weights=None, **kwargs):
|
|
99
|
-
"""A DataFrame-inheriting class for weighted microdata.
|
|
100
|
-
|
|
99
|
+
"""A DataFrame-inheriting class for weighted microdata.
|
|
100
|
+
|
|
101
|
+
Weights can be provided at initialisation, or using set_weights or
|
|
102
|
+
set_weight_col.
|
|
101
103
|
|
|
102
104
|
:param weights: Array of weights.
|
|
103
105
|
:type weights: np.array
|
|
@@ -231,8 +233,9 @@ class MicroDataFrame(pd.DataFrame):
|
|
|
231
233
|
return fn
|
|
232
234
|
|
|
233
235
|
def get_args_as_micro_series(*kwarg_names: tuple) -> Callable:
|
|
234
|
-
"""Decorator for auto-parsing column names into MicroSeries objects.
|
|
235
|
-
|
|
236
|
+
"""Decorator for auto-parsing column names into MicroSeries objects.
|
|
237
|
+
|
|
238
|
+
If given, kwarg_names limits arguments checked to keyword arguments
|
|
236
239
|
specified.
|
|
237
240
|
|
|
238
241
|
:param arg_names: argument names to restrict to.
|
|
@@ -292,8 +295,10 @@ class MicroDataFrame(pd.DataFrame):
|
|
|
292
295
|
weights: Union[np.ndarray, str],
|
|
293
296
|
preserve_old: Optional[bool] = False,
|
|
294
297
|
) -> None:
|
|
295
|
-
"""Sets the weights for the MicroDataFrame.
|
|
296
|
-
|
|
298
|
+
"""Sets the weights for the MicroDataFrame.
|
|
299
|
+
|
|
300
|
+
If a string is received, it will be assumed to be the column name of
|
|
301
|
+
the weight column.
|
|
297
302
|
|
|
298
303
|
:param weights: Array of weights.
|
|
299
304
|
:param preserve_old: If True, keeps the old weights as a column when
|
|
@@ -14,10 +14,10 @@ def _weighted_top_share(
|
|
|
14
14
|
) -> float:
|
|
15
15
|
"""Share of the sum held by the top ``top_x_pct`` of weight.
|
|
16
16
|
|
|
17
|
-
Sort by value ascending, cumulate the weight, pick the slice from
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
17
|
+
Sort by value ascending, cumulate the weight, pick the slice from the top
|
|
18
|
+
that covers exactly ``top_x_pct`` of total weight, and distribute the tied-
|
|
19
|
+
at-cutoff row proportionally so constant values return exactly
|
|
20
|
+
``top_x_pct`` rather than 1.0.
|
|
21
21
|
"""
|
|
22
22
|
if top_x_pct <= 0:
|
|
23
23
|
return 0.0
|
|
@@ -51,8 +51,9 @@ def _weighted_top_share(
|
|
|
51
51
|
|
|
52
52
|
class MicroSeries(pd.Series):
|
|
53
53
|
def __init__(self, *args, weights: np.array = None, **kwargs):
|
|
54
|
-
"""A Series-inheriting class for weighted microdata.
|
|
55
|
-
|
|
54
|
+
"""A Series-inheriting class for weighted microdata.
|
|
55
|
+
|
|
56
|
+
Weights can be provided at initialisation, or using set_weights.
|
|
56
57
|
|
|
57
58
|
:param weights: Array of weights.
|
|
58
59
|
:type weights: np.array
|
|
@@ -275,9 +276,9 @@ class MicroSeries(pd.Series):
|
|
|
275
276
|
"""Pandas ``cov`` — **unweighted**.
|
|
276
277
|
|
|
277
278
|
MicroSeries does not yet compute weighted covariance. Emits a
|
|
278
|
-
``UserWarning`` so callers aren't silently given an unweighted
|
|
279
|
-
|
|
280
|
-
|
|
279
|
+
``UserWarning`` so callers aren't silently given an unweighted number
|
|
280
|
+
after ``.sum()`` and ``.mean()`` worked as expected. See issue tracker
|
|
281
|
+
for a weighted implementation.
|
|
281
282
|
"""
|
|
282
283
|
warnings.warn(
|
|
283
284
|
"MicroSeries.cov() falls through to pandas and is "
|
|
@@ -293,8 +294,8 @@ class MicroSeries(pd.Series):
|
|
|
293
294
|
"""Pandas ``corr`` — **unweighted**.
|
|
294
295
|
|
|
295
296
|
MicroSeries does not yet compute weighted correlation. Emits a
|
|
296
|
-
``UserWarning`` so callers aren't silently given an unweighted
|
|
297
|
-
|
|
297
|
+
``UserWarning`` so callers aren't silently given an unweighted number.
|
|
298
|
+
See issue tracker for a weighted implementation.
|
|
298
299
|
"""
|
|
299
300
|
warnings.warn(
|
|
300
301
|
"MicroSeries.corr() falls through to pandas and is "
|
|
@@ -519,13 +520,13 @@ class MicroSeries(pd.Series):
|
|
|
519
520
|
def rank(self, pct: Optional[bool] = False) -> pd.Series:
|
|
520
521
|
"""Weighted rank of each element.
|
|
521
522
|
|
|
522
|
-
Each element's rank is the cumulative weight of all values that
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
523
|
+
Each element's rank is the cumulative weight of all values that are
|
|
524
|
+
less than or equal to it. Tied values therefore share the same rank, so
|
|
525
|
+
downstream bucketing (``decile_rank``, ``quintile_rank``, etc.) lands
|
|
526
|
+
tied rows in the same bucket.
|
|
526
527
|
|
|
527
|
-
:param pct: If True, divide ranks by the total weight so they
|
|
528
|
-
|
|
528
|
+
:param pct: If True, divide ranks by the total weight so they lie in
|
|
529
|
+
``(0, 1]``.
|
|
529
530
|
:type pct: bool
|
|
530
531
|
:returns: MicroSeries of ranks aligned to ``self``.
|
|
531
532
|
:rtype: MicroSeries
|
|
@@ -449,11 +449,11 @@ def test_mean_no_warning() -> None:
|
|
|
449
449
|
def test_sum_with_non_default_index() -> None:
|
|
450
450
|
"""Weighted sum must not silently return 0 with a non-default index.
|
|
451
451
|
|
|
452
|
-
Regression test for the bug where ``set_weights`` stored the weights
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
452
|
+
Regression test for the bug where ``set_weights`` stored the weights Series
|
|
453
|
+
with a default ``RangeIndex`` regardless of ``self.index``. Element-wise
|
|
454
|
+
ops like ``self.multiply(self.weights)`` then aligned on label, producing
|
|
455
|
+
all-NaN and a silent ``0.0`` from ``.sum()`` while ``.mean()`` (which uses
|
|
456
|
+
a positional ndarray) stayed correct.
|
|
457
457
|
"""
|
|
458
458
|
# MicroSeries with custom integer index.
|
|
459
459
|
s = mdf.MicroSeries([1, 2, 3], index=[100, 200, 300], weights=[10, 20, 30])
|
|
@@ -541,10 +541,9 @@ def test_merge_preserves_weights_per_surviving_row() -> None:
|
|
|
541
541
|
"""Regression: merge must propagate weights onto the merged rows.
|
|
542
542
|
|
|
543
543
|
Previously the implementation passed ``self.weights`` straight to the
|
|
544
|
-
MicroDataFrame constructor, so any merge that changed row count
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
DataFrame (M)``.
|
|
544
|
+
MicroDataFrame constructor, so any merge that changed row count (inner
|
|
545
|
+
filtering, left-with-missing, many-to-many, outer) raised ``ValueError:
|
|
546
|
+
Length of weights (N) does not match length of DataFrame (M)``.
|
|
548
547
|
"""
|
|
549
548
|
# Inner join filters rows.
|
|
550
549
|
left = mdf.MicroDataFrame(
|
|
@@ -587,10 +586,10 @@ def test_merge_preserves_weights_per_surviving_row() -> None:
|
|
|
587
586
|
def test_groupby_does_not_leak_tmp_weights_column() -> None:
|
|
588
587
|
"""Regression: groupby used to mutate self by adding __tmp_weights.
|
|
589
588
|
|
|
590
|
-
Previously, ``MicroDataFrame.groupby`` set ``self["__tmp_weights"]``
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
589
|
+
Previously, ``MicroDataFrame.groupby`` set ``self["__tmp_weights"]`` and
|
|
590
|
+
never cleaned it up, so ``df.columns`` afterwards included the weight
|
|
591
|
+
column and any later ``df.sum()`` or iteration over columns picked it up as
|
|
592
|
+
data.
|
|
594
593
|
"""
|
|
595
594
|
df = mdf.MicroDataFrame({"g": ["a", "a", "b"], "v": [1, 2, 3]}, weights=[1, 2, 3])
|
|
596
595
|
original_cols = list(df.columns)
|
|
@@ -616,11 +615,10 @@ def test_groupby_does_not_leak_tmp_weights_column() -> None:
|
|
|
616
615
|
def test_quantile_skips_zero_weight_rows() -> None:
|
|
617
616
|
"""Regression: quantile(0) shouldn't pick a zero-weight element.
|
|
618
617
|
|
|
619
|
-
Previously, ``np.searchsorted(cumsum_norm, 0, side='left')`` returned
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
the CDF.
|
|
618
|
+
Previously, ``np.searchsorted(cumsum_norm, 0, side='left')`` returned 0
|
|
619
|
+
even when that first sorted element had zero weight, so ``MicroSeries([10,
|
|
620
|
+
20, 30], weights=[0, 1, 1]).quantile(0)`` returned 10 instead of 20. The
|
|
621
|
+
fix drops zero-weight rows before computing the CDF.
|
|
624
622
|
"""
|
|
625
623
|
s = mdf.MicroSeries([10, 20, 30], weights=[0, 1, 1])
|
|
626
624
|
assert s.quantile(0.0) == 20
|
|
@@ -681,10 +679,9 @@ def test_top_x_pct_share_handles_ties_and_edges() -> None:
|
|
|
681
679
|
def test_gini_negatives_option_applied() -> None:
|
|
682
680
|
"""Regression: gini(negatives=...) was silently ignored.
|
|
683
681
|
|
|
684
|
-
Both branches of the old implementation sorted ``self`` directly
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
nothing.
|
|
682
|
+
Both branches of the old implementation sorted ``self`` directly rather
|
|
683
|
+
than the local ``x`` that was mutated by the ``negatives`` option, so
|
|
684
|
+
``negatives='zero'`` and ``negatives='shift'`` did nothing.
|
|
688
685
|
"""
|
|
689
686
|
s = mdf.MicroSeries([-5, 0, 10], weights=[1, 1, 1])
|
|
690
687
|
|
|
@@ -715,10 +712,9 @@ def test_gini_negatives_option_applied() -> None:
|
|
|
715
712
|
def test_std_var_are_weighted() -> None:
|
|
716
713
|
"""Regression: std/var used to silently fall through to pandas.
|
|
717
714
|
|
|
718
|
-
The old implementation had no override, so a MicroSeries with very
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
sample.
|
|
715
|
+
The old implementation had no override, so a MicroSeries with very uneven
|
|
716
|
+
weights returned the unweighted 1.0. Now std and var treat the weights as
|
|
717
|
+
frequency counts, matching numpy on the replicated sample.
|
|
722
718
|
"""
|
|
723
719
|
s = mdf.MicroSeries([1, 2, 3], weights=[100, 1, 1])
|
|
724
720
|
# Unweighted would be 1.0. Weighted std pulls toward the heavy row.
|
|
@@ -751,8 +747,8 @@ def test_std_var_are_weighted() -> None:
|
|
|
751
747
|
def test_cov_corr_warn_when_fallthrough() -> None:
|
|
752
748
|
"""Regression: cov/corr silently returned unweighted pandas values.
|
|
753
749
|
|
|
754
|
-
They still fall through to pandas (a weighted impl is a separate
|
|
755
|
-
|
|
750
|
+
They still fall through to pandas (a weighted impl is a separate issue) but
|
|
751
|
+
now emit a UserWarning so callers aren't misled.
|
|
756
752
|
"""
|
|
757
753
|
s1 = mdf.MicroSeries([1, 2, 3], weights=[1, 1, 1])
|
|
758
754
|
s2 = mdf.MicroSeries([2, 4, 6], weights=[1, 1, 1])
|
|
@@ -773,9 +769,9 @@ def test_cov_corr_warn_when_fallthrough() -> None:
|
|
|
773
769
|
def test_count_skips_nan_by_default() -> None:
|
|
774
770
|
"""Regression: ``count()`` included NaN-row weight, contrary to pandas.
|
|
775
771
|
|
|
776
|
-
Pandas ``Series.count`` skips NaN; MicroSeries returned the full
|
|
777
|
-
|
|
778
|
-
|
|
772
|
+
Pandas ``Series.count`` skips NaN; MicroSeries returned the full weight sum
|
|
773
|
+
regardless. The fix matches pandas semantics and adds a ``skipna`` kwarg so
|
|
774
|
+
callers can opt out.
|
|
779
775
|
"""
|
|
780
776
|
s = mdf.MicroSeries([1.0, np.nan, 3.0], weights=[10, 20, 30])
|
|
781
777
|
assert s.count() == 40.0
|
|
@@ -794,12 +790,12 @@ def test_count_skips_nan_by_default() -> None:
|
|
|
794
790
|
def test_rank_ties_share_bucket() -> None:
|
|
795
791
|
"""Regression: rank used to assign ties to different ranks/buckets.
|
|
796
792
|
|
|
797
|
-
Previously ``rank`` returned the running cumulative weight in sort
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
793
|
+
Previously ``rank`` returned the running cumulative weight in sort order,
|
|
794
|
+
so every row — tied or not — got a distinct value. As a result
|
|
795
|
+
``MicroSeries([5]*5, weights=[1]*5).decile_rank()`` returned ``[2, 4, 6, 8,
|
|
796
|
+
10]`` rather than all 10. With max-rank semantics, tied values share the
|
|
797
|
+
cumulative weight at the end of their tie group, so bucketing is stable
|
|
798
|
+
under ties.
|
|
803
799
|
"""
|
|
804
800
|
# All tied: every element lands in the top decile.
|
|
805
801
|
s = mdf.MicroSeries([5] * 5, weights=[1] * 5)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: microdf-python
|
|
3
|
-
Version: 1.3.
|
|
3
|
+
Version: 1.3.2
|
|
4
4
|
Summary: Weighted pandas DataFrames and Series for survey microdata
|
|
5
5
|
Author-email: Max Ghenis <max@policyengine.org>
|
|
6
6
|
License: MIT
|
|
@@ -50,13 +50,12 @@ import microdf as mdf
|
|
|
50
50
|
import pandas as pd
|
|
51
51
|
|
|
52
52
|
# Create sample data with weights
|
|
53
|
-
df = pd.DataFrame(
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
})
|
|
53
|
+
df = pd.DataFrame(
|
|
54
|
+
{"income": [10_000, 20_000, 30_000, 40_000, 50_000], "weights": [1, 2, 3, 2, 1]}
|
|
55
|
+
)
|
|
57
56
|
|
|
58
57
|
# Create a MicroDataFrame
|
|
59
|
-
mdf_df = mdf.MicroDataFrame(df, weights=
|
|
58
|
+
mdf_df = mdf.MicroDataFrame(df, weights="weights")
|
|
60
59
|
|
|
61
60
|
# All operations are weight-aware
|
|
62
61
|
print(mdf_df.income.mean()) # Weighted mean
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|