microdf-python 1.0.1__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {microdf_python-1.0.1/microdf_python.egg-info → microdf_python-1.1.0}/PKG-INFO +2 -4
- {microdf_python-1.0.1 → microdf_python-1.1.0}/microdf/microdataframe.py +426 -46
- {microdf_python-1.0.1 → microdf_python-1.1.0}/microdf/microseries.py +8 -0
- {microdf_python-1.0.1 → microdf_python-1.1.0}/microdf/tests/test_microseries_dataframe.py +65 -0
- {microdf_python-1.0.1 → microdf_python-1.1.0/microdf_python.egg-info}/PKG-INFO +2 -4
- {microdf_python-1.0.1 → microdf_python-1.1.0}/microdf_python.egg-info/requires.txt +0 -3
- {microdf_python-1.0.1 → microdf_python-1.1.0}/pyproject.toml +4 -5
- {microdf_python-1.0.1 → microdf_python-1.1.0}/LICENSE +0 -0
- {microdf_python-1.0.1 → microdf_python-1.1.0}/README.md +0 -0
- {microdf_python-1.0.1 → microdf_python-1.1.0}/microdf/__init__.py +0 -0
- {microdf_python-1.0.1 → microdf_python-1.1.0}/microdf/tests/conftest.py +0 -0
- {microdf_python-1.0.1 → microdf_python-1.1.0}/microdf_python.egg-info/SOURCES.txt +0 -0
- {microdf_python-1.0.1 → microdf_python-1.1.0}/microdf_python.egg-info/dependency_links.txt +0 -0
- {microdf_python-1.0.1 → microdf_python-1.1.0}/microdf_python.egg-info/top_level.txt +0 -0
- {microdf_python-1.0.1 → microdf_python-1.1.0}/setup.cfg +0 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: microdf-python
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: Weighted pandas DataFrames and Series for survey microdata
|
|
5
|
-
Author-email: Max Ghenis <max@
|
|
5
|
+
Author-email: Max Ghenis <max@policyengine.org>
|
|
6
6
|
License: MIT
|
|
7
7
|
Requires-Python: >=3.9
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
@@ -20,8 +20,6 @@ Requires-Dist: linecheck; extra == "dev"
|
|
|
20
20
|
Requires-Dist: pytest; extra == "dev"
|
|
21
21
|
Requires-Dist: pytest-cov; extra == "dev"
|
|
22
22
|
Requires-Dist: setuptools; extra == "dev"
|
|
23
|
-
Provides-Extra: docs
|
|
24
|
-
Requires-Dist: jupyter_book; extra == "docs"
|
|
25
23
|
Dynamic: license-file
|
|
26
24
|
|
|
27
25
|
[](https://github.com/PolicyEngine/microdf/actions)
|
|
@@ -12,6 +12,80 @@ from microdf.microseries import MicroSeries, MicroSeriesGroupBy
|
|
|
12
12
|
logger = logging.getLogger(__name__)
|
|
13
13
|
|
|
14
14
|
|
|
15
|
+
class _MicroLocIndexer:
|
|
16
|
+
"""Custom loc indexer that returns MicroDataFrame with proper weights."""
|
|
17
|
+
|
|
18
|
+
def __init__(self, mdf: "MicroDataFrame"):
|
|
19
|
+
self._mdf = mdf
|
|
20
|
+
# Get the parent's loc indexer
|
|
21
|
+
self._parent_loc = pd.DataFrame.loc.fget(mdf)
|
|
22
|
+
|
|
23
|
+
def __getitem__(self, key):
|
|
24
|
+
# Use the parent DataFrame's loc indexer
|
|
25
|
+
result = self._parent_loc[key]
|
|
26
|
+
|
|
27
|
+
if isinstance(result, pd.DataFrame):
|
|
28
|
+
# Get the filtered weights based on the result's index
|
|
29
|
+
new_weights = self._mdf.weights.reindex(result.index)
|
|
30
|
+
return MicroDataFrame(result, weights=new_weights)
|
|
31
|
+
elif isinstance(result, pd.Series):
|
|
32
|
+
# Single row or column selected
|
|
33
|
+
if result.name in self._mdf.columns:
|
|
34
|
+
# Column was selected - return MicroSeries with all weights
|
|
35
|
+
return MicroSeries(result, weights=self._mdf.weights)
|
|
36
|
+
else:
|
|
37
|
+
# Row was selected - return as-is (scalar values for each col)
|
|
38
|
+
return result
|
|
39
|
+
else:
|
|
40
|
+
# Scalar value
|
|
41
|
+
return result
|
|
42
|
+
|
|
43
|
+
def __setitem__(self, key, value):
|
|
44
|
+
self._parent_loc[key] = value
|
|
45
|
+
self._mdf._link_all_weights()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class _MicroILocIndexer:
|
|
49
|
+
"""Custom iloc indexer that returns MicroDataFrame with proper weights."""
|
|
50
|
+
|
|
51
|
+
def __init__(self, mdf: "MicroDataFrame"):
|
|
52
|
+
self._mdf = mdf
|
|
53
|
+
# Get the parent's iloc indexer
|
|
54
|
+
self._parent_iloc = pd.DataFrame.iloc.fget(mdf)
|
|
55
|
+
|
|
56
|
+
def __getitem__(self, key):
|
|
57
|
+
# Use the parent DataFrame's iloc indexer
|
|
58
|
+
result = self._parent_iloc[key]
|
|
59
|
+
|
|
60
|
+
if isinstance(result, pd.DataFrame):
|
|
61
|
+
# Get the filtered weights based on the result's index
|
|
62
|
+
new_weights = self._mdf.weights.iloc[
|
|
63
|
+
self._mdf.index.get_indexer(result.index)
|
|
64
|
+
]
|
|
65
|
+
new_weights = pd.Series(new_weights.values, index=result.index)
|
|
66
|
+
return MicroDataFrame(result, weights=new_weights)
|
|
67
|
+
elif isinstance(result, pd.Series):
|
|
68
|
+
# Single row or column selected
|
|
69
|
+
if isinstance(key, tuple) and len(key) == 2:
|
|
70
|
+
# df.iloc[:, col_idx] - column selection
|
|
71
|
+
row_key = key[0]
|
|
72
|
+
if isinstance(row_key, slice) and row_key == slice(None):
|
|
73
|
+
# All rows selected for a column
|
|
74
|
+
return MicroSeries(result, weights=self._mdf.weights)
|
|
75
|
+
# Check if this is a column (result index matches mdf index)
|
|
76
|
+
if result.index.equals(self._mdf.index):
|
|
77
|
+
return MicroSeries(result, weights=self._mdf.weights)
|
|
78
|
+
# Row selection - return as-is
|
|
79
|
+
return result
|
|
80
|
+
else:
|
|
81
|
+
# Scalar value
|
|
82
|
+
return result
|
|
83
|
+
|
|
84
|
+
def __setitem__(self, key, value):
|
|
85
|
+
self._parent_iloc[key] = value
|
|
86
|
+
self._mdf._link_all_weights()
|
|
87
|
+
|
|
88
|
+
|
|
15
89
|
class MicroDataFrame(pd.DataFrame):
|
|
16
90
|
def __init__(self, *args, weights=None, **kwargs):
|
|
17
91
|
"""A DataFrame-inheriting class for weighted microdata. Weights can be
|
|
@@ -26,6 +100,23 @@ class MicroDataFrame(pd.DataFrame):
|
|
|
26
100
|
self._link_all_weights()
|
|
27
101
|
self.override_df_functions()
|
|
28
102
|
|
|
103
|
+
@property
|
|
104
|
+
def loc(self) -> _MicroLocIndexer:
|
|
105
|
+
"""Label-based indexer that preserves MicroDataFrame type and weights.
|
|
106
|
+
|
|
107
|
+
:return: Custom loc indexer for MicroDataFrame
|
|
108
|
+
"""
|
|
109
|
+
return _MicroLocIndexer(self)
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def iloc(self) -> _MicroILocIndexer:
|
|
113
|
+
"""Integer-based indexer that preserves MicroDataFrame type and
|
|
114
|
+
weights.
|
|
115
|
+
|
|
116
|
+
:return: Custom iloc indexer for MicroDataFrame
|
|
117
|
+
"""
|
|
118
|
+
return _MicroILocIndexer(self)
|
|
119
|
+
|
|
29
120
|
def override_df_functions(self) -> None:
|
|
30
121
|
"""Override DataFrame functions to work with weighted operations."""
|
|
31
122
|
for name in MicroSeries.FUNCTIONS:
|
|
@@ -44,14 +135,17 @@ class MicroDataFrame(pd.DataFrame):
|
|
|
44
135
|
"""
|
|
45
136
|
|
|
46
137
|
def fn(*args, **kwargs) -> pd.Series:
|
|
47
|
-
results =
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
138
|
+
results = {}
|
|
139
|
+
for col in self.columns:
|
|
140
|
+
if pd.api.types.is_numeric_dtype(self[col]):
|
|
141
|
+
try:
|
|
142
|
+
results[col] = getattr(self[col], name)(
|
|
143
|
+
*args, **kwargs
|
|
144
|
+
)
|
|
145
|
+
except Exception:
|
|
146
|
+
# Skip columns that can't be aggregated
|
|
147
|
+
pass
|
|
148
|
+
return pd.Series(results)
|
|
55
149
|
|
|
56
150
|
return fn
|
|
57
151
|
|
|
@@ -63,14 +157,24 @@ class MicroDataFrame(pd.DataFrame):
|
|
|
63
157
|
"""
|
|
64
158
|
|
|
65
159
|
def fn(*args, **kwargs) -> pd.DataFrame:
|
|
66
|
-
results =
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
160
|
+
results = []
|
|
161
|
+
columns = []
|
|
162
|
+
for col in self.columns:
|
|
163
|
+
if pd.api.types.is_numeric_dtype(self[col]):
|
|
164
|
+
try:
|
|
165
|
+
result = getattr(self[col], name)(*args, **kwargs)
|
|
166
|
+
results.append(result)
|
|
167
|
+
columns.append(col)
|
|
168
|
+
except Exception:
|
|
169
|
+
# Skip columns that can't be aggregated
|
|
170
|
+
pass
|
|
171
|
+
|
|
172
|
+
if results:
|
|
173
|
+
df = pd.DataFrame(results)
|
|
174
|
+
df.index = columns
|
|
175
|
+
return df
|
|
176
|
+
else:
|
|
177
|
+
return pd.DataFrame()
|
|
74
178
|
|
|
75
179
|
return fn
|
|
76
180
|
|
|
@@ -88,24 +192,37 @@ class MicroDataFrame(pd.DataFrame):
|
|
|
88
192
|
|
|
89
193
|
if is_array:
|
|
90
194
|
# Use vector function behavior
|
|
91
|
-
results =
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
195
|
+
results = []
|
|
196
|
+
columns = []
|
|
197
|
+
for col in self.columns:
|
|
198
|
+
if pd.api.types.is_numeric_dtype(self[col]):
|
|
199
|
+
try:
|
|
200
|
+
result = getattr(self[col], name)(*args, **kwargs)
|
|
201
|
+
results.append(result)
|
|
202
|
+
columns.append(col)
|
|
203
|
+
except Exception:
|
|
204
|
+
# Skip columns that can't be aggregated
|
|
205
|
+
pass
|
|
206
|
+
|
|
207
|
+
if results:
|
|
208
|
+
df = pd.DataFrame(results)
|
|
209
|
+
df.index = columns
|
|
210
|
+
return df
|
|
211
|
+
else:
|
|
212
|
+
return pd.DataFrame()
|
|
99
213
|
else:
|
|
100
214
|
# Use scalar function behavior
|
|
101
|
-
results =
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
215
|
+
results = {}
|
|
216
|
+
for col in self.columns:
|
|
217
|
+
if pd.api.types.is_numeric_dtype(self[col]):
|
|
218
|
+
try:
|
|
219
|
+
results[col] = getattr(self[col], name)(
|
|
220
|
+
*args, **kwargs
|
|
221
|
+
)
|
|
222
|
+
except Exception:
|
|
223
|
+
# Skip columns that can't be aggregated
|
|
224
|
+
pass
|
|
225
|
+
return pd.Series(results)
|
|
109
226
|
|
|
110
227
|
return fn
|
|
111
228
|
|
|
@@ -185,6 +302,7 @@ class MicroDataFrame(pd.DataFrame):
|
|
|
185
302
|
if isinstance(weights, str):
|
|
186
303
|
self.weights_col = weights
|
|
187
304
|
self.weights = pd.Series(self[weights], dtype=float)
|
|
305
|
+
self._link_all_weights()
|
|
188
306
|
elif weights is not None:
|
|
189
307
|
if len(weights) != len(self):
|
|
190
308
|
raise ValueError(
|
|
@@ -203,11 +321,24 @@ class MicroDataFrame(pd.DataFrame):
|
|
|
203
321
|
"""Sets the weights for the MicroDataFrame by specifying the name of
|
|
204
322
|
the weight column.
|
|
205
323
|
|
|
206
|
-
|
|
324
|
+
.. deprecated:: 1.0.2
|
|
325
|
+
Use :meth:`set_weights` with a string argument instead.
|
|
326
|
+
This method will be removed in a future version.
|
|
327
|
+
|
|
328
|
+
:param column: Name of the column to use as weights.
|
|
207
329
|
:param preserve_old: If True, keeps the old weights as a column when
|
|
208
330
|
new weights are provided.
|
|
209
|
-
:type
|
|
331
|
+
:type column: str
|
|
210
332
|
"""
|
|
333
|
+
import warnings
|
|
334
|
+
|
|
335
|
+
warnings.warn(
|
|
336
|
+
"set_weight_col is deprecated and will be removed in a "
|
|
337
|
+
"future version. Use set_weights(column_name) instead.",
|
|
338
|
+
DeprecationWarning,
|
|
339
|
+
stacklevel=2,
|
|
340
|
+
)
|
|
341
|
+
|
|
211
342
|
if preserve_old and self.weights_col is not None:
|
|
212
343
|
self["old_" + self.weights_col] = self.weights
|
|
213
344
|
|
|
@@ -215,6 +346,15 @@ class MicroDataFrame(pd.DataFrame):
|
|
|
215
346
|
self.weights_col = column
|
|
216
347
|
self._link_all_weights()
|
|
217
348
|
|
|
349
|
+
def nullify_weights(self) -> None:
|
|
350
|
+
"""Set all weights to 1, effectively making the DataFrame unweighted.
|
|
351
|
+
|
|
352
|
+
This is useful for comparing weighted and unweighted statistics or when
|
|
353
|
+
you want to temporarily ignore weights.
|
|
354
|
+
"""
|
|
355
|
+
self.weights = np.ones(len(self))
|
|
356
|
+
self._link_all_weights()
|
|
357
|
+
|
|
218
358
|
def __getitem__(
|
|
219
359
|
self, key: Union[str, List]
|
|
220
360
|
) -> Union[pd.Series, pd.DataFrame]:
|
|
@@ -303,6 +443,138 @@ class MicroDataFrame(pd.DataFrame):
|
|
|
303
443
|
res = MicroDataFrame(res, weights=self.weights.copy(deep))
|
|
304
444
|
return res
|
|
305
445
|
|
|
446
|
+
def drop(
|
|
447
|
+
self,
|
|
448
|
+
labels=None,
|
|
449
|
+
axis=0,
|
|
450
|
+
index=None,
|
|
451
|
+
columns=None,
|
|
452
|
+
level=None,
|
|
453
|
+
inplace=False,
|
|
454
|
+
errors="raise",
|
|
455
|
+
):
|
|
456
|
+
"""Drop specified labels from rows or columns.
|
|
457
|
+
|
|
458
|
+
This method supports all parameters of pandas DataFrame.drop(),
|
|
459
|
+
including the 'inplace' parameter.
|
|
460
|
+
|
|
461
|
+
:param labels: Index or column labels to drop.
|
|
462
|
+
:param axis: Whether to drop labels from the index (0 or 'index') or
|
|
463
|
+
columns (1 or 'columns').
|
|
464
|
+
:param index: Alternative to specifying axis (labels, axis=0 is
|
|
465
|
+
equivalent to index=labels).
|
|
466
|
+
:param columns: Alternative to specifying axis (labels, axis=1 is
|
|
467
|
+
equivalent to columns=labels).
|
|
468
|
+
:param level: For MultiIndex, level from which the labels will be
|
|
469
|
+
removed.
|
|
470
|
+
:param inplace: If False, return a copy. Otherwise, do operation
|
|
471
|
+
inplace and return None.
|
|
472
|
+
:param errors: If 'ignore', suppress error and only existing labels are
|
|
473
|
+
dropped.
|
|
474
|
+
:return: MicroDataFrame or None if inplace=True.
|
|
475
|
+
"""
|
|
476
|
+
if inplace:
|
|
477
|
+
weights_backup = self.weights.copy()
|
|
478
|
+
# Perform in-place drop on the parent DataFrame
|
|
479
|
+
super().drop(
|
|
480
|
+
labels=labels,
|
|
481
|
+
axis=axis,
|
|
482
|
+
index=index,
|
|
483
|
+
columns=columns,
|
|
484
|
+
level=level,
|
|
485
|
+
inplace=True,
|
|
486
|
+
errors=errors,
|
|
487
|
+
)
|
|
488
|
+
self.weights = weights_backup
|
|
489
|
+
self._link_all_weights()
|
|
490
|
+
return None
|
|
491
|
+
else:
|
|
492
|
+
res = super().drop(
|
|
493
|
+
labels=labels,
|
|
494
|
+
axis=axis,
|
|
495
|
+
index=index,
|
|
496
|
+
columns=columns,
|
|
497
|
+
level=level,
|
|
498
|
+
inplace=False,
|
|
499
|
+
errors=errors,
|
|
500
|
+
)
|
|
501
|
+
return MicroDataFrame(res, weights=self.weights)
|
|
502
|
+
|
|
503
|
+
def merge(
|
|
504
|
+
self,
|
|
505
|
+
right,
|
|
506
|
+
how="inner",
|
|
507
|
+
on=None,
|
|
508
|
+
left_on=None,
|
|
509
|
+
right_on=None,
|
|
510
|
+
left_index=False,
|
|
511
|
+
right_index=False,
|
|
512
|
+
sort=False,
|
|
513
|
+
suffixes=("_x", "_y"),
|
|
514
|
+
copy=True,
|
|
515
|
+
indicator=False,
|
|
516
|
+
validate=None,
|
|
517
|
+
):
|
|
518
|
+
"""Merge DataFrame or named Series objects with a database-style join.
|
|
519
|
+
|
|
520
|
+
This method overrides pandas DataFrame.merge() to return a
|
|
521
|
+
MicroDataFrame.
|
|
522
|
+
|
|
523
|
+
:param right: Object to merge with.
|
|
524
|
+
:param how: Type of merge to be performed.
|
|
525
|
+
:param on: Column or index level names to join on.
|
|
526
|
+
:param left_on: Column or index level names to join on in the left
|
|
527
|
+
DataFrame.
|
|
528
|
+
:param right_on: Column or index level names to join on in the right
|
|
529
|
+
DataFrame.
|
|
530
|
+
:param left_index: Use the index from the left DataFrame as the join
|
|
531
|
+
key(s).
|
|
532
|
+
:param right_index: Use the index from the right DataFrame as the join
|
|
533
|
+
key(s).
|
|
534
|
+
:param sort: Sort the join keys lexicographically in the result
|
|
535
|
+
DataFrame.
|
|
536
|
+
:param suffixes: A length-2 sequence where each element is optionally a
|
|
537
|
+
string indicating the suffix to add to overlapping column names.
|
|
538
|
+
:param copy: If False, avoid copy if possible.
|
|
539
|
+
:param indicator: If True, adds a column to output DataFrame called
|
|
540
|
+
"_merge".
|
|
541
|
+
:param validate: If specified, checks if merge is of specified type.
|
|
542
|
+
:return: MicroDataFrame with merged data.
|
|
543
|
+
"""
|
|
544
|
+
res = super().merge(
|
|
545
|
+
right,
|
|
546
|
+
how=how,
|
|
547
|
+
on=on,
|
|
548
|
+
left_on=left_on,
|
|
549
|
+
right_on=right_on,
|
|
550
|
+
left_index=left_index,
|
|
551
|
+
right_index=right_index,
|
|
552
|
+
sort=sort,
|
|
553
|
+
suffixes=suffixes,
|
|
554
|
+
copy=copy,
|
|
555
|
+
indicator=indicator,
|
|
556
|
+
validate=validate,
|
|
557
|
+
)
|
|
558
|
+
|
|
559
|
+
# For inner join, both dataframes must have the same weights on
|
|
560
|
+
# matching rows. For now, we'll use the left dataframe's weights.
|
|
561
|
+
# This is a simplification and may need more sophisticated handling
|
|
562
|
+
return MicroDataFrame(res, weights=self.weights)
|
|
563
|
+
|
|
564
|
+
def __getattr__(self, name):
|
|
565
|
+
"""Allow accessing columns as attributes (e.g., df.column_name).
|
|
566
|
+
|
|
567
|
+
This enables more intuitive column access while preserving MicroSeries
|
|
568
|
+
functionality when accessing columns.
|
|
569
|
+
|
|
570
|
+
:param name: Attribute name to access
|
|
571
|
+
:return: MicroSeries if the attribute is a column, otherwise delegates
|
|
572
|
+
to parent
|
|
573
|
+
"""
|
|
574
|
+
if name in self.columns:
|
|
575
|
+
return self[name]
|
|
576
|
+
return super().__getattr__(name)
|
|
577
|
+
|
|
306
578
|
def equals(self, other: "MicroDataFrame") -> bool:
|
|
307
579
|
equal_values = super().equals(other)
|
|
308
580
|
equal_weights = self.weights.equals(other.weights)
|
|
@@ -462,6 +734,7 @@ class MicroDataFrame(pd.DataFrame):
|
|
|
462
734
|
|
|
463
735
|
class MicroDataFrameGroupBy(pd.core.groupby.generic.DataFrameGroupBy):
|
|
464
736
|
def _init(self, by: Union[str, List]):
|
|
737
|
+
self._by = by
|
|
465
738
|
self.columns = list(self.obj.columns)
|
|
466
739
|
if isinstance(by, list):
|
|
467
740
|
for column in by:
|
|
@@ -469,18 +742,32 @@ class MicroDataFrameGroupBy(pd.core.groupby.generic.DataFrameGroupBy):
|
|
|
469
742
|
elif isinstance(by, str):
|
|
470
743
|
self.columns.remove(by)
|
|
471
744
|
self.columns.remove("__tmp_weights")
|
|
745
|
+
# Filter to only numeric columns
|
|
746
|
+
self.numeric_columns = [
|
|
747
|
+
col
|
|
748
|
+
for col in self.columns
|
|
749
|
+
if pd.api.types.is_numeric_dtype(self.obj[col])
|
|
750
|
+
]
|
|
751
|
+
# Store reference to weights groupby for column selection
|
|
752
|
+
self._weights_groupby = copy.deepcopy(
|
|
753
|
+
super().__getitem__("__tmp_weights")
|
|
754
|
+
)
|
|
472
755
|
for fn_name in MicroSeries.SCALAR_FUNCTIONS:
|
|
473
756
|
|
|
474
757
|
def get_fn(name):
|
|
475
758
|
def fn(*args, **kwargs):
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
759
|
+
results = {}
|
|
760
|
+
for col in self.numeric_columns:
|
|
761
|
+
try:
|
|
762
|
+
results[col] = getattr(getattr(self, col), name)(
|
|
479
763
|
*args, **kwargs
|
|
480
764
|
)
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
765
|
+
except Exception:
|
|
766
|
+
# Skip columns that can't be aggregated
|
|
767
|
+
pass
|
|
768
|
+
# Return plain DataFrame - aggregated results don't have
|
|
769
|
+
# per-row weights (weights were already applied)
|
|
770
|
+
return pd.DataFrame(results) if results else pd.DataFrame()
|
|
484
771
|
|
|
485
772
|
return fn
|
|
486
773
|
|
|
@@ -489,15 +776,108 @@ class MicroDataFrameGroupBy(pd.core.groupby.generic.DataFrameGroupBy):
|
|
|
489
776
|
|
|
490
777
|
def get_fn(name) -> Callable:
|
|
491
778
|
def fn(*args, **kwargs) -> Union[pd.Series, pd.DataFrame]:
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
779
|
+
results = {}
|
|
780
|
+
for col in self.numeric_columns:
|
|
781
|
+
try:
|
|
782
|
+
results[col] = getattr(getattr(self, col), name)(
|
|
495
783
|
*args, **kwargs
|
|
496
784
|
)
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
785
|
+
except Exception:
|
|
786
|
+
# Skip columns that can't be aggregated
|
|
787
|
+
pass
|
|
788
|
+
# Return plain DataFrame - aggregated results don't have
|
|
789
|
+
# per-row weights (weights were already applied)
|
|
790
|
+
return pd.DataFrame(results) if results else pd.DataFrame()
|
|
500
791
|
|
|
501
792
|
return fn
|
|
502
793
|
|
|
503
794
|
setattr(self, fn_name, get_fn(fn_name))
|
|
795
|
+
|
|
796
|
+
def __getitem__(
|
|
797
|
+
self, key: Union[str, List]
|
|
798
|
+
) -> Union["MicroSeriesGroupBy", "MicroDataFrameGroupBy"]:
|
|
799
|
+
"""Select columns from the groupby object while preserving weights.
|
|
800
|
+
|
|
801
|
+
This ensures that operations like groupby(col)["y"].sum() or
|
|
802
|
+
groupby(col)[["y"]].sum() use weighted aggregation.
|
|
803
|
+
|
|
804
|
+
:param key: Column name or list of column names
|
|
805
|
+
:return: MicroSeriesGroupBy for single column, MicroDataFrameGroupBy
|
|
806
|
+
for multiple columns
|
|
807
|
+
"""
|
|
808
|
+
if isinstance(key, str):
|
|
809
|
+
# Single column - return MicroSeriesGroupBy
|
|
810
|
+
result = super().__getitem__(key)
|
|
811
|
+
result.__class__ = MicroSeriesGroupBy
|
|
812
|
+
result._init()
|
|
813
|
+
result.weights = self._weights_groupby
|
|
814
|
+
return result
|
|
815
|
+
else:
|
|
816
|
+
# Multiple columns - return a new MicroDataFrameGroupBy
|
|
817
|
+
# with only the selected columns
|
|
818
|
+
result = super().__getitem__(key)
|
|
819
|
+
result.__class__ = MicroDataFrameGroupBy
|
|
820
|
+
# Re-initialize with the subset of columns
|
|
821
|
+
result._by = self._by
|
|
822
|
+
result.columns = list(key) if hasattr(key, "__iter__") else [key]
|
|
823
|
+
result.numeric_columns = [
|
|
824
|
+
col
|
|
825
|
+
for col in result.columns
|
|
826
|
+
if pd.api.types.is_numeric_dtype(result.obj[col])
|
|
827
|
+
]
|
|
828
|
+
result._weights_groupby = self._weights_groupby
|
|
829
|
+
# Set up the column attributes as MicroSeriesGroupBy
|
|
830
|
+
for col in result.columns:
|
|
831
|
+
col_gb = super().__getitem__(col)
|
|
832
|
+
col_gb.__class__ = MicroSeriesGroupBy
|
|
833
|
+
col_gb._init()
|
|
834
|
+
col_gb.weights = self._weights_groupby
|
|
835
|
+
setattr(result, col, col_gb)
|
|
836
|
+
# Set up the scalar and vector functions
|
|
837
|
+
for fn_name in MicroSeries.SCALAR_FUNCTIONS:
|
|
838
|
+
|
|
839
|
+
def get_scalar_fn(name, res):
|
|
840
|
+
def fn(*args, **kwargs):
|
|
841
|
+
results = {}
|
|
842
|
+
for col in res.numeric_columns:
|
|
843
|
+
try:
|
|
844
|
+
results[col] = getattr(
|
|
845
|
+
getattr(res, col), name
|
|
846
|
+
)(*args, **kwargs)
|
|
847
|
+
except Exception:
|
|
848
|
+
pass
|
|
849
|
+
# Return plain DataFrame - aggregated results don't
|
|
850
|
+
# have per-row weights (weights were already applied)
|
|
851
|
+
return (
|
|
852
|
+
pd.DataFrame(results)
|
|
853
|
+
if results
|
|
854
|
+
else pd.DataFrame()
|
|
855
|
+
)
|
|
856
|
+
|
|
857
|
+
return fn
|
|
858
|
+
|
|
859
|
+
setattr(result, fn_name, get_scalar_fn(fn_name, result))
|
|
860
|
+
for fn_name in MicroSeries.VECTOR_FUNCTIONS:
|
|
861
|
+
|
|
862
|
+
def get_vector_fn(name, res):
|
|
863
|
+
def fn(*args, **kwargs):
|
|
864
|
+
results = {}
|
|
865
|
+
for col in res.numeric_columns:
|
|
866
|
+
try:
|
|
867
|
+
results[col] = getattr(
|
|
868
|
+
getattr(res, col), name
|
|
869
|
+
)(*args, **kwargs)
|
|
870
|
+
except Exception:
|
|
871
|
+
pass
|
|
872
|
+
# Return plain DataFrame - aggregated results don't
|
|
873
|
+
# have per-row weights (weights were already applied)
|
|
874
|
+
return (
|
|
875
|
+
pd.DataFrame(results)
|
|
876
|
+
if results
|
|
877
|
+
else pd.DataFrame()
|
|
878
|
+
)
|
|
879
|
+
|
|
880
|
+
return fn
|
|
881
|
+
|
|
882
|
+
setattr(result, fn_name, get_vector_fn(fn_name, result))
|
|
883
|
+
return result
|
|
@@ -66,6 +66,14 @@ class MicroSeries(pd.Series):
|
|
|
66
66
|
|
|
67
67
|
self.weights = pd.Series(weights, dtype=float)
|
|
68
68
|
|
|
69
|
+
def nullify_weights(self) -> None:
|
|
70
|
+
"""Set all weights to 1, effectively making the Series unweighted.
|
|
71
|
+
|
|
72
|
+
This is useful for comparing weighted and unweighted statistics or when
|
|
73
|
+
you want to temporarily ignore weights.
|
|
74
|
+
"""
|
|
75
|
+
self.weights = pd.Series(np.ones(len(self)), dtype=float)
|
|
76
|
+
|
|
69
77
|
@vector_function
|
|
70
78
|
def weight(self) -> pd.Series:
|
|
71
79
|
"""Calculates the weighted value of the MicroSeries.
|
|
@@ -23,6 +23,14 @@ def test_df_init() -> None:
|
|
|
23
23
|
df.set_weight_col("w")
|
|
24
24
|
assert df.a.mean() == np.average(arr, weights=w)
|
|
25
25
|
|
|
26
|
+
# Test set_weights with string (column name)
|
|
27
|
+
df2 = mdf.MicroDataFrame()
|
|
28
|
+
df2["a"] = arr
|
|
29
|
+
df2["w"] = w
|
|
30
|
+
df2.set_weights("w") # Using string column name instead of set_weight_col
|
|
31
|
+
assert df2.a.mean() == np.average(arr, weights=w)
|
|
32
|
+
assert np.array_equal(df2.weights.values, w)
|
|
33
|
+
|
|
26
34
|
|
|
27
35
|
def test_handles_empty_index() -> None:
|
|
28
36
|
arr = np.array([0, 1, 1])
|
|
@@ -279,3 +287,60 @@ def test_reset_index_inplace() -> None:
|
|
|
279
287
|
assert "second" in mdf_multi.columns
|
|
280
288
|
assert list(mdf_multi.index) == [0, 1, 2, 3]
|
|
281
289
|
np.testing.assert_array_equal(mdf_multi.weights.values, weights)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def test_loc_preserves_weights() -> None:
|
|
293
|
+
"""Test that .loc[] returns MicroDataFrame with proper weights (issue
|
|
294
|
+
#265)."""
|
|
295
|
+
df = mdf.MicroDataFrame(
|
|
296
|
+
{"one": [1, 1, 1, 1, 1]}, weights=[10, 20, 30, 40, 50]
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
# Filter all rows (should get same weights)
|
|
300
|
+
filtered = df.loc[df.one == 1]
|
|
301
|
+
assert isinstance(filtered, MicroDataFrame)
|
|
302
|
+
assert filtered.one.sum() == 150.0 # Weighted sum
|
|
303
|
+
|
|
304
|
+
# Partial filter
|
|
305
|
+
df2 = mdf.MicroDataFrame(
|
|
306
|
+
{"x": [1, 2, 3, 4, 5]}, weights=[10, 20, 30, 40, 50]
|
|
307
|
+
)
|
|
308
|
+
subset = df2.loc[df2.x > 2]
|
|
309
|
+
assert isinstance(subset, MicroDataFrame)
|
|
310
|
+
assert subset.x.sum() == 500.0 # 3*30 + 4*40 + 5*50 = 500
|
|
311
|
+
np.testing.assert_array_equal(subset.weights.values, [30.0, 40.0, 50.0])
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def test_iloc_preserves_weights() -> None:
|
|
315
|
+
"""Test that .iloc[] returns MicroDataFrame with proper weights."""
|
|
316
|
+
df = mdf.MicroDataFrame(
|
|
317
|
+
{"x": [1, 2, 3, 4, 5]}, weights=[10, 20, 30, 40, 50]
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
# Select rows by position
|
|
321
|
+
subset = df.iloc[2:5]
|
|
322
|
+
assert isinstance(subset, MicroDataFrame)
|
|
323
|
+
assert subset.x.sum() == 500.0 # 3*30 + 4*40 + 5*50 = 500
|
|
324
|
+
np.testing.assert_array_equal(subset.weights.values, [30.0, 40.0, 50.0])
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def test_groupby_column_selection() -> None:
|
|
328
|
+
"""Test that groupby column selection preserves weights (issue #193)."""
|
|
329
|
+
d = mdf.MicroDataFrame(
|
|
330
|
+
dict(g=["a", "a", "b"], y=[1, 2, 3]), weights=[4, 5, 6]
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
# Test single column string selection
|
|
334
|
+
result_str = d.groupby("g")["y"].sum()
|
|
335
|
+
assert result_str["a"] == 14.0 # 1*4 + 2*5 = 14
|
|
336
|
+
assert result_str["b"] == 18.0 # 3*6 = 18
|
|
337
|
+
|
|
338
|
+
# Test list column selection
|
|
339
|
+
result_list = d.groupby("g")[["y"]].sum()
|
|
340
|
+
assert result_list.loc["a", "y"] == 14.0
|
|
341
|
+
assert result_list.loc["b", "y"] == 18.0
|
|
342
|
+
|
|
343
|
+
# Aggregated results should be plain DataFrame (no spurious weight column)
|
|
344
|
+
result_all = d.groupby("g").sum()
|
|
345
|
+
assert "weight" not in result_all.columns
|
|
346
|
+
assert list(result_all.columns) == ["y"]
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: microdf-python
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: Weighted pandas DataFrames and Series for survey microdata
|
|
5
|
-
Author-email: Max Ghenis <max@
|
|
5
|
+
Author-email: Max Ghenis <max@policyengine.org>
|
|
6
6
|
License: MIT
|
|
7
7
|
Requires-Python: >=3.9
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
@@ -20,8 +20,6 @@ Requires-Dist: linecheck; extra == "dev"
|
|
|
20
20
|
Requires-Dist: pytest; extra == "dev"
|
|
21
21
|
Requires-Dist: pytest-cov; extra == "dev"
|
|
22
22
|
Requires-Dist: setuptools; extra == "dev"
|
|
23
|
-
Provides-Extra: docs
|
|
24
|
-
Requires-Dist: jupyter_book; extra == "docs"
|
|
25
23
|
Dynamic: license-file
|
|
26
24
|
|
|
27
25
|
[](https://github.com/PolicyEngine/microdf/actions)
|
|
@@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "microdf-python"
|
|
7
|
-
version = "1.0
|
|
7
|
+
version = "1.1.0"
|
|
8
8
|
description = "Weighted pandas DataFrames and Series for survey microdata"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = [
|
|
11
|
-
{ name = "Max Ghenis", email = "max@
|
|
11
|
+
{ name = "Max Ghenis", email = "max@policyengine.org" }
|
|
12
12
|
]
|
|
13
13
|
license = { text = "MIT" }
|
|
14
14
|
requires-python = ">=3.9"
|
|
@@ -30,9 +30,8 @@ dev = [
|
|
|
30
30
|
"pytest-cov",
|
|
31
31
|
"setuptools",
|
|
32
32
|
]
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
]
|
|
33
|
+
# Note: Documentation uses MyST (Jupyter Book 2.0) which is installed via npm
|
|
34
|
+
# Run: cd docs && myst build --html
|
|
36
35
|
|
|
37
36
|
[tool.setuptools.packages.find]
|
|
38
37
|
where = ["."]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|