microdf-python 1.2.2__tar.gz → 1.3.0__tar.gz

{microdf_python-1.2.2/microdf_python.egg-info → microdf_python-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: microdf-python
-Version: 1.2.2
+Version: 1.3.0
 Summary: Weighted pandas DataFrames and Series for survey microdata
 Author-email: Max Ghenis <max@policyengine.org>
 License: MIT
@@ -11,12 +11,8 @@ Requires-Dist: numpy
 Requires-Dist: pandas
 Provides-Extra: dev
 Requires-Dist: codecov; extra == "dev"
-Requires-Dist: flake8; extra == "dev"
-Requires-Dist: flake8-pyproject; extra == "dev"
-Requires-Dist: black; extra == "dev"
+Requires-Dist: ruff>=0.9.0; extra == "dev"
 Requires-Dist: docformatter; extra == "dev"
-Requires-Dist: isort; extra == "dev"
-Requires-Dist: linecheck; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
 Requires-Dist: setuptools; extra == "dev"

{microdf_python-1.2.2 → microdf_python-1.3.0}/microdf/microdataframe.py

@@ -147,9 +147,7 @@ class MicroDataFrame(pd.DataFrame):
             for col in self.columns:
                 if pd.api.types.is_numeric_dtype(self[col]):
                     try:
-                        results[col] = getattr(self[col], name)(
-                            *args, **kwargs
-                        )
+                        results[col] = getattr(self[col], name)(*args, **kwargs)
                     except Exception:
                         # Skip columns that can't be aggregated
                         pass
@@ -224,9 +222,7 @@ class MicroDataFrame(pd.DataFrame):
                 for col in self.columns:
                     if pd.api.types.is_numeric_dtype(self[col]):
                         try:
-                            results[col] = getattr(self[col], name)(
-                                *args, **kwargs
-                            )
+                            results[col] = getattr(self[col], name)(*args, **kwargs)
                         except Exception:
                             # Skip columns that can't be aggregated
                             pass
@@ -309,7 +305,11 @@ class MicroDataFrame(pd.DataFrame):
 
         if isinstance(weights, str):
             self.weights_col = weights
-            self.weights = pd.Series(self[weights], dtype=float)
+            self.weights = pd.Series(
+                np.asarray(self[weights]),
+                index=self.index,
+                dtype=float,
+            )
             self._link_all_weights()
         elif weights is not None:
             if len(weights) != len(self):
@@ -318,14 +318,21 @@ class MicroDataFrame(pd.DataFrame):
                     f"length of DataFrame ({len(self)})."
                 )
             self.weights_col = None
+            # Align weights to self.index. Without this, weighted ops
+            # (self[col].multiply(self.weights) in .sum()) align on
+            # label, so any non-default index silently produces all-NaN
+            # and aggregations collapse to 0. If a Series is passed in,
+            # strip its index so we position-align to self.index.
+            if isinstance(weights, pd.Series):
+                weights = weights.values
             with warnings.catch_warnings():
                 warnings.filterwarnings("ignore", category=UserWarning)
-                self.weights = pd.Series(weights, dtype=float)
+                self.weights = pd.Series(
+                    np.asarray(weights), index=self.index, dtype=float
+                )
             self._link_all_weights()
 
-    def set_weight_col(
-        self, column: str, preserve_old: Optional[bool] = False
-    ) -> None:
+    def set_weight_col(self, column: str, preserve_old: Optional[bool] = False) -> None:
         """Sets the weights for the MicroDataFrame by specifying the name of
         the weight column.
 
@@ -422,8 +429,9 @@ class MicroDataFrame(pd.DataFrame):
         :return: MicroDataFrame with reset index or None if inplace=True.
         """
         if inplace:
-            weights_backup = self.weights.copy()
-            # Perform in-place reset on the parent DataFrame
+            # Snapshot weight *values* positionally — the index is about
+            # to change and reset_index preserves row order.
+            weight_values = np.asarray(self.weights.values, dtype=float)
             super().reset_index(
                 level=level,
                 drop=drop,
@@ -433,7 +441,7 @@ class MicroDataFrame(pd.DataFrame):
                 allow_duplicates=allow_duplicates,
                 names=names,
             )
-            self.weights = weights_backup
+            self.weights = pd.Series(weight_values, index=self.index, dtype=float)
             self._link_all_weights()
             return None
         else:
@@ -446,13 +454,22 @@ class MicroDataFrame(pd.DataFrame):
                 allow_duplicates=allow_duplicates,
                 names=names,
             )
-            return MicroDataFrame(res, weights=self.weights)
+            out = MicroDataFrame(res, weights=self.weights.values)
+            # Ensure weights align to res.index (reset_index changes the
+            # index but preserves row order, so pass values positionally).
+            out.weights = pd.Series(
+                np.asarray(self.weights.values, dtype=float),
+                index=out.index,
+                dtype=float,
+            )
+            return out
 
     def copy(self, deep: Optional[bool] = True) -> "MicroDataFrame":
         res = super().copy(deep)
-        # This changes the original columns to Series. Undo it:
-        for col in self.columns:
-            self[col] = MicroSeries(self[col])
+        # super().copy() corrupts self's column types to plain Series.
+        # Restore them in O(N) instead of O(N²) by calling
+        # _link_all_weights once rather than per-column __setitem__.
+        self._link_all_weights()
         res = MicroDataFrame(res, weights=self.weights.copy(deep))
         return res
 
@@ -486,8 +503,11 @@ class MicroDataFrame(pd.DataFrame):
             dropped.
         :return: MicroDataFrame or None if inplace=True.
         """
+        row_drop = axis in (0, "index") or index is not None
         if inplace:
-            weights_backup = self.weights.copy()
+            # Snapshot the pre-drop weights keyed by the pre-drop index so
+            # we can reindex to the surviving rows after the drop.
+            pre_drop_weights = pd.Series(self.weights.values, index=self.index.copy())
             # Perform in-place drop on the parent DataFrame
             super().drop(
                 labels=labels,
@@ -498,7 +518,15 @@ class MicroDataFrame(pd.DataFrame):
                 inplace=True,
                 errors=errors,
             )
-            self.weights = weights_backup
+            if row_drop:
+                surviving = pre_drop_weights.reindex(self.index)
+                self.weights = pd.Series(
+                    surviving.values, index=self.index, dtype=float
+                )
+            else:
+                self.weights = pd.Series(
+                    pre_drop_weights.values, index=self.index, dtype=float
+                )
             self._link_all_weights()
             return None
         else:
@@ -511,7 +539,19 @@ class MicroDataFrame(pd.DataFrame):
                 inplace=False,
                 errors=errors,
             )
-            return MicroDataFrame(res, weights=self.weights)
+            if row_drop:
+                # Row drop: keep only the weights for surviving rows,
+                # in the order of the resulting DataFrame.
+                pre_drop_weights = pd.Series(self.weights.values, index=self.index)
+                new_weights = pre_drop_weights.reindex(res.index).values
+            else:
+                new_weights = self.weights.values
+            out = MicroDataFrame(res, weights=new_weights)
+            # Guard against the set_weights path building weights with a
+            # default RangeIndex, which would misalign against res.index
+            # and silently zero weighted aggregations.
+            out.weights = pd.Series(new_weights, index=out.index, dtype=float)
+            return out
 
     def merge(
         self,
@@ -554,7 +594,17 @@ class MicroDataFrame(pd.DataFrame):
         :param validate: If specified, checks if merge is of specified type.
         :return: MicroDataFrame with merged data.
         """
-        res = super().merge(
+        # Attach the left weights as a temporary column so pandas' merge
+        # propagates them onto every surviving output row (including
+        # many-to-many row duplications, inner-join filtering, and
+        # left-with-missing NaNs). We then strip the column back off.
+        tmp = "__microdf_weights__"
+        # Avoid clobbering if this exact name is already used.
+        while tmp in self.columns or tmp in right.columns:
+            tmp += "_"
+        left_df = pd.DataFrame(self).copy()
+        left_df[tmp] = np.asarray(self.weights.values, dtype=float)
+        res = left_df.merge(
             right,
             how=how,
             on=on,
@@ -568,11 +618,17 @@ class MicroDataFrame(pd.DataFrame):
             indicator=indicator,
             validate=validate,
         )
-
-        # For inner join, both dataframes must have the same weights on
-        # matching rows. For now, we'll use the left dataframe's weights.
-        # This is a simplification and may need more sophisticated handling
-        return MicroDataFrame(res, weights=self.weights)
+        # Pull out the propagated weights. Rows with no left match in a
+        # right/outer join get NaN weight — fill with 0 so they don't
+        # poison later aggregations (a user who needs a different
+        # convention can override afterwards).
+        merged_weights = res[tmp].fillna(0).to_numpy(dtype=float)
+        res = res.drop(columns=[tmp])
+        out = MicroDataFrame(res, weights=merged_weights)
+        # Ensure the weights Series aligns with res.index regardless of
+        # the default-RangeIndex behavior of set_weights.
+        out.weights = pd.Series(merged_weights, index=out.index, dtype=float)
+        return out
 
     def __getattr__(self, name):
         """Allow accessing columns as attributes (e.g., df.column_name).
@@ -594,9 +650,7 @@ class MicroDataFrame(pd.DataFrame):
         return equal_values and equal_weights
 
     @get_args_as_micro_series()
-    def groupby(
-        self, by: Union[str, List], *args, **kwargs
-    ) -> "MicroDataFrameGroupBy":
+    def groupby(self, by: Union[str, List], *args, **kwargs) -> "MicroDataFrameGroupBy":
         """Returns a GroupBy object with MicroSeriesGroupBy objects for each
         column.
 
@@ -606,10 +660,16 @@ class MicroDataFrame(pd.DataFrame):
         return: DataFrameGroupBy object with columns using weights
         rtype: DataFrameGroupBy
         """
-        self["__tmp_weights"] = self.weights
-        gb = super().groupby(by, *args, **kwargs)
+        # Build the groupby on a *copy* that carries a ``__tmp_weights``
+        # column. We used to set this column on ``self`` directly, which
+        # permanently leaked the weight column onto the caller's
+        # DataFrame — any later ``df.sum()`` or ``list(df.columns)``
+        # would then include it.
+        staged = pd.DataFrame(self).copy()
+        staged["__tmp_weights"] = np.asarray(self.weights.values, dtype=float)
+        gb = staged.groupby(by, *args, **kwargs)
         weights = copy.deepcopy(gb["__tmp_weights"])
-        for col in self.columns:  # df.groupby(...)[col]s use weights
+        for col in staged.columns:  # df.groupby(...)[col]s use weights
             res = gb[col]
             res.__class__ = MicroSeriesGroupBy
             res._init()
@@ -757,14 +817,10 @@ class MicroDataFrameGroupBy(pd.core.groupby.generic.DataFrameGroupBy):
         self.columns.remove("__tmp_weights")
         # Filter to only numeric columns
         self.numeric_columns = [
-            col
-            for col in self.columns
-            if pd.api.types.is_numeric_dtype(self.obj[col])
+            col for col in self.columns if pd.api.types.is_numeric_dtype(self.obj[col])
         ]
         # Store reference to weights groupby for column selection
-        self._weights_groupby = copy.deepcopy(
-            super().__getitem__("__tmp_weights")
-        )
+        self._weights_groupby = copy.deepcopy(super().__getitem__("__tmp_weights"))
         for fn_name in MicroSeries.SCALAR_FUNCTIONS:
 
             def get_fn(name):
@@ -854,18 +910,14 @@ class MicroDataFrameGroupBy(pd.core.groupby.generic.DataFrameGroupBy):
                         results = {}
                         for col in res.numeric_columns:
                             try:
-                                results[col] = getattr(
-                                    getattr(res, col), name
-                                )(*args, **kwargs)
+                                results[col] = getattr(getattr(res, col), name)(
+                                    *args, **kwargs
+                                )
                             except Exception:
                                 pass
                         # Return plain DataFrame - aggregated results don't
                         # have per-row weights (weights were already applied)
-                        return (
-                            pd.DataFrame(results)
-                            if results
-                            else pd.DataFrame()
-                        )
+                        return pd.DataFrame(results) if results else pd.DataFrame()
 
                     return fn
 
@@ -877,18 +929,14 @@ class MicroDataFrameGroupBy(pd.core.groupby.generic.DataFrameGroupBy):
                         results = {}
                         for col in res.numeric_columns:
                             try:
-                                results[col] = getattr(
-                                    getattr(res, col), name
-                                )(*args, **kwargs)
+                                results[col] = getattr(getattr(res, col), name)(
+                                    *args, **kwargs
+                                )
                             except Exception:
                                 pass
                         # Return plain DataFrame - aggregated results don't
                         # have per-row weights (weights were already applied)
-                        return (
-                            pd.DataFrame(results)
-                            if results
-                            else pd.DataFrame()
-                        )
+                        return pd.DataFrame(results) if results else pd.DataFrame()
 
                     return fn