nested-pandas 0.2.0__tar.gz → 0.2.2__tar.gz

{nested_pandas-0.2.0/src/nested_pandas.egg-info → nested_pandas-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nested-pandas
-Version: 0.2.0
+Version: 0.2.2
 Author-email: LINCC Frameworks <brantd@uw.edu>
 License: MIT License
         

{nested_pandas-0.2.0 → nested_pandas-0.2.2}/src/nested_pandas/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.0'
-__version_tuple__ = version_tuple = (0, 2, 0)
+__version__ = version = '0.2.2'
+__version_tuple__ = version_tuple = (0, 2, 2)

{nested_pandas-0.2.0 → nested_pandas-0.2.2}/src/nested_pandas/nestedframe/core.py

@@ -211,7 +211,8 @@ class NestedFrame(pd.DataFrame):
         out_df = df[base_columns][~df.index.duplicated(keep="first")]
 
         # add nested
-        nested_columns = [col for col in df.columns if col not in base_columns]
+        if nested_columns is None:
+            nested_columns = [col for col in df.columns if col not in base_columns]
         return out_df.add_nested(df[nested_columns], name=name)
 
     @classmethod

{nested_pandas-0.2.0 → nested_pandas-0.2.2}/src/nested_pandas/series/accessor.py

@@ -251,14 +251,18 @@ class NestSeriesAccessor(Mapping):
             The flat-array field.
         """
 
-        # TODO: we should make proper missed values handling here
+        flat_chunks = []
+        for nested_chunk in self._series.array._chunked_array.iterchunks():
+            struct_array = cast(pa.StructArray, nested_chunk)
+            list_array = cast(pa.ListArray, struct_array.field(field))
+            flat_array = list_array.flatten()
+            flat_chunks.append(flat_array)
+
+        flat_chunked_array = pa.chunked_array(flat_chunks)
 
-        struct_array = cast(pa.StructArray, pa.array(self._series))
-        list_array = cast(pa.ListArray, struct_array.field(field))
-        flat_array = list_array.flatten()
         return pd.Series(
-            flat_array,
-            dtype=pd.ArrowDtype(flat_array.type),
+            flat_chunked_array,
+            dtype=pd.ArrowDtype(flat_chunked_array.type),
             index=self.get_flat_index(),
             name=field,
             copy=False,
@@ -277,11 +281,16 @@ class NestSeriesAccessor(Mapping):
         pd.Series
             The list-array field.
         """
-        struct_array = cast(pa.StructArray, pa.array(self._series))
-        list_array = struct_array.field(field)
+
+        list_chunks = []
+        for nested_chunk in self._series.array._chunked_array.iterchunks():
+            struct_array = cast(pa.StructArray, nested_chunk)
+            list_array = struct_array.field(field)
+            list_chunks.append(list_array)
+        list_chunked_array = pa.chunked_array(list_chunks)
         return pd.Series(
-            list_array,
-            dtype=pd.ArrowDtype(list_array.type),
+            list_chunked_array,
+            dtype=pd.ArrowDtype(list_chunked_array.type),
             index=self._series.index,
             name=field,
             copy=False,

{nested_pandas-0.2.0 → nested_pandas-0.2.2/src/nested_pandas.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nested-pandas
-Version: 0.2.0
+Version: 0.2.2
 Author-email: LINCC Frameworks <brantd@uw.edu>
 License: MIT License
         

{nested_pandas-0.2.0 → nested_pandas-0.2.2}/tests/nested_pandas/nestedframe/test_nestedframe.py

@@ -320,6 +320,24 @@ def test_recover_from_flat():
     assert nf2.equals(nf)
 
 
+def test_from_flat_omitting_columns():
+    """test that from_flat successfully produces subsets"""
+    flat = NestedFrame(
+        {"a": [1, 1, 1, 2, 2], "b": [2, 2, 2, 4, 4], "c": [1, 2, 3, 4, 5], "d": [2, 4, 6, 8, 10]},
+        index=[0, 0, 0, 1, 1],
+    )
+
+    # omit a base column
+    nf = NestedFrame.from_flat(flat, base_columns=["b"], nested_columns=["c", "d"])
+    assert list(nf.columns) == ["b", "nested"]
+    assert list(nf.nested.nest.fields) == ["c", "d"]
+
+    # omit a nested column
+    nf = NestedFrame.from_flat(flat, base_columns=["a", "b"], nested_columns=["c"])
+    assert list(nf.columns) == ["a", "b", "nested"]
+    assert list(nf.nested.nest.fields) == ["c"]
+
+
 def test_from_lists():
     """Test NestedFrame.from_lists behavior"""
     nf = NestedFrame(

{nested_pandas-0.2.0 → nested_pandas-0.2.2}/tests/nested_pandas/series/test_accessor.py

@@ -543,6 +543,32 @@ def test_get_list_series():
     )
 
 
+def test_get_list_series_multiple_chunks():
+    """Test that .nest.get_list_series() works when underlying array is chunked"""
+    struct_array = pa.StructArray.from_arrays(
+        arrays=[
+            [np.array([1, 2, 3]), np.array([4, 5, 6])],
+            [np.array([6, 4, 2]), np.array([1, 2, 3])],
+        ],
+        names=["a", "b"],
+    )
+    chunked_array = pa.chunked_array([struct_array] * 3)
+    series = pd.Series(chunked_array, dtype=NestedDtype(chunked_array.type), index=[5, 7, 9, 11, 13, 15])
+    assert series.array.num_chunks == 3
+
+    lists = series.nest.get_list_series("a")
+
+    assert_series_equal(
+        lists,
+        pd.Series(
+            data=[np.array([1, 2, 3]), np.array([4, 5, 6])] * 3,
+            dtype=pd.ArrowDtype(pa.list_(pa.int64())),
+            index=[5, 7, 9, 11, 13, 15],
+            name="a",
+        ),
+    )
+
+
 def test_get():
     """Test .nest.get() which is implemented by the base class"""
     series = pack_seq(
@@ -588,6 +614,33 @@ def test___getitem___single_field():
     )
 
 
+def test___getitem___single_field_multiple_chunks():
+    """Reproduces issue 142
+
+    https://github.com/lincc-frameworks/nested-pandas/issues/142
+    """
+    struct_array = pa.StructArray.from_arrays(
+        arrays=[
+            [np.array([1.0, 2.0, 3.0]), np.array([1.0, 2.0, 1.0])],
+            [np.array([4.0, 5.0, 6.0]), np.array([3.0, 4.0, 5.0])],
+        ],
+        names=["a", "b"],
+    )
+    chunked_array = pa.chunked_array([struct_array] * 3)
+    series = pd.Series(chunked_array, dtype=NestedDtype(chunked_array.type), index=[0, 1, 2, 3, 4, 5])
+    assert series.array.num_chunks == 3
+
+    assert_series_equal(
+        series.nest["a"],
+        pd.Series(
+            np.array([1.0, 2.0, 3.0, 1.0, 2.0, 1.0] * 3),
+            dtype=pd.ArrowDtype(pa.float64()),
+            index=[0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5],
+            name="a",
+        ),
+    )
+
+
 def test___getitem___multiple_fields():
     """Test that the .nest[["b", "a"]] works for multiple fields."""
     arrays = [