nested-pandas 0.4.2__tar.gz → 0.4.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.copier-answers.yml +2 -2
  2. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/smoke-test.yml +1 -1
  3. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/testing-and-coverage.yml +6 -8
  4. {nested_pandas-0.4.2/src/nested_pandas.egg-info → nested_pandas-0.4.4}/PKG-INFO +3 -3
  5. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/gettingstarted/quickstart.ipynb +3 -3
  6. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/reference/accessor.rst +1 -1
  7. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/reference/ext_array.rst +11 -0
  8. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/pyproject.toml +4 -13
  9. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/_version.py +2 -2
  10. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/nestedframe/core.py +109 -12
  11. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/nestedframe/io.py +2 -2
  12. nested_pandas-0.4.4/src/nested_pandas/series/_storage/__init__.py +3 -0
  13. nested_pandas-0.4.4/src/nested_pandas/series/_storage/list_struct_storage.py +84 -0
  14. nested_pandas-0.4.4/src/nested_pandas/series/_storage/struct_list_storage.py +74 -0
  15. nested_pandas-0.4.4/src/nested_pandas/series/_storage/table_storage.py +66 -0
  16. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/series/accessor.py +136 -33
  17. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/series/dtype.py +34 -3
  18. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/series/ext_array.py +214 -153
  19. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/series/packer.py +2 -2
  20. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/series/utils.py +142 -30
  21. {nested_pandas-0.4.2 → nested_pandas-0.4.4/src/nested_pandas.egg-info}/PKG-INFO +3 -3
  22. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas.egg-info/SOURCES.txt +6 -1
  23. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas.egg-info/requires.txt +1 -1
  24. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/nestedframe/test_nestedframe.py +131 -5
  25. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/series/test_accessor.py +98 -2
  26. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/series/test_dtype.py +16 -1
  27. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/series/test_ext_array.py +40 -45
  28. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/series/test_packer.py +13 -1
  29. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/series/test_series_utils.py +31 -0
  30. nested_pandas-0.4.4/tests/test_data/vsx-x-ztfdr22_lc-m31.parquet +0 -0
  31. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.git_archival.txt +0 -0
  32. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.gitattributes +0 -0
  33. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/ISSUE_TEMPLATE/0-general_issue.md +0 -0
  34. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/ISSUE_TEMPLATE/1-bug_report.md +0 -0
  35. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/ISSUE_TEMPLATE/2-feature_request.md +0 -0
  36. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/ISSUE_TEMPLATE/README.md +0 -0
  37. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/dependabot.yml +0 -0
  38. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/pull_request_template.md +0 -0
  39. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/README.md +0 -0
  40. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/asv-main.yml +0 -0
  41. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/asv-nightly.yml +0 -0
  42. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/asv-pr.yml +0 -0
  43. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/build-documentation.yml +0 -0
  44. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/pre-commit-ci.yml +0 -0
  45. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/publish-benchmarks-pr.yml +0 -0
  46. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/publish-to-pypi.yml +0 -0
  47. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.gitignore +0 -0
  48. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.pre-commit-config.yaml +0 -0
  49. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.readthedocs.yml +0 -0
  50. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.setup_dev.sh +0 -0
  51. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/LICENSE +0 -0
  52. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/README.md +0 -0
  53. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/benchmarks/README.md +0 -0
  54. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/benchmarks/__init__.py +0 -0
  55. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/benchmarks/asv.conf.json +0 -0
  56. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/benchmarks/benchmarks.py +0 -0
  57. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/Makefile +0 -0
  58. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/_static/custom.css +0 -0
  59. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/_templates/autosummary/base.rst +0 -0
  60. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/_templates/autosummary/class.rst +0 -0
  61. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/_templates/autosummary/module.rst +0 -0
  62. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/about/internals.rst +0 -0
  63. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/about/npd_internals.png +0 -0
  64. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/about.rst +0 -0
  65. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/conf.py +0 -0
  66. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/gettingstarted/contributing.rst +0 -0
  67. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/gettingstarted/installation.rst +0 -0
  68. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/gettingstarted.rst +0 -0
  69. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/index.rst +0 -0
  70. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/intro_images/loc_into_nested.png +0 -0
  71. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/intro_images/nestedframe_example.png +0 -0
  72. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/intro_images/pandas_dfs.png +0 -0
  73. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/intro_images/reduce.png +0 -0
  74. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/pre_executed/performance.ipynb +0 -0
  75. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/reference/nesteddtype.rst +0 -0
  76. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/reference/nestedframe.rst +0 -0
  77. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/reference/packer.rst +0 -0
  78. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/reference/utils.rst +0 -0
  79. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/reference.rst +0 -0
  80. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/requirements.txt +0 -0
  81. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/tutorials/README.md +0 -0
  82. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/tutorials/data_loading_notebook.ipynb +0 -0
  83. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/tutorials/data_manipulation.ipynb +0 -0
  84. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/tutorials/low_level.ipynb +0 -0
  85. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/tutorials/nested_spectra.ipynb +0 -0
  86. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/tutorials.rst +0 -0
  87. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/setup.cfg +0 -0
  88. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/__init__.py +0 -0
  89. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/datasets/__init__.py +0 -0
  90. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/datasets/generation.py +0 -0
  91. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/nestedframe/__init__.py +0 -0
  92. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/nestedframe/expr.py +0 -0
  93. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/py.typed +0 -0
  94. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/series/__init__.py +0 -0
  95. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/utils/__init__.py +0 -0
  96. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/utils/utils.py +0 -0
  97. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas.egg-info/dependency_links.txt +0 -0
  98. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas.egg-info/top_level.txt +0 -0
  99. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/conftest.py +0 -0
  100. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/datasets/test_generation.py +0 -0
  101. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/e2e_tests/test_issue89.py +0 -0
  102. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/nestedframe/test_io.py +0 -0
  103. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/test_packaging.py +0 -0
  104. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/utils/test_utils.py +0 -0
  105. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/test_data/nested.parquet +0 -0
  106. {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/test_data/not_nestable.parquet +0 -0
@@ -1,5 +1,5 @@
1
1
  # Changes here will be overwritten by Copier
2
- _commit: v2.0.6
2
+ _commit: v2.0.7
3
3
  _src_path: gh:lincc-frameworks/python-project-template
4
4
  author_email: brantd@uw.edu
5
5
  author_name: LINCC Frameworks
@@ -18,8 +18,8 @@ project_license: MIT
18
18
  project_name: nested-pandas
19
19
  project_organization: lincc-frameworks
20
20
  python_versions:
21
- - '3.9'
22
21
  - '3.10'
23
22
  - '3.11'
24
23
  - '3.12'
25
24
  - '3.13'
25
+ test_lowest_version: all
@@ -20,7 +20,7 @@ jobs:
20
20
  runs-on: ubuntu-latest
21
21
  strategy:
22
22
  matrix:
23
- python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
23
+ python-version: ['3.10', '3.11', '3.12', '3.13']
24
24
 
25
25
  steps:
26
26
  - uses: actions/checkout@v4
@@ -1,3 +1,4 @@
1
+
1
2
  # This workflow will install Python dependencies, run tests and report code coverage with a variety of Python versions
2
3
  # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
3
4
 
@@ -15,7 +16,7 @@ jobs:
15
16
  runs-on: ubuntu-latest
16
17
  strategy:
17
18
  matrix:
18
- python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
19
+ python-version: ['3.10', '3.11', '3.12', '3.13']
19
20
 
20
21
  steps:
21
22
  - uses: actions/checkout@v4
@@ -36,17 +37,14 @@ jobs:
36
37
  uses: codecov/codecov-action@v5
37
38
  with:
38
39
  token: ${{ secrets.CODECOV_TOKEN }}
39
-
40
40
  test-lowest-versions:
41
-
42
41
  runs-on: ubuntu-latest
43
-
44
42
  steps:
45
43
  - uses: actions/checkout@v4
46
- - name: Set up Python 3.9
44
+ - name: Set up Python 3.10
47
45
  uses: actions/setup-python@v5
48
46
  with:
49
- python-version: '3.9'
47
+ python-version: '3.10'
50
48
  - name: Install dependencies
51
49
  run: |
52
50
  sudo apt-get update
@@ -54,8 +52,8 @@ jobs:
54
52
  uv venv venv
55
53
  source venv/bin/activate
56
54
  uv pip compile --resolution=lowest -o requirements_lowest.txt pyproject.toml
57
- uv pip install --constraint=requirements_lowest.txt -e '.[dev]'
55
+ uv pip install --constraint=requirements_lowest.txt -e .[dev]
58
56
  - name: Run unit tests with pytest
59
57
  run: |
60
58
  source venv/bin/activate
61
- python -m pytest
59
+ python -m pytest
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nested-pandas
3
- Version: 0.4.2
3
+ Version: 0.4.4
4
4
  Summary: An extension of pandas for efficient representation of nested associated datasets.
5
5
  Author-email: LINCC Frameworks <brantd@uw.edu>
6
6
  License: MIT License
@@ -32,11 +32,11 @@ Classifier: Intended Audience :: Developers
32
32
  Classifier: Intended Audience :: Science/Research
33
33
  Classifier: Operating System :: OS Independent
34
34
  Classifier: Programming Language :: Python
35
- Requires-Python: >=3.9
35
+ Requires-Python: >=3.10
36
36
  Description-Content-Type: text/markdown
37
37
  License-File: LICENSE
38
38
  Requires-Dist: numpy>=2
39
- Requires-Dist: pandas<2.3,>=2.2.3
39
+ Requires-Dist: pandas<2.4,>=2.2.3
40
40
  Requires-Dist: pyarrow>=18
41
41
  Requires-Dist: universal_pathlib>=0.2
42
42
  Provides-Extra: dev
@@ -345,7 +345,7 @@
345
345
  ],
346
346
  "metadata": {
347
347
  "kernelspec": {
348
- "display_name": "nested",
348
+ "display_name": "Python 3 (ipykernel)",
349
349
  "language": "python",
350
350
  "name": "python3"
351
351
  },
@@ -359,9 +359,9 @@
359
359
  "name": "python",
360
360
  "nbconvert_exporter": "python",
361
361
  "pygments_lexer": "ipython3",
362
- "version": "3.10.4"
362
+ "version": "3.13.3"
363
363
  }
364
364
  },
365
365
  "nbformat": 4,
366
- "nbformat_minor": 2
366
+ "nbformat_minor": 4
367
367
  }
@@ -17,6 +17,7 @@ Functions
17
17
 
18
18
  NestSeriesAccessor.to_lists
19
19
  NestSeriesAccessor.to_flat
20
+ NestSeriesAccessor.to_flatten_inner
20
21
  NestSeriesAccessor.with_field
21
22
  NestSeriesAccessor.with_flat_field
22
23
  NestSeriesAccessor.with_list_field
@@ -26,4 +27,3 @@ Functions
26
27
  NestSeriesAccessor.get_flat_index
27
28
  NestSeriesAccessor.get_flat_series
28
29
  NestSeriesAccessor.get_list_series
29
-
@@ -15,6 +15,16 @@ Functions
15
15
  .. autosummary::
16
16
  :toctree: api/
17
17
 
18
+ series.ext_array.NestedExtensionArray.dtype
19
+ series.ext_array.NestedExtensionArray.nbytes
20
+ series.ext_array.NestedExtensionArray.list_array
21
+ series.ext_array.NestedExtensionArray.struct_array
22
+ series.ext_array.NestedExtensionArray.py_table
23
+ series.ext_array.NestedExtensionArray.list_offsets
24
+ series.ext_array.NestedExtensionArray.field_names
25
+ series.ext_array.NestedExtensionArray.list_lengths
26
+ series.ext_array.NestedExtensionArray.flat_length
27
+ series.ext_array.NestedExtensionArray.num_chunks
18
28
  series.ext_array.NestedExtensionArray.to_numpy
19
29
  series.ext_array.NestedExtensionArray.isna
20
30
  series.ext_array.NestedExtensionArray.take
@@ -24,6 +34,7 @@ Functions
24
34
  series.ext_array.NestedExtensionArray.from_sequence
25
35
  series.ext_array.NestedExtensionArray.to_arrow_ext_array
26
36
  series.ext_array.NestedExtensionArray.to_pyarrow_scalar
37
+ series.ext_array.NestedExtensionArray.get_list_index
27
38
  series.ext_array.NestedExtensionArray.iter_field_lists
28
39
  series.ext_array.NestedExtensionArray.view_fields
29
40
  series.ext_array.NestedExtensionArray.set_flat_field
@@ -16,11 +16,11 @@ classifiers = [
16
16
  "Programming Language :: Python",
17
17
  ]
18
18
  dynamic = ["version"]
19
- requires-python = ">=3.9"
19
+ requires-python = ">=3.10"
20
20
  dependencies = [
21
21
  "numpy>=2",
22
22
  # We use internal pd._libs.missing and experimental ArrowExtensionArray
23
- "pandas>=2.2.3,<2.3",
23
+ "pandas>=2.2.3,<2.4",
24
24
  "pyarrow>=18",
25
25
  "universal_pathlib>=0.2",
26
26
  ]
@@ -60,18 +60,9 @@ testpaths = [
60
60
  ]
61
61
  addopts = "--doctest-modules --doctest-glob=*.rst"
62
62
 
63
- [tool.black]
64
- line-length = 110
65
- target-version = ["py39"]
66
-
67
- [tool.isort]
68
- profile = "black"
69
- line_length = 110
70
-
71
63
  [tool.ruff]
72
64
  line-length = 110
73
- target-version = "py39"
74
-
65
+ target-version = "py310"
75
66
  [tool.ruff.lint]
76
67
  select = [
77
68
  # pycodestyle
@@ -103,7 +94,6 @@ select = [
103
94
  # Numpy v2.0 compatibility
104
95
  "NPY201",
105
96
  ]
106
-
107
97
  ignore = [
108
98
  "UP006", # Allow non standard library generics in type hints
109
99
  "UP007", # Allow Union in type hints
@@ -113,6 +103,7 @@ ignore = [
113
103
  "UP015", # Allow redundant open parameters
114
104
  "UP028", # Allow yield in for loop
115
105
  ]
106
+
116
107
  [tool.setuptools.package-data]
117
108
  nested_pandas = ["py.typed"]
118
109
 
@@ -17,5 +17,5 @@ __version__: str
17
17
  __version_tuple__: VERSION_TUPLE
18
18
  version_tuple: VERSION_TUPLE
19
19
 
20
- __version__ = version = '0.4.2'
21
- __version_tuple__ = version_tuple = (0, 4, 2)
20
+ __version__ = version = '0.4.4'
21
+ __version_tuple__ = version_tuple = (0, 4, 4)
@@ -1,6 +1,8 @@
1
1
  # typing.Self and "|" union syntax don't exist in Python 3.9
2
2
  from __future__ import annotations
3
3
 
4
+ from collections import defaultdict
5
+
4
6
  import numpy as np
5
7
  import pandas as pd
6
8
  import pyarrow as pa
@@ -9,6 +11,7 @@ from pandas._libs import lib
9
11
  from pandas._typing import Any, AnyAll, Axis, IndexLabel
10
12
  from pandas.api.extensions import no_default
11
13
  from pandas.core.computation.eval import Expr, ensure_scope
14
+ from pandas.core.dtypes.inference import is_list_like
12
15
 
13
16
  from nested_pandas.nestedframe.expr import (
14
17
  _identify_aliases,
@@ -18,6 +21,7 @@ from nested_pandas.nestedframe.expr import (
18
21
  )
19
22
  from nested_pandas.series.dtype import NestedDtype
20
23
  from nested_pandas.series.packer import pack, pack_lists, pack_sorted_df_into_struct
24
+ from nested_pandas.series.utils import is_pa_type_a_list
21
25
 
22
26
  pd.set_option("display.max_rows", 30)
23
27
  pd.set_option("display.min_rows", 5)
@@ -41,6 +45,32 @@ class NestedFrame(pd.DataFrame):
41
45
  # contains those aliases, keyed by the cleaned name.
42
46
  _metadata = ["_aliases"]
43
47
 
48
+ def __init__(self, *args, **kwargs) -> None:
49
+ super().__init__(*args, **kwargs)
50
+ self._cast_cols_to_nested(struct_list=False)
51
+
52
+ def _cast_cols_to_nested(self, *, struct_list: bool) -> None:
53
+ """Cast arrow columns to nested.
54
+
55
+ Parameters
56
+ ----------
57
+ struct_list : bool
58
+ If `False` cast list-struct columns only. If `True`, also
59
+ try to cast struct-list columns validating if they have
60
+ valid nested structure.
61
+ """
62
+ for column, dtype in self.dtypes.items():
63
+ if not isinstance(dtype, pd.ArrowDtype):
64
+ continue
65
+ pa_type = dtype.pyarrow_dtype
66
+ if not is_pa_type_a_list(pa_type) and not (struct_list and pa.types.is_struct(pa_type)):
67
+ continue
68
+ try:
69
+ nested_dtype = NestedDtype(pa_type)
70
+ except (TypeError, ValueError):
71
+ continue
72
+ self[column] = self[column].astype(nested_dtype)
73
+
44
74
  @property
45
75
  def _constructor(self) -> Self: # type: ignore[name-defined] # noqa: F821
46
76
  return NestedFrame
@@ -101,9 +131,16 @@ class NestedFrame(pd.DataFrame):
101
131
  return None
102
132
  return chunk.to_html(max_rows=1, max_cols=5, show_dimensions=True, index=False, header=False)
103
133
 
134
+ # Handle sizing, trim html dataframe if output will be truncated
135
+ df_shape = self.shape # grab original shape information for later
136
+ if pd.get_option("display.max_rows") is None or df_shape[0] > pd.get_option("display.max_rows"):
137
+ html_df = self.head(pd.get_option("display.min_rows") + 1)
138
+ else:
139
+ html_df = self.copy()
140
+
104
141
  # replace index to ensure proper behavior for duplicate index values
105
- index_values = self.index
106
- html_df = self.reset_index(drop=True)
142
+ index_values = html_df.index
143
+ html_df = html_df.reset_index(drop=True)
107
144
 
108
145
  # Apply repacking to all nested columns
109
146
  repr = html_df.style.format(
@@ -122,15 +159,17 @@ class NestedFrame(pd.DataFrame):
122
159
 
123
160
  # Recover some truncation formatting, limited to head truncation
124
161
  if pd.get_option("display.max_rows") is None:
162
+ # Just display header
125
163
  return repr.to_html(max_rows=0)
126
- elif repr.data.shape[0] > pd.get_option("display.max_rows"):
164
+ elif df_shape[0] > pd.get_option("display.max_rows"):
165
+ # when over the max_rows threshold, display with truncation ("..." row at the end)
127
166
  html_repr = repr.to_html(max_rows=pd.get_option("display.min_rows"))
128
167
  else:
129
168
  # when under the max_rows threshold, display all rows (behavior of 0 here)
130
169
  html_repr = repr.to_html(max_rows=0)
131
170
 
132
171
  # Manually append dimensionality to a styler output
133
- html_repr += f"{repr.data.shape[0]} rows x {repr.data.shape[1]} columns"
172
+ html_repr += f"{df_shape[0]} rows x {df_shape[1]} columns"
134
173
 
135
174
  return html_repr
136
175
 
@@ -166,10 +205,14 @@ class NestedFrame(pd.DataFrame):
166
205
 
167
206
  def __getitem__(self, item):
168
207
  """Adds custom __getitem__ behavior for nested columns"""
208
+ if isinstance(item, str):
209
+ return self._getitem_str(item)
210
+ elif self._is_key_list(item):
211
+ return self._getitem_list(item)
169
212
 
170
- if not isinstance(item, str):
171
- return super().__getitem__(item)
213
+ return super().__getitem__(item)
172
214
 
215
+ def _getitem_str(self, item):
173
216
  # Preempt the nested check if the item is a base column, with or without
174
217
  # dots and backticks.
175
218
  if item in self.columns:
@@ -191,6 +234,29 @@ class NestedFrame(pd.DataFrame):
191
234
  else:
192
235
  raise KeyError(f"Column '{cleaned_item}' not found in nested columns or base columns")
193
236
 
237
+ def _is_key_list(self, item):
238
+ if not is_list_like(item):
239
+ return False
240
+ for k in item:
241
+ if not isinstance(k, str):
242
+ return False
243
+ if not self._is_known_column(k):
244
+ return False
245
+ return True
246
+
247
+ def _getitem_list(self, item):
248
+ non_nested_keys = [k for k in item if k in self.columns]
249
+ result = super().__getitem__(non_nested_keys)
250
+ components = [self._parse_hierarchical_components(k) for k in item]
251
+ nested_components = [c for c in components if self._is_known_hierarchical_column(c)]
252
+ nested_columns = defaultdict(set)
253
+ for comps in nested_components:
254
+ nested_columns[comps[0]].add(".".join(comps[1:]))
255
+ for c in nested_columns:
256
+ unselected_columns = [f for f in self.dtypes[c].field_names if f not in nested_columns[c]]
257
+ result[c] = self[c].nest.without_field(unselected_columns)
258
+ return result
259
+
194
260
  def __setitem__(self, key, value):
195
261
  """Adds custom __setitem__ behavior for nested columns"""
196
262
  components = self._parse_hierarchical_components(key)
@@ -224,7 +290,12 @@ class NestedFrame(pd.DataFrame):
224
290
  self._update_inplace(new_df)
225
291
  return None
226
292
 
227
- return super().__setitem__(key, value)
293
+ super().__setitem__(key, value)
294
+ self._cast_cols_to_nested(struct_list=False)
295
+
296
+ def __delitem__(self, key):
297
+ """Delete a column or a nested field using dot notation (e.g., del nf['nested.x'])"""
298
+ self.drop([key], axis=1, inplace=True)
228
299
 
229
300
  def add_nested(
230
301
  self,
@@ -467,7 +538,7 @@ class NestedFrame(pd.DataFrame):
467
538
  # This is a simple heuristic but infers more than its dtype
468
539
  # which will probably be an object.
469
540
  sample_val = df[col].iloc[0]
470
- if not hasattr(sample_val, "__iter__") and not isinstance(sample_val, (str, bytes)):
541
+ if not hasattr(sample_val, "__iter__") and not isinstance(sample_val, str | bytes):
471
542
  raise ValueError(
472
543
  f"Cannot pack column {col} which does not contain an iterable list based "
473
544
  "on its first value, {sample_val}."
@@ -554,7 +625,10 @@ class NestedFrame(pd.DataFrame):
554
625
  # drop targeted sub-columns for each nested column
555
626
  for col in nested_cols:
556
627
  sub_cols = [label.split(".")[1] for label in nested_labels if label.split(".")[0] == col]
557
- self = self.assign(**{f"{col}": self[col].nest.without_field(sub_cols)})
628
+ if inplace:
629
+ self[col] = self[col].nest.without_field(sub_cols)
630
+ else:
631
+ self = self.assign(**{f"{col}": self[col].nest.without_field(sub_cols)})
558
632
 
559
633
  # drop remaining base columns
560
634
  if len(base_labels) > 0:
@@ -568,7 +642,7 @@ class NestedFrame(pd.DataFrame):
568
642
  errors=errors,
569
643
  )
570
644
  else:
571
- return self
645
+ return self if not inplace else None
572
646
  # Otherwise just drop like pandas
573
647
  return super().drop(
574
648
  labels=labels,
@@ -1216,7 +1290,7 @@ class NestedFrame(pd.DataFrame):
1216
1290
  else:
1217
1291
  iterators.append(self[layer].array.iter_field_lists(col))
1218
1292
 
1219
- results = [func(*cols, *extra_args, **kwargs) for cols in zip(*iterators)]
1293
+ results = [func(*cols, *extra_args, **kwargs) for cols in zip(*iterators, strict=True)]
1220
1294
  results_nf = NestedFrame(results, index=self.index)
1221
1295
 
1222
1296
  if infer_nesting:
@@ -1242,6 +1316,28 @@ class NestedFrame(pd.DataFrame):
1242
1316
 
1243
1317
  return results_nf
1244
1318
 
1319
+ def to_pandas(self, list_struct=False) -> pd.DataFrame:
1320
+ """Convert to an ordinal pandas DataFrame, with no NestedDtype series.
1321
+
1322
+ NestedDtype is cast to pd.ArrowDtype
1323
+
1324
+ Parameters
1325
+ ----------
1326
+ list_struct: bool
1327
+ If True, cast nested columns to pandas struct-list arrow extension
1328
+ array columns. If False (default), cast nested columns to
1329
+ list-struct array columns.
1330
+
1331
+ Returns
1332
+ -------
1333
+ pd.DataFrame
1334
+ Ordinal pandas DataFrame.
1335
+ """
1336
+ df = pd.DataFrame(self)
1337
+ for col in self.nested_columns:
1338
+ df[col] = df[col].array.to_arrow_ext_array(list_struct=list_struct)
1339
+ return df
1340
+
1245
1341
  def to_parquet(self, path, **kwargs) -> None:
1246
1342
  """Creates parquet file(s) with the data of a NestedFrame, either
1247
1343
  as a single parquet file where each nested dataset is packed into its
@@ -1269,13 +1365,14 @@ class NestedFrame(pd.DataFrame):
1269
1365
  >>> nf = generate_data(5,5, seed=1)
1270
1366
  >>> nf.to_parquet("nestedframe.parquet")
1271
1367
  """
1368
+ df = self.to_pandas(list_struct=False)
1272
1369
 
1273
1370
  # Write through pyarrow
1274
1371
  # This is potentially not zero-copy
1275
1372
  # Note: Without pandas metadata, index writing is not as robust set
1276
1373
  # preserve_index=None for best behavior but index will generally
1277
1374
  # need to be set manually on load
1278
- table = pa.Table.from_pandas(self, preserve_index=None)
1375
+ table = pa.Table.from_pandas(df, preserve_index=None)
1279
1376
 
1280
1377
  # Drop pandas metadata to make sure nesteddtypes are not preserved
1281
1378
  # Do this by rebuilding the schema
@@ -87,7 +87,7 @@ def read_parquet(
87
87
  # First load through pyarrow
88
88
  # Check if `data` is a file-like object or a sequence
89
89
  if hasattr(data, "read") or (
90
- isinstance(data, Sequence) and not isinstance(data, (str, bytes, bytearray))
90
+ isinstance(data, Sequence) and not isinstance(data, str | bytes | bytearray)
91
91
  ):
92
92
  # If `data` is a file-like object or a sequence, pass it directly to pyarrow
93
93
  table = pq.read_table(data, columns=columns, **kwargs)
@@ -103,7 +103,7 @@ def read_parquet(
103
103
  # was from a nested column.
104
104
  if columns is not None:
105
105
  nested_structures: dict[str, list[int]] = {}
106
- for i, (col_in, col_pa) in enumerate(zip(columns, table.column_names)):
106
+ for i, (col_in, col_pa) in enumerate(zip(columns, table.column_names, strict=True)):
107
107
  # if the column name is not the same, it was a partial load
108
108
  if col_in != col_pa:
109
109
  # get the top-level column name
@@ -0,0 +1,3 @@
1
+ from .list_struct_storage import ListStructStorage # noqa: F401
2
+ from .struct_list_storage import StructListStorage # noqa: F401
3
+ from .table_storage import TableStorage # noqa: F401
@@ -0,0 +1,84 @@
1
+ from __future__ import annotations # Python 3.9 requires it for X | Y type hints
2
+
3
+ from typing import TYPE_CHECKING, Any
4
+
5
+ import pyarrow as pa
6
+
7
+ from nested_pandas.series.utils import transpose_struct_list_chunked, validate_list_struct_type
8
+
9
+ if TYPE_CHECKING:
10
+ from nested_pandas.series._storage.struct_list_storage import StructListStorage
11
+ from nested_pandas.series._storage.table_storage import TableStorage
12
+
13
+
14
+ class ListStructStorage:
15
+ """Store nested data as a PyArrow list-struct array.
16
+
17
+ Parameters
18
+ ----------
19
+ array : pa.ListArray or pa.ChunkedArray
20
+ Pyarrow list-array with a struct value type. An array or a chunk-array
21
+ """
22
+
23
+ _data: pa.ChunkedArray
24
+
25
+ def __init__(self, array: pa.ListArray | pa.ChunkedArray) -> None:
26
+ if isinstance(array, pa.ListArray):
27
+ array = pa.chunked_array([array])
28
+ if not isinstance(array, pa.ChunkedArray):
29
+ raise ValueError("array must be of type pa.ChunkedArray")
30
+ validate_list_struct_type(array.type)
31
+ self._data = array
32
+
33
+ @property
34
+ def data(self) -> pa.ChunkedArray:
35
+ return self._data
36
+
37
+ @classmethod
38
+ def from_struct_list_storage(cls, struct_list_storage: StructListStorage) -> Self: # type: ignore # noqa: F821
39
+ """Construct from a StructListStorage object.
40
+
41
+ Parameters
42
+ ----------
43
+ struct_list_storage : StructListStorage
44
+ StructListStorage object.
45
+ """
46
+ data = transpose_struct_list_chunked(struct_list_storage.data, validate=False)
47
+ return cls(data)
48
+
49
+ @classmethod
50
+ def from_table_storage(cls, table_storage: TableStorage) -> Self: # type: ignore # noqa: F821
51
+ """Construct from a TableStorage object.
52
+
53
+ Parameters
54
+ ----------
55
+ table_storage : TableStorage
56
+ TableStorage object.
57
+ """
58
+ from nested_pandas.series._storage import StructListStorage
59
+
60
+ struct_list_storage = StructListStorage.from_table_storage(table_storage)
61
+ return cls.from_struct_list_storage(struct_list_storage)
62
+
63
+ def __len__(self) -> int:
64
+ return len(self._data)
65
+
66
+ def __eq__(self, other: Any) -> bool:
67
+ if not isinstance(other, type(self)):
68
+ return False
69
+ return self._data == other._data
70
+
71
+ @property
72
+ def nbytes(self) -> int:
73
+ """Number of bytes consumed by the data in memory."""
74
+ return self._data.nbytes
75
+
76
+ @property
77
+ def type(self) -> pa.ListType:
78
+ """Pyarrow type of the underlying array."""
79
+ return self._data.type
80
+
81
+ @property
82
+ def num_chunks(self) -> int:
83
+ """Number of chunks in the underlying array."""
84
+ return self._data.num_chunks
@@ -0,0 +1,74 @@
1
+ from __future__ import annotations # Python 3.9 requires it for X | Y type hints
2
+
3
+ from collections.abc import Iterator
4
+ from typing import TYPE_CHECKING
5
+
6
+ import pyarrow as pa
7
+
8
+ from nested_pandas.series.utils import (
9
+ table_to_struct_array,
10
+ transpose_list_struct_chunked,
11
+ validate_struct_list_array_for_equal_lengths,
12
+ )
13
+
14
+ if TYPE_CHECKING:
15
+ from nested_pandas.series._storage.list_struct_storage import ListStructStorage
16
+ from nested_pandas.series._storage.table_storage import TableStorage
17
+
18
+
19
+ class StructListStorage:
20
+ """Store nested data as a PyArrow struct-list array.
21
+
22
+ Parameters
23
+ ----------
24
+ array : pa.StructArray or pa.ChunkedArray
25
+ Pyarrow struct-array with all fields to be list-arrays.
26
+ All list-values must be "aligned", e.g., have the same length.
27
+ validate : bool (default True)
28
+ Check that all the lists have the same lengths for each struct-value.
29
+ """
30
+
31
+ _data: pa.ChunkedArray
32
+
33
+ def __init__(self, array: pa.StructArray | pa.ChunkedArray, *, validate: bool = True) -> None:
34
+ if isinstance(array, pa.StructArray):
35
+ array = pa.chunked_array([array])
36
+ if not isinstance(array, pa.ChunkedArray):
37
+ raise ValueError("array must be a StructArray or ChunkedArray")
38
+
39
+ if validate:
40
+ for chunk in array.chunks:
41
+ validate_struct_list_array_for_equal_lengths(chunk)
42
+
43
+ self._data = array
44
+
45
+ @property
46
+ def data(self) -> pa.ChunkedArray:
47
+ return self._data
48
+
49
+ @classmethod
50
+ def from_list_struct_storage(cls, list_struct_storage: ListStructStorage) -> Self: # type: ignore # noqa: F821
51
+ """Construct from a ListStructStorage object.
52
+
53
+ Parameters
54
+ ----------
55
+ list_struct_storage : ListStructStorage
56
+ ListStructStorage object.
57
+ """
58
+ data = transpose_list_struct_chunked(list_struct_storage.data)
59
+ return cls(data, validate=False)
60
+
61
+ @classmethod
62
+ def from_table_storage(cls, table_storage: TableStorage) -> Self: # type: ignore # noqa: F821
63
+ """Construct from a TableStorage object.
64
+
65
+ Parameters
66
+ ----------
67
+ table_storage : TableStorage
68
+ TableStorage object.
69
+ """
70
+ data = table_to_struct_array(table_storage.data)
71
+ return cls(data, validate=False)
72
+
73
+ def __iter__(self) -> Iterator[pa.StructScalar]:
74
+ return iter(self._data)