nested-pandas 0.4.2__tar.gz → 0.4.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.copier-answers.yml +2 -2
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/smoke-test.yml +1 -1
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/testing-and-coverage.yml +6 -8
- {nested_pandas-0.4.2/src/nested_pandas.egg-info → nested_pandas-0.4.4}/PKG-INFO +3 -3
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/gettingstarted/quickstart.ipynb +3 -3
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/reference/accessor.rst +1 -1
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/reference/ext_array.rst +11 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/pyproject.toml +4 -13
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/_version.py +2 -2
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/nestedframe/core.py +109 -12
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/nestedframe/io.py +2 -2
- nested_pandas-0.4.4/src/nested_pandas/series/_storage/__init__.py +3 -0
- nested_pandas-0.4.4/src/nested_pandas/series/_storage/list_struct_storage.py +84 -0
- nested_pandas-0.4.4/src/nested_pandas/series/_storage/struct_list_storage.py +74 -0
- nested_pandas-0.4.4/src/nested_pandas/series/_storage/table_storage.py +66 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/series/accessor.py +136 -33
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/series/dtype.py +34 -3
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/series/ext_array.py +214 -153
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/series/packer.py +2 -2
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/series/utils.py +142 -30
- {nested_pandas-0.4.2 → nested_pandas-0.4.4/src/nested_pandas.egg-info}/PKG-INFO +3 -3
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas.egg-info/SOURCES.txt +6 -1
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas.egg-info/requires.txt +1 -1
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/nestedframe/test_nestedframe.py +131 -5
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/series/test_accessor.py +98 -2
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/series/test_dtype.py +16 -1
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/series/test_ext_array.py +40 -45
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/series/test_packer.py +13 -1
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/series/test_series_utils.py +31 -0
- nested_pandas-0.4.4/tests/test_data/vsx-x-ztfdr22_lc-m31.parquet +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.git_archival.txt +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.gitattributes +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/ISSUE_TEMPLATE/0-general_issue.md +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/ISSUE_TEMPLATE/1-bug_report.md +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/ISSUE_TEMPLATE/2-feature_request.md +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/ISSUE_TEMPLATE/README.md +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/dependabot.yml +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/pull_request_template.md +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/README.md +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/asv-main.yml +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/asv-nightly.yml +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/asv-pr.yml +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/build-documentation.yml +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/pre-commit-ci.yml +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/publish-benchmarks-pr.yml +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.github/workflows/publish-to-pypi.yml +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.gitignore +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.pre-commit-config.yaml +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.readthedocs.yml +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/.setup_dev.sh +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/LICENSE +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/README.md +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/benchmarks/README.md +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/benchmarks/__init__.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/benchmarks/asv.conf.json +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/benchmarks/benchmarks.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/Makefile +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/_static/custom.css +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/_templates/autosummary/base.rst +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/_templates/autosummary/class.rst +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/_templates/autosummary/module.rst +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/about/internals.rst +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/about/npd_internals.png +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/about.rst +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/conf.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/gettingstarted/contributing.rst +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/gettingstarted/installation.rst +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/gettingstarted.rst +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/index.rst +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/intro_images/loc_into_nested.png +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/intro_images/nestedframe_example.png +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/intro_images/pandas_dfs.png +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/intro_images/reduce.png +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/pre_executed/performance.ipynb +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/reference/nesteddtype.rst +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/reference/nestedframe.rst +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/reference/packer.rst +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/reference/utils.rst +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/reference.rst +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/requirements.txt +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/tutorials/README.md +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/tutorials/data_loading_notebook.ipynb +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/tutorials/data_manipulation.ipynb +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/tutorials/low_level.ipynb +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/tutorials/nested_spectra.ipynb +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/docs/tutorials.rst +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/setup.cfg +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/__init__.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/datasets/__init__.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/datasets/generation.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/nestedframe/__init__.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/nestedframe/expr.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/py.typed +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/series/__init__.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/utils/__init__.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas/utils/utils.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas.egg-info/dependency_links.txt +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/src/nested_pandas.egg-info/top_level.txt +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/conftest.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/datasets/test_generation.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/e2e_tests/test_issue89.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/nestedframe/test_io.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/test_packaging.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/nested_pandas/utils/test_utils.py +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/test_data/nested.parquet +0 -0
- {nested_pandas-0.4.2 → nested_pandas-0.4.4}/tests/test_data/not_nestable.parquet +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# Changes here will be overwritten by Copier
|
|
2
|
-
_commit: v2.0.
|
|
2
|
+
_commit: v2.0.7
|
|
3
3
|
_src_path: gh:lincc-frameworks/python-project-template
|
|
4
4
|
author_email: brantd@uw.edu
|
|
5
5
|
author_name: LINCC Frameworks
|
|
@@ -18,8 +18,8 @@ project_license: MIT
|
|
|
18
18
|
project_name: nested-pandas
|
|
19
19
|
project_organization: lincc-frameworks
|
|
20
20
|
python_versions:
|
|
21
|
-
- '3.9'
|
|
22
21
|
- '3.10'
|
|
23
22
|
- '3.11'
|
|
24
23
|
- '3.12'
|
|
25
24
|
- '3.13'
|
|
25
|
+
test_lowest_version: all
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
|
|
1
2
|
# This workflow will install Python dependencies, run tests and report code coverage with a variety of Python versions
|
|
2
3
|
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
|
|
3
4
|
|
|
@@ -15,7 +16,7 @@ jobs:
|
|
|
15
16
|
runs-on: ubuntu-latest
|
|
16
17
|
strategy:
|
|
17
18
|
matrix:
|
|
18
|
-
python-version: ['3.
|
|
19
|
+
python-version: ['3.10', '3.11', '3.12', '3.13']
|
|
19
20
|
|
|
20
21
|
steps:
|
|
21
22
|
- uses: actions/checkout@v4
|
|
@@ -36,17 +37,14 @@ jobs:
|
|
|
36
37
|
uses: codecov/codecov-action@v5
|
|
37
38
|
with:
|
|
38
39
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
39
|
-
|
|
40
40
|
test-lowest-versions:
|
|
41
|
-
|
|
42
41
|
runs-on: ubuntu-latest
|
|
43
|
-
|
|
44
42
|
steps:
|
|
45
43
|
- uses: actions/checkout@v4
|
|
46
|
-
- name: Set up Python 3.
|
|
44
|
+
- name: Set up Python 3.10
|
|
47
45
|
uses: actions/setup-python@v5
|
|
48
46
|
with:
|
|
49
|
-
python-version: '3.
|
|
47
|
+
python-version: '3.10'
|
|
50
48
|
- name: Install dependencies
|
|
51
49
|
run: |
|
|
52
50
|
sudo apt-get update
|
|
@@ -54,8 +52,8 @@ jobs:
|
|
|
54
52
|
uv venv venv
|
|
55
53
|
source venv/bin/activate
|
|
56
54
|
uv pip compile --resolution=lowest -o requirements_lowest.txt pyproject.toml
|
|
57
|
-
uv pip install --constraint=requirements_lowest.txt -e
|
|
55
|
+
uv pip install --constraint=requirements_lowest.txt -e .[dev]
|
|
58
56
|
- name: Run unit tests with pytest
|
|
59
57
|
run: |
|
|
60
58
|
source venv/bin/activate
|
|
61
|
-
python -m pytest
|
|
59
|
+
python -m pytest
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nested-pandas
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.4
|
|
4
4
|
Summary: An extension of pandas for efficient representation of nested associated datasets.
|
|
5
5
|
Author-email: LINCC Frameworks <brantd@uw.edu>
|
|
6
6
|
License: MIT License
|
|
@@ -32,11 +32,11 @@ Classifier: Intended Audience :: Developers
|
|
|
32
32
|
Classifier: Intended Audience :: Science/Research
|
|
33
33
|
Classifier: Operating System :: OS Independent
|
|
34
34
|
Classifier: Programming Language :: Python
|
|
35
|
-
Requires-Python: >=3.
|
|
35
|
+
Requires-Python: >=3.10
|
|
36
36
|
Description-Content-Type: text/markdown
|
|
37
37
|
License-File: LICENSE
|
|
38
38
|
Requires-Dist: numpy>=2
|
|
39
|
-
Requires-Dist: pandas<2.
|
|
39
|
+
Requires-Dist: pandas<2.4,>=2.2.3
|
|
40
40
|
Requires-Dist: pyarrow>=18
|
|
41
41
|
Requires-Dist: universal_pathlib>=0.2
|
|
42
42
|
Provides-Extra: dev
|
|
@@ -345,7 +345,7 @@
|
|
|
345
345
|
],
|
|
346
346
|
"metadata": {
|
|
347
347
|
"kernelspec": {
|
|
348
|
-
"display_name": "
|
|
348
|
+
"display_name": "Python 3 (ipykernel)",
|
|
349
349
|
"language": "python",
|
|
350
350
|
"name": "python3"
|
|
351
351
|
},
|
|
@@ -359,9 +359,9 @@
|
|
|
359
359
|
"name": "python",
|
|
360
360
|
"nbconvert_exporter": "python",
|
|
361
361
|
"pygments_lexer": "ipython3",
|
|
362
|
-
"version": "3.
|
|
362
|
+
"version": "3.13.3"
|
|
363
363
|
}
|
|
364
364
|
},
|
|
365
365
|
"nbformat": 4,
|
|
366
|
-
"nbformat_minor":
|
|
366
|
+
"nbformat_minor": 4
|
|
367
367
|
}
|
|
@@ -17,6 +17,7 @@ Functions
|
|
|
17
17
|
|
|
18
18
|
NestSeriesAccessor.to_lists
|
|
19
19
|
NestSeriesAccessor.to_flat
|
|
20
|
+
NestSeriesAccessor.to_flatten_inner
|
|
20
21
|
NestSeriesAccessor.with_field
|
|
21
22
|
NestSeriesAccessor.with_flat_field
|
|
22
23
|
NestSeriesAccessor.with_list_field
|
|
@@ -26,4 +27,3 @@ Functions
|
|
|
26
27
|
NestSeriesAccessor.get_flat_index
|
|
27
28
|
NestSeriesAccessor.get_flat_series
|
|
28
29
|
NestSeriesAccessor.get_list_series
|
|
29
|
-
|
|
@@ -15,6 +15,16 @@ Functions
|
|
|
15
15
|
.. autosummary::
|
|
16
16
|
:toctree: api/
|
|
17
17
|
|
|
18
|
+
series.ext_array.NestedExtensionArray.dtype
|
|
19
|
+
series.ext_array.NestedExtensionArray.nbytes
|
|
20
|
+
series.ext_array.NestedExtensionArray.list_array
|
|
21
|
+
series.ext_array.NestedExtensionArray.struct_array
|
|
22
|
+
series.ext_array.NestedExtensionArray.py_table
|
|
23
|
+
series.ext_array.NestedExtensionArray.list_offsets
|
|
24
|
+
series.ext_array.NestedExtensionArray.field_names
|
|
25
|
+
series.ext_array.NestedExtensionArray.list_lengths
|
|
26
|
+
series.ext_array.NestedExtensionArray.flat_length
|
|
27
|
+
series.ext_array.NestedExtensionArray.num_chunks
|
|
18
28
|
series.ext_array.NestedExtensionArray.to_numpy
|
|
19
29
|
series.ext_array.NestedExtensionArray.isna
|
|
20
30
|
series.ext_array.NestedExtensionArray.take
|
|
@@ -24,6 +34,7 @@ Functions
|
|
|
24
34
|
series.ext_array.NestedExtensionArray.from_sequence
|
|
25
35
|
series.ext_array.NestedExtensionArray.to_arrow_ext_array
|
|
26
36
|
series.ext_array.NestedExtensionArray.to_pyarrow_scalar
|
|
37
|
+
series.ext_array.NestedExtensionArray.get_list_index
|
|
27
38
|
series.ext_array.NestedExtensionArray.iter_field_lists
|
|
28
39
|
series.ext_array.NestedExtensionArray.view_fields
|
|
29
40
|
series.ext_array.NestedExtensionArray.set_flat_field
|
|
@@ -16,11 +16,11 @@ classifiers = [
|
|
|
16
16
|
"Programming Language :: Python",
|
|
17
17
|
]
|
|
18
18
|
dynamic = ["version"]
|
|
19
|
-
requires-python = ">=3.
|
|
19
|
+
requires-python = ">=3.10"
|
|
20
20
|
dependencies = [
|
|
21
21
|
"numpy>=2",
|
|
22
22
|
# We use internal pd._libs.missing and experimental ArrowExtensionArray
|
|
23
|
-
"pandas>=2.2.3,<2.
|
|
23
|
+
"pandas>=2.2.3,<2.4",
|
|
24
24
|
"pyarrow>=18",
|
|
25
25
|
"universal_pathlib>=0.2",
|
|
26
26
|
]
|
|
@@ -60,18 +60,9 @@ testpaths = [
|
|
|
60
60
|
]
|
|
61
61
|
addopts = "--doctest-modules --doctest-glob=*.rst"
|
|
62
62
|
|
|
63
|
-
[tool.black]
|
|
64
|
-
line-length = 110
|
|
65
|
-
target-version = ["py39"]
|
|
66
|
-
|
|
67
|
-
[tool.isort]
|
|
68
|
-
profile = "black"
|
|
69
|
-
line_length = 110
|
|
70
|
-
|
|
71
63
|
[tool.ruff]
|
|
72
64
|
line-length = 110
|
|
73
|
-
target-version = "
|
|
74
|
-
|
|
65
|
+
target-version = "py310"
|
|
75
66
|
[tool.ruff.lint]
|
|
76
67
|
select = [
|
|
77
68
|
# pycodestyle
|
|
@@ -103,7 +94,6 @@ select = [
|
|
|
103
94
|
# Numpy v2.0 compatibility
|
|
104
95
|
"NPY201",
|
|
105
96
|
]
|
|
106
|
-
|
|
107
97
|
ignore = [
|
|
108
98
|
"UP006", # Allow non standard library generics in type hints
|
|
109
99
|
"UP007", # Allow Union in type hints
|
|
@@ -113,6 +103,7 @@ ignore = [
|
|
|
113
103
|
"UP015", # Allow redundant open parameters
|
|
114
104
|
"UP028", # Allow yield in for loop
|
|
115
105
|
]
|
|
106
|
+
|
|
116
107
|
[tool.setuptools.package-data]
|
|
117
108
|
nested_pandas = ["py.typed"]
|
|
118
109
|
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# typing.Self and "|" union syntax don't exist in Python 3.9
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
|
+
from collections import defaultdict
|
|
5
|
+
|
|
4
6
|
import numpy as np
|
|
5
7
|
import pandas as pd
|
|
6
8
|
import pyarrow as pa
|
|
@@ -9,6 +11,7 @@ from pandas._libs import lib
|
|
|
9
11
|
from pandas._typing import Any, AnyAll, Axis, IndexLabel
|
|
10
12
|
from pandas.api.extensions import no_default
|
|
11
13
|
from pandas.core.computation.eval import Expr, ensure_scope
|
|
14
|
+
from pandas.core.dtypes.inference import is_list_like
|
|
12
15
|
|
|
13
16
|
from nested_pandas.nestedframe.expr import (
|
|
14
17
|
_identify_aliases,
|
|
@@ -18,6 +21,7 @@ from nested_pandas.nestedframe.expr import (
|
|
|
18
21
|
)
|
|
19
22
|
from nested_pandas.series.dtype import NestedDtype
|
|
20
23
|
from nested_pandas.series.packer import pack, pack_lists, pack_sorted_df_into_struct
|
|
24
|
+
from nested_pandas.series.utils import is_pa_type_a_list
|
|
21
25
|
|
|
22
26
|
pd.set_option("display.max_rows", 30)
|
|
23
27
|
pd.set_option("display.min_rows", 5)
|
|
@@ -41,6 +45,32 @@ class NestedFrame(pd.DataFrame):
|
|
|
41
45
|
# contains those aliases, keyed by the cleaned name.
|
|
42
46
|
_metadata = ["_aliases"]
|
|
43
47
|
|
|
48
|
+
def __init__(self, *args, **kwargs) -> None:
|
|
49
|
+
super().__init__(*args, **kwargs)
|
|
50
|
+
self._cast_cols_to_nested(struct_list=False)
|
|
51
|
+
|
|
52
|
+
def _cast_cols_to_nested(self, *, struct_list: bool) -> None:
|
|
53
|
+
"""Cast arrow columns to nested.
|
|
54
|
+
|
|
55
|
+
Parameters
|
|
56
|
+
----------
|
|
57
|
+
struct_list : bool
|
|
58
|
+
If `False` cast list-struct columns only. If `True`, also
|
|
59
|
+
try to cast struct-list columns validating if they have
|
|
60
|
+
valid nested structure.
|
|
61
|
+
"""
|
|
62
|
+
for column, dtype in self.dtypes.items():
|
|
63
|
+
if not isinstance(dtype, pd.ArrowDtype):
|
|
64
|
+
continue
|
|
65
|
+
pa_type = dtype.pyarrow_dtype
|
|
66
|
+
if not is_pa_type_a_list(pa_type) and not (struct_list and pa.types.is_struct(pa_type)):
|
|
67
|
+
continue
|
|
68
|
+
try:
|
|
69
|
+
nested_dtype = NestedDtype(pa_type)
|
|
70
|
+
except (TypeError, ValueError):
|
|
71
|
+
continue
|
|
72
|
+
self[column] = self[column].astype(nested_dtype)
|
|
73
|
+
|
|
44
74
|
@property
|
|
45
75
|
def _constructor(self) -> Self: # type: ignore[name-defined] # noqa: F821
|
|
46
76
|
return NestedFrame
|
|
@@ -101,9 +131,16 @@ class NestedFrame(pd.DataFrame):
|
|
|
101
131
|
return None
|
|
102
132
|
return chunk.to_html(max_rows=1, max_cols=5, show_dimensions=True, index=False, header=False)
|
|
103
133
|
|
|
134
|
+
# Handle sizing, trim html dataframe if output will be truncated
|
|
135
|
+
df_shape = self.shape # grab original shape information for later
|
|
136
|
+
if pd.get_option("display.max_rows") is None or df_shape[0] > pd.get_option("display.max_rows"):
|
|
137
|
+
html_df = self.head(pd.get_option("display.min_rows") + 1)
|
|
138
|
+
else:
|
|
139
|
+
html_df = self.copy()
|
|
140
|
+
|
|
104
141
|
# replace index to ensure proper behavior for duplicate index values
|
|
105
|
-
index_values =
|
|
106
|
-
html_df =
|
|
142
|
+
index_values = html_df.index
|
|
143
|
+
html_df = html_df.reset_index(drop=True)
|
|
107
144
|
|
|
108
145
|
# Apply repacking to all nested columns
|
|
109
146
|
repr = html_df.style.format(
|
|
@@ -122,15 +159,17 @@ class NestedFrame(pd.DataFrame):
|
|
|
122
159
|
|
|
123
160
|
# Recover some truncation formatting, limited to head truncation
|
|
124
161
|
if pd.get_option("display.max_rows") is None:
|
|
162
|
+
# Just display header
|
|
125
163
|
return repr.to_html(max_rows=0)
|
|
126
|
-
elif
|
|
164
|
+
elif df_shape[0] > pd.get_option("display.max_rows"):
|
|
165
|
+
# when over the max_rows threshold, display with truncation ("..." row at the end)
|
|
127
166
|
html_repr = repr.to_html(max_rows=pd.get_option("display.min_rows"))
|
|
128
167
|
else:
|
|
129
168
|
# when under the max_rows threshold, display all rows (behavior of 0 here)
|
|
130
169
|
html_repr = repr.to_html(max_rows=0)
|
|
131
170
|
|
|
132
171
|
# Manually append dimensionality to a styler output
|
|
133
|
-
html_repr += f"{
|
|
172
|
+
html_repr += f"{df_shape[0]} rows x {df_shape[1]} columns"
|
|
134
173
|
|
|
135
174
|
return html_repr
|
|
136
175
|
|
|
@@ -166,10 +205,14 @@ class NestedFrame(pd.DataFrame):
|
|
|
166
205
|
|
|
167
206
|
def __getitem__(self, item):
|
|
168
207
|
"""Adds custom __getitem__ behavior for nested columns"""
|
|
208
|
+
if isinstance(item, str):
|
|
209
|
+
return self._getitem_str(item)
|
|
210
|
+
elif self._is_key_list(item):
|
|
211
|
+
return self._getitem_list(item)
|
|
169
212
|
|
|
170
|
-
|
|
171
|
-
return super().__getitem__(item)
|
|
213
|
+
return super().__getitem__(item)
|
|
172
214
|
|
|
215
|
+
def _getitem_str(self, item):
|
|
173
216
|
# Preempt the nested check if the item is a base column, with or without
|
|
174
217
|
# dots and backticks.
|
|
175
218
|
if item in self.columns:
|
|
@@ -191,6 +234,29 @@ class NestedFrame(pd.DataFrame):
|
|
|
191
234
|
else:
|
|
192
235
|
raise KeyError(f"Column '{cleaned_item}' not found in nested columns or base columns")
|
|
193
236
|
|
|
237
|
+
def _is_key_list(self, item):
|
|
238
|
+
if not is_list_like(item):
|
|
239
|
+
return False
|
|
240
|
+
for k in item:
|
|
241
|
+
if not isinstance(k, str):
|
|
242
|
+
return False
|
|
243
|
+
if not self._is_known_column(k):
|
|
244
|
+
return False
|
|
245
|
+
return True
|
|
246
|
+
|
|
247
|
+
def _getitem_list(self, item):
|
|
248
|
+
non_nested_keys = [k for k in item if k in self.columns]
|
|
249
|
+
result = super().__getitem__(non_nested_keys)
|
|
250
|
+
components = [self._parse_hierarchical_components(k) for k in item]
|
|
251
|
+
nested_components = [c for c in components if self._is_known_hierarchical_column(c)]
|
|
252
|
+
nested_columns = defaultdict(set)
|
|
253
|
+
for comps in nested_components:
|
|
254
|
+
nested_columns[comps[0]].add(".".join(comps[1:]))
|
|
255
|
+
for c in nested_columns:
|
|
256
|
+
unselected_columns = [f for f in self.dtypes[c].field_names if f not in nested_columns[c]]
|
|
257
|
+
result[c] = self[c].nest.without_field(unselected_columns)
|
|
258
|
+
return result
|
|
259
|
+
|
|
194
260
|
def __setitem__(self, key, value):
|
|
195
261
|
"""Adds custom __setitem__ behavior for nested columns"""
|
|
196
262
|
components = self._parse_hierarchical_components(key)
|
|
@@ -224,7 +290,12 @@ class NestedFrame(pd.DataFrame):
|
|
|
224
290
|
self._update_inplace(new_df)
|
|
225
291
|
return None
|
|
226
292
|
|
|
227
|
-
|
|
293
|
+
super().__setitem__(key, value)
|
|
294
|
+
self._cast_cols_to_nested(struct_list=False)
|
|
295
|
+
|
|
296
|
+
def __delitem__(self, key):
|
|
297
|
+
"""Delete a column or a nested field using dot notation (e.g., del nf['nested.x'])"""
|
|
298
|
+
self.drop([key], axis=1, inplace=True)
|
|
228
299
|
|
|
229
300
|
def add_nested(
|
|
230
301
|
self,
|
|
@@ -467,7 +538,7 @@ class NestedFrame(pd.DataFrame):
|
|
|
467
538
|
# This is a simple heuristic but infers more than its dtype
|
|
468
539
|
# which will probably be an object.
|
|
469
540
|
sample_val = df[col].iloc[0]
|
|
470
|
-
if not hasattr(sample_val, "__iter__") and not isinstance(sample_val,
|
|
541
|
+
if not hasattr(sample_val, "__iter__") and not isinstance(sample_val, str | bytes):
|
|
471
542
|
raise ValueError(
|
|
472
543
|
f"Cannot pack column {col} which does not contain an iterable list based "
|
|
473
544
|
"on its first value, {sample_val}."
|
|
@@ -554,7 +625,10 @@ class NestedFrame(pd.DataFrame):
|
|
|
554
625
|
# drop targeted sub-columns for each nested column
|
|
555
626
|
for col in nested_cols:
|
|
556
627
|
sub_cols = [label.split(".")[1] for label in nested_labels if label.split(".")[0] == col]
|
|
557
|
-
|
|
628
|
+
if inplace:
|
|
629
|
+
self[col] = self[col].nest.without_field(sub_cols)
|
|
630
|
+
else:
|
|
631
|
+
self = self.assign(**{f"{col}": self[col].nest.without_field(sub_cols)})
|
|
558
632
|
|
|
559
633
|
# drop remaining base columns
|
|
560
634
|
if len(base_labels) > 0:
|
|
@@ -568,7 +642,7 @@ class NestedFrame(pd.DataFrame):
|
|
|
568
642
|
errors=errors,
|
|
569
643
|
)
|
|
570
644
|
else:
|
|
571
|
-
return self
|
|
645
|
+
return self if not inplace else None
|
|
572
646
|
# Otherwise just drop like pandas
|
|
573
647
|
return super().drop(
|
|
574
648
|
labels=labels,
|
|
@@ -1216,7 +1290,7 @@ class NestedFrame(pd.DataFrame):
|
|
|
1216
1290
|
else:
|
|
1217
1291
|
iterators.append(self[layer].array.iter_field_lists(col))
|
|
1218
1292
|
|
|
1219
|
-
results = [func(*cols, *extra_args, **kwargs) for cols in zip(*iterators)]
|
|
1293
|
+
results = [func(*cols, *extra_args, **kwargs) for cols in zip(*iterators, strict=True)]
|
|
1220
1294
|
results_nf = NestedFrame(results, index=self.index)
|
|
1221
1295
|
|
|
1222
1296
|
if infer_nesting:
|
|
@@ -1242,6 +1316,28 @@ class NestedFrame(pd.DataFrame):
|
|
|
1242
1316
|
|
|
1243
1317
|
return results_nf
|
|
1244
1318
|
|
|
1319
|
+
def to_pandas(self, list_struct=False) -> pd.DataFrame:
|
|
1320
|
+
"""Convert to an ordinal pandas DataFrame, with no NestedDtype series.
|
|
1321
|
+
|
|
1322
|
+
NestedDtype is cast to pd.ArrowDtype
|
|
1323
|
+
|
|
1324
|
+
Parameters
|
|
1325
|
+
----------
|
|
1326
|
+
list_struct: bool
|
|
1327
|
+
If True, cast nested columns to pandas struct-list arrow extension
|
|
1328
|
+
array columns. If False (default), cast nested columns to
|
|
1329
|
+
list-struct array columns.
|
|
1330
|
+
|
|
1331
|
+
Returns
|
|
1332
|
+
-------
|
|
1333
|
+
pd.DataFrame
|
|
1334
|
+
Ordinal pandas DataFrame.
|
|
1335
|
+
"""
|
|
1336
|
+
df = pd.DataFrame(self)
|
|
1337
|
+
for col in self.nested_columns:
|
|
1338
|
+
df[col] = df[col].array.to_arrow_ext_array(list_struct=list_struct)
|
|
1339
|
+
return df
|
|
1340
|
+
|
|
1245
1341
|
def to_parquet(self, path, **kwargs) -> None:
|
|
1246
1342
|
"""Creates parquet file(s) with the data of a NestedFrame, either
|
|
1247
1343
|
as a single parquet file where each nested dataset is packed into its
|
|
@@ -1269,13 +1365,14 @@ class NestedFrame(pd.DataFrame):
|
|
|
1269
1365
|
>>> nf = generate_data(5,5, seed=1)
|
|
1270
1366
|
>>> nf.to_parquet("nestedframe.parquet")
|
|
1271
1367
|
"""
|
|
1368
|
+
df = self.to_pandas(list_struct=False)
|
|
1272
1369
|
|
|
1273
1370
|
# Write through pyarrow
|
|
1274
1371
|
# This is potentially not zero-copy
|
|
1275
1372
|
# Note: Without pandas metadata, index writing is not as robust set
|
|
1276
1373
|
# preserve_index=None for best behavior but index will generally
|
|
1277
1374
|
# need to be set manually on load
|
|
1278
|
-
table = pa.Table.from_pandas(
|
|
1375
|
+
table = pa.Table.from_pandas(df, preserve_index=None)
|
|
1279
1376
|
|
|
1280
1377
|
# Drop pandas metadata to make sure nesteddtypes are not preserved
|
|
1281
1378
|
# Do this by rebuilding the schema
|
|
@@ -87,7 +87,7 @@ def read_parquet(
|
|
|
87
87
|
# First load through pyarrow
|
|
88
88
|
# Check if `data` is a file-like object or a sequence
|
|
89
89
|
if hasattr(data, "read") or (
|
|
90
|
-
isinstance(data, Sequence) and not isinstance(data,
|
|
90
|
+
isinstance(data, Sequence) and not isinstance(data, str | bytes | bytearray)
|
|
91
91
|
):
|
|
92
92
|
# If `data` is a file-like object or a sequence, pass it directly to pyarrow
|
|
93
93
|
table = pq.read_table(data, columns=columns, **kwargs)
|
|
@@ -103,7 +103,7 @@ def read_parquet(
|
|
|
103
103
|
# was from a nested column.
|
|
104
104
|
if columns is not None:
|
|
105
105
|
nested_structures: dict[str, list[int]] = {}
|
|
106
|
-
for i, (col_in, col_pa) in enumerate(zip(columns, table.column_names)):
|
|
106
|
+
for i, (col_in, col_pa) in enumerate(zip(columns, table.column_names, strict=True)):
|
|
107
107
|
# if the column name is not the same, it was a partial load
|
|
108
108
|
if col_in != col_pa:
|
|
109
109
|
# get the top-level column name
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
from __future__ import annotations # Python 3.9 requires it for X | Y type hints
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any
|
|
4
|
+
|
|
5
|
+
import pyarrow as pa
|
|
6
|
+
|
|
7
|
+
from nested_pandas.series.utils import transpose_struct_list_chunked, validate_list_struct_type
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from nested_pandas.series._storage.struct_list_storage import StructListStorage
|
|
11
|
+
from nested_pandas.series._storage.table_storage import TableStorage
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ListStructStorage:
|
|
15
|
+
"""Store nested data as a PyArrow list-struct array.
|
|
16
|
+
|
|
17
|
+
Parameters
|
|
18
|
+
----------
|
|
19
|
+
array : pa.ListArray or pa.ChunkedArray
|
|
20
|
+
Pyarrow list-array with a struct value type. An array or a chunk-array
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
_data: pa.ChunkedArray
|
|
24
|
+
|
|
25
|
+
def __init__(self, array: pa.ListArray | pa.ChunkedArray) -> None:
|
|
26
|
+
if isinstance(array, pa.ListArray):
|
|
27
|
+
array = pa.chunked_array([array])
|
|
28
|
+
if not isinstance(array, pa.ChunkedArray):
|
|
29
|
+
raise ValueError("array must be of type pa.ChunkedArray")
|
|
30
|
+
validate_list_struct_type(array.type)
|
|
31
|
+
self._data = array
|
|
32
|
+
|
|
33
|
+
@property
|
|
34
|
+
def data(self) -> pa.ChunkedArray:
|
|
35
|
+
return self._data
|
|
36
|
+
|
|
37
|
+
@classmethod
|
|
38
|
+
def from_struct_list_storage(cls, struct_list_storage: StructListStorage) -> Self: # type: ignore # noqa: F821
|
|
39
|
+
"""Construct from a StructListStorage object.
|
|
40
|
+
|
|
41
|
+
Parameters
|
|
42
|
+
----------
|
|
43
|
+
struct_list_storage : StructListStorage
|
|
44
|
+
StructListStorage object.
|
|
45
|
+
"""
|
|
46
|
+
data = transpose_struct_list_chunked(struct_list_storage.data, validate=False)
|
|
47
|
+
return cls(data)
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def from_table_storage(cls, table_storage: TableStorage) -> Self: # type: ignore # noqa: F821
|
|
51
|
+
"""Construct from a TableStorage object.
|
|
52
|
+
|
|
53
|
+
Parameters
|
|
54
|
+
----------
|
|
55
|
+
table_storage : TableStorage
|
|
56
|
+
TableStorage object.
|
|
57
|
+
"""
|
|
58
|
+
from nested_pandas.series._storage import StructListStorage
|
|
59
|
+
|
|
60
|
+
struct_list_storage = StructListStorage.from_table_storage(table_storage)
|
|
61
|
+
return cls.from_struct_list_storage(struct_list_storage)
|
|
62
|
+
|
|
63
|
+
def __len__(self) -> int:
|
|
64
|
+
return len(self._data)
|
|
65
|
+
|
|
66
|
+
def __eq__(self, other: Any) -> bool:
|
|
67
|
+
if not isinstance(other, type(self)):
|
|
68
|
+
return False
|
|
69
|
+
return self._data == other._data
|
|
70
|
+
|
|
71
|
+
@property
|
|
72
|
+
def nbytes(self) -> int:
|
|
73
|
+
"""Number of bytes consumed by the data in memory."""
|
|
74
|
+
return self._data.nbytes
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def type(self) -> pa.ListType:
|
|
78
|
+
"""Pyarrow type of the underlying array."""
|
|
79
|
+
return self._data.type
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def num_chunks(self) -> int:
|
|
83
|
+
"""Number of chunks in the underlying array."""
|
|
84
|
+
return self._data.num_chunks
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
from __future__ import annotations # Python 3.9 requires it for X | Y type hints
|
|
2
|
+
|
|
3
|
+
from collections.abc import Iterator
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
5
|
+
|
|
6
|
+
import pyarrow as pa
|
|
7
|
+
|
|
8
|
+
from nested_pandas.series.utils import (
|
|
9
|
+
table_to_struct_array,
|
|
10
|
+
transpose_list_struct_chunked,
|
|
11
|
+
validate_struct_list_array_for_equal_lengths,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from nested_pandas.series._storage.list_struct_storage import ListStructStorage
|
|
16
|
+
from nested_pandas.series._storage.table_storage import TableStorage
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class StructListStorage:
|
|
20
|
+
"""Store nested data as a PyArrow struct-list array.
|
|
21
|
+
|
|
22
|
+
Parameters
|
|
23
|
+
----------
|
|
24
|
+
array : pa.StructArray or pa.ChunkedArray
|
|
25
|
+
Pyarrow struct-array with all fields to be list-arrays.
|
|
26
|
+
All list-values must be "aligned", e.g., have the same length.
|
|
27
|
+
validate : bool (default True)
|
|
28
|
+
Check that all the lists have the same lengths for each struct-value.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
_data: pa.ChunkedArray
|
|
32
|
+
|
|
33
|
+
def __init__(self, array: pa.StructArray | pa.ChunkedArray, *, validate: bool = True) -> None:
|
|
34
|
+
if isinstance(array, pa.StructArray):
|
|
35
|
+
array = pa.chunked_array([array])
|
|
36
|
+
if not isinstance(array, pa.ChunkedArray):
|
|
37
|
+
raise ValueError("array must be a StructArray or ChunkedArray")
|
|
38
|
+
|
|
39
|
+
if validate:
|
|
40
|
+
for chunk in array.chunks:
|
|
41
|
+
validate_struct_list_array_for_equal_lengths(chunk)
|
|
42
|
+
|
|
43
|
+
self._data = array
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def data(self) -> pa.ChunkedArray:
|
|
47
|
+
return self._data
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def from_list_struct_storage(cls, list_struct_storage: ListStructStorage) -> Self: # type: ignore # noqa: F821
|
|
51
|
+
"""Construct from a ListStructStorage object.
|
|
52
|
+
|
|
53
|
+
Parameters
|
|
54
|
+
----------
|
|
55
|
+
list_struct_storage : ListStructStorage
|
|
56
|
+
ListStructStorage object.
|
|
57
|
+
"""
|
|
58
|
+
data = transpose_list_struct_chunked(list_struct_storage.data)
|
|
59
|
+
return cls(data, validate=False)
|
|
60
|
+
|
|
61
|
+
@classmethod
|
|
62
|
+
def from_table_storage(cls, table_storage: TableStorage) -> Self: # type: ignore # noqa: F821
|
|
63
|
+
"""Construct from a TableStorage object.
|
|
64
|
+
|
|
65
|
+
Parameters
|
|
66
|
+
----------
|
|
67
|
+
table_storage : TableStorage
|
|
68
|
+
TableStorage object.
|
|
69
|
+
"""
|
|
70
|
+
data = table_to_struct_array(table_storage.data)
|
|
71
|
+
return cls(data, validate=False)
|
|
72
|
+
|
|
73
|
+
def __iter__(self) -> Iterator[pa.StructScalar]:
|
|
74
|
+
return iter(self._data)
|