nested-pandas 0.2.2__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.copier-answers.yml +1 -1
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.github/ISSUE_TEMPLATE/1-bug_report.md +2 -2
- nested_pandas-0.3.0/.github/workflows/asv-main.yml +68 -0
- nested_pandas-0.3.0/.github/workflows/asv-nightly.yml +72 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.github/workflows/asv-pr.yml +20 -36
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.github/workflows/build-documentation.yml +1 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.github/workflows/pre-commit-ci.yml +2 -1
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.github/workflows/publish-to-pypi.yml +1 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.pre-commit-config.yaml +3 -1
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.readthedocs.yml +1 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/LICENSE +1 -1
- {nested_pandas-0.2.2/src/nested_pandas.egg-info → nested_pandas-0.3.0}/PKG-INFO +3 -3
- nested_pandas-0.3.0/benchmarks/README.md +12 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/benchmarks/asv.conf.json +1 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/docs/conf.py +1 -1
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/docs/index.rst +1 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/docs/tutorials/data_loading_notebook.ipynb +8 -8
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/docs/tutorials/data_manipulation.ipynb +2 -2
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/docs/tutorials/low_level.ipynb +9 -9
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/docs/tutorials/nested_spectra.ipynb +5 -5
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/pyproject.toml +3 -1
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas/_version.py +2 -2
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas/nestedframe/core.py +271 -95
- nested_pandas-0.3.0/src/nested_pandas/nestedframe/utils.py +61 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas/series/ext_array.py +10 -2
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas/series/packer.py +11 -4
- {nested_pandas-0.2.2 → nested_pandas-0.3.0/src/nested_pandas.egg-info}/PKG-INFO +3 -3
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas.egg-info/SOURCES.txt +1 -1
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas.egg-info/requires.txt +1 -1
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/tests/nested_pandas/nestedframe/test_nestedframe.py +359 -15
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/tests/nested_pandas/series/test_accessor.py +25 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/tests/nested_pandas/series/test_packer.py +94 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/tests/nested_pandas/utils/test_utils.py +22 -0
- nested_pandas-0.2.2/.github/workflows/asv-main.yml +0 -101
- nested_pandas-0.2.2/.github/workflows/asv-nightly.yml +0 -93
- nested_pandas-0.2.2/src/nested_pandas/nestedframe/utils.py +0 -35
- nested_pandas-0.2.2/tests/nested_pandas/nestedframe/test_nestedframe_utils.py +0 -17
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.git_archival.txt +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.gitattributes +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.github/ISSUE_TEMPLATE/0-general_issue.md +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.github/ISSUE_TEMPLATE/2-feature_request.md +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.github/dependabot.yml +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.github/pull_request_template.md +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.github/workflows/publish-benchmarks-pr.yml +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.github/workflows/smoke-test.yml +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.github/workflows/testing-and-coverage.yml +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.gitignore +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/.setup_dev.sh +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/README.md +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/benchmarks/__init__.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/benchmarks/benchmarks.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/docs/Makefile +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/docs/_static/custom.css +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/docs/gettingstarted/contributing.rst +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/docs/gettingstarted/installation.rst +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/docs/gettingstarted/quickstart.ipynb +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/docs/gettingstarted.rst +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/docs/requirements.txt +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/docs/tutorials/README.md +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/docs/tutorials.rst +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/nestedframe.png +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/setup.cfg +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas/__init__.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas/datasets/__init__.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas/datasets/generation.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas/nestedframe/__init__.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas/nestedframe/io.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas/py.typed +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas/series/__init__.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas/series/accessor.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas/series/dtype.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas/series/utils.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas/utils/__init__.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas/utils/utils.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas.egg-info/dependency_links.txt +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/src/nested_pandas.egg-info/top_level.txt +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/tests/nested_pandas/conftest.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/tests/nested_pandas/datasets/test_generation.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/tests/nested_pandas/e2e_tests/test_issue89.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/tests/nested_pandas/nestedframe/test_io.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/tests/nested_pandas/series/test_dtype.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/tests/nested_pandas/series/test_ext_array.py +0 -0
- {nested_pandas-0.2.2 → nested_pandas-0.3.0}/tests/nested_pandas/test_packaging.py +0 -0
|
@@ -13,5 +13,5 @@ assignees: ''
|
|
|
13
13
|
Please check the following:
|
|
14
14
|
|
|
15
15
|
- [ ] I have described the situation in which the bug arose, including what code was executed, information about my environment, and any applicable data others will need to reproduce the problem.
|
|
16
|
-
- [ ] I have included available evidence of the unexpected behavior (including error messages, screenshots, and/or plots) as well as a
|
|
17
|
-
- [ ] If I have a solution in mind, I have provided an explanation and/or pseudocode and/or task list.
|
|
16
|
+
- [ ] I have included available evidence of the unexpected behavior (including error messages, screenshots, and/or plots) as well as a description of what I expected instead.
|
|
17
|
+
- [ ] If I have a solution in mind, I have provided an explanation and/or pseudocode and/or task list.
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# This workflow will run benchmarks with airspeed velocity (asv),
|
|
2
|
+
# store the new results in the "benchmarks" branch and publish them
|
|
3
|
+
# to a dashboard on GH Pages.
|
|
4
|
+
name: Run ASV benchmarks for main
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
branches: [ main ]
|
|
9
|
+
|
|
10
|
+
env:
|
|
11
|
+
PYTHON_VERSION: "3.10"
|
|
12
|
+
ASV_VERSION: "0.6.4"
|
|
13
|
+
WORKING_DIR: ${{github.workspace}}/benchmarks
|
|
14
|
+
|
|
15
|
+
concurrency:
|
|
16
|
+
group: ${{github.workflow}}-${{github.ref}}
|
|
17
|
+
cancel-in-progress: true
|
|
18
|
+
|
|
19
|
+
jobs:
|
|
20
|
+
asv-main:
|
|
21
|
+
runs-on: ubuntu-latest
|
|
22
|
+
permissions:
|
|
23
|
+
contents: write
|
|
24
|
+
defaults:
|
|
25
|
+
run:
|
|
26
|
+
working-directory: ${{env.WORKING_DIR}}
|
|
27
|
+
steps:
|
|
28
|
+
- name: Set up Python ${{env.PYTHON_VERSION}}
|
|
29
|
+
uses: actions/setup-python@v5
|
|
30
|
+
with:
|
|
31
|
+
python-version: ${{env.PYTHON_VERSION}}
|
|
32
|
+
- name: Checkout main branch of the repository
|
|
33
|
+
uses: actions/checkout@v4
|
|
34
|
+
with:
|
|
35
|
+
fetch-depth: 0
|
|
36
|
+
- name: Install dependencies
|
|
37
|
+
run: pip install "asv[virtualenv]==${{env.ASV_VERSION}}"
|
|
38
|
+
- name: Configure git
|
|
39
|
+
run: |
|
|
40
|
+
git config user.name "github-actions[bot]"
|
|
41
|
+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
|
42
|
+
- name: Create ASV machine config file
|
|
43
|
+
run: asv machine --machine gh-runner --yes
|
|
44
|
+
- name: Fetch previous results from the "benchmarks" branch
|
|
45
|
+
run: |
|
|
46
|
+
if git ls-remote --exit-code origin benchmarks > /dev/null 2>&1; then
|
|
47
|
+
git merge origin/benchmarks \
|
|
48
|
+
--allow-unrelated-histories \
|
|
49
|
+
--no-commit
|
|
50
|
+
mv ../_results .
|
|
51
|
+
fi
|
|
52
|
+
- name: Run ASV for the main branch
|
|
53
|
+
run: asv run ALL --skip-existing --verbose || true
|
|
54
|
+
- name: Submit new results to the "benchmarks" branch
|
|
55
|
+
uses: JamesIves/github-pages-deploy-action@v4
|
|
56
|
+
with:
|
|
57
|
+
branch: benchmarks
|
|
58
|
+
folder: ${{env.WORKING_DIR}}/_results
|
|
59
|
+
target-folder: _results
|
|
60
|
+
- name: Generate dashboard HTML
|
|
61
|
+
run: |
|
|
62
|
+
asv show
|
|
63
|
+
asv publish
|
|
64
|
+
- name: Deploy to Github pages
|
|
65
|
+
uses: JamesIves/github-pages-deploy-action@v4
|
|
66
|
+
with:
|
|
67
|
+
branch: gh-pages
|
|
68
|
+
folder: ${{env.WORKING_DIR}}/_html
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# This workflow will run daily at 06:45.
|
|
2
|
+
# It will run benchmarks with airspeed velocity (asv)
|
|
3
|
+
# and compare performance with the previous nightly build.
|
|
4
|
+
name: Run benchmarks nightly job
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
schedule:
|
|
8
|
+
- cron: 45 6 * * *
|
|
9
|
+
workflow_dispatch:
|
|
10
|
+
|
|
11
|
+
env:
|
|
12
|
+
PYTHON_VERSION: "3.10"
|
|
13
|
+
ASV_VERSION: "0.6.4"
|
|
14
|
+
WORKING_DIR: ${{github.workspace}}/benchmarks
|
|
15
|
+
NIGHTLY_HASH_FILE: nightly-hash
|
|
16
|
+
|
|
17
|
+
jobs:
|
|
18
|
+
asv-nightly:
|
|
19
|
+
runs-on: ubuntu-latest
|
|
20
|
+
defaults:
|
|
21
|
+
run:
|
|
22
|
+
working-directory: ${{env.WORKING_DIR}}
|
|
23
|
+
steps:
|
|
24
|
+
- name: Set up Python ${{env.PYTHON_VERSION}}
|
|
25
|
+
uses: actions/setup-python@v5
|
|
26
|
+
with:
|
|
27
|
+
python-version: ${{env.PYTHON_VERSION}}
|
|
28
|
+
- name: Checkout main branch of the repository
|
|
29
|
+
uses: actions/checkout@v4
|
|
30
|
+
with:
|
|
31
|
+
fetch-depth: 0
|
|
32
|
+
- name: Install dependencies
|
|
33
|
+
run: pip install "asv[virtualenv]==${{env.ASV_VERSION}}"
|
|
34
|
+
- name: Configure git
|
|
35
|
+
run: |
|
|
36
|
+
git config user.name "github-actions[bot]"
|
|
37
|
+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
|
38
|
+
- name: Create ASV machine config file
|
|
39
|
+
run: asv machine --machine gh-runner --yes
|
|
40
|
+
- name: Fetch previous results from the "benchmarks" branch
|
|
41
|
+
run: |
|
|
42
|
+
if git ls-remote --exit-code origin benchmarks > /dev/null 2>&1; then
|
|
43
|
+
git merge origin/benchmarks \
|
|
44
|
+
--allow-unrelated-histories \
|
|
45
|
+
--no-commit
|
|
46
|
+
mv ../_results .
|
|
47
|
+
fi
|
|
48
|
+
- name: Get nightly dates under comparison
|
|
49
|
+
id: nightly-dates
|
|
50
|
+
run: |
|
|
51
|
+
echo "yesterday=$(date -d yesterday +'%Y-%m-%d')" >> $GITHUB_OUTPUT
|
|
52
|
+
echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
|
|
53
|
+
- name: Use last nightly commit hash from cache
|
|
54
|
+
uses: actions/cache@v4
|
|
55
|
+
with:
|
|
56
|
+
path: ${{env.WORKING_DIR}}
|
|
57
|
+
key: nightly-results-${{steps.nightly-dates.outputs.yesterday}}
|
|
58
|
+
- name: Run comparison of main against last nightly build
|
|
59
|
+
run: |
|
|
60
|
+
HASH_FILE=${{env.NIGHTLY_HASH_FILE}}
|
|
61
|
+
CURRENT_HASH=${{github.sha}}
|
|
62
|
+
if [ -f $HASH_FILE ]; then
|
|
63
|
+
PREV_HASH=$(cat $HASH_FILE)
|
|
64
|
+
asv continuous $PREV_HASH $CURRENT_HASH --verbose || true
|
|
65
|
+
asv compare $PREV_HASH $CURRENT_HASH --sort ratio --verbose
|
|
66
|
+
fi
|
|
67
|
+
echo $CURRENT_HASH > $HASH_FILE
|
|
68
|
+
- name: Update last nightly hash in cache
|
|
69
|
+
uses: actions/cache@v4
|
|
70
|
+
with:
|
|
71
|
+
path: ${{env.WORKING_DIR}}
|
|
72
|
+
key: nightly-results-${{steps.nightly-dates.outputs.today}}
|
|
@@ -11,76 +11,60 @@ on:
|
|
|
11
11
|
workflow_dispatch:
|
|
12
12
|
|
|
13
13
|
concurrency:
|
|
14
|
-
group: ${{
|
|
14
|
+
group: ${{github.workflow}}-${{github.ref}}
|
|
15
15
|
cancel-in-progress: true
|
|
16
16
|
|
|
17
17
|
env:
|
|
18
18
|
PYTHON_VERSION: "3.10"
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
ASV_VERSION: "0.6.4"
|
|
20
|
+
WORKING_DIR: ${{github.workspace}}/benchmarks
|
|
21
|
+
ARTIFACTS_DIR: ${{github.workspace}}/artifacts
|
|
21
22
|
|
|
22
23
|
jobs:
|
|
23
|
-
setup-python:
|
|
24
|
-
runs-on: ubuntu-latest
|
|
25
|
-
steps:
|
|
26
|
-
- name: Cache Python ${{ env.PYTHON_VERSION }}
|
|
27
|
-
uses: actions/cache@v4
|
|
28
|
-
with:
|
|
29
|
-
path: ~/.cache/pip
|
|
30
|
-
key: python-${{ env.PYTHON_VERSION }}
|
|
31
|
-
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
|
32
|
-
uses: actions/setup-python@v5
|
|
33
|
-
with:
|
|
34
|
-
python-version: ${{ env.PYTHON_VERSION }}
|
|
35
24
|
asv-pr:
|
|
36
25
|
runs-on: ubuntu-latest
|
|
37
|
-
needs: setup-python
|
|
38
26
|
defaults:
|
|
39
27
|
run:
|
|
40
|
-
working-directory: ${{
|
|
28
|
+
working-directory: ${{env.WORKING_DIR}}
|
|
41
29
|
steps:
|
|
30
|
+
- name: Set up Python ${{env.PYTHON_VERSION}}
|
|
31
|
+
uses: actions/setup-python@v5
|
|
32
|
+
with:
|
|
33
|
+
python-version: ${{env.PYTHON_VERSION}}
|
|
42
34
|
- name: Checkout PR branch of the repository
|
|
43
35
|
uses: actions/checkout@v4
|
|
44
36
|
with:
|
|
45
37
|
fetch-depth: 0
|
|
46
38
|
- name: Display Workflow Run Information
|
|
47
39
|
run: |
|
|
48
|
-
echo "Workflow Run ID: ${{
|
|
49
|
-
- name: Cache Python ${{ env.PYTHON_VERSION }}
|
|
50
|
-
uses: actions/cache@v4
|
|
51
|
-
with:
|
|
52
|
-
path: ~/.cache/pip
|
|
53
|
-
key: python-${{ env.PYTHON_VERSION }}
|
|
40
|
+
echo "Workflow Run ID: ${{github.run_id}}"
|
|
54
41
|
- name: Install dependencies
|
|
55
|
-
run:
|
|
56
|
-
sudo apt-get update
|
|
57
|
-
python -m pip install --upgrade pip
|
|
58
|
-
pip install asv==0.6.1 virtualenv tabulate lf-asv-formatter
|
|
42
|
+
run: pip install "asv[virtualenv]==${{env.ASV_VERSION}}" lf-asv-formatter
|
|
59
43
|
- name: Make artifacts directory
|
|
60
|
-
run: mkdir -p ${{
|
|
44
|
+
run: mkdir -p ${{env.ARTIFACTS_DIR}}
|
|
61
45
|
- name: Save pull request number
|
|
62
|
-
run: echo ${{
|
|
46
|
+
run: echo ${{github.event.pull_request.number}} > ${{env.ARTIFACTS_DIR}}/pr
|
|
63
47
|
- name: Get current job logs URL
|
|
64
48
|
uses: Tiryoh/gha-jobid-action@v1
|
|
65
49
|
id: jobs
|
|
66
50
|
with:
|
|
67
|
-
github_token: ${{
|
|
68
|
-
job_name: ${{
|
|
51
|
+
github_token: ${{secrets.GITHUB_TOKEN}}
|
|
52
|
+
job_name: ${{github.job}}
|
|
69
53
|
- name: Create ASV machine config file
|
|
70
54
|
run: asv machine --machine gh-runner --yes
|
|
71
55
|
- name: Save comparison of PR against main branch
|
|
72
56
|
run: |
|
|
73
|
-
git remote add upstream https://github.com/${{
|
|
57
|
+
git remote add upstream https://github.com/${{github.repository}}.git
|
|
74
58
|
git fetch upstream
|
|
75
59
|
asv continuous upstream/main HEAD --verbose || true
|
|
76
60
|
asv compare upstream/main HEAD --sort ratio --verbose | tee output
|
|
77
|
-
python -m lf_asv_formatter --asv_version "$(
|
|
61
|
+
python -m lf_asv_formatter --asv_version "$(asv --version | awk '{print $2}')"
|
|
78
62
|
printf "\n\nClick [here]($STEP_URL) to view all benchmarks." >> output
|
|
79
|
-
mv output ${{
|
|
63
|
+
mv output ${{env.ARTIFACTS_DIR}}
|
|
80
64
|
env:
|
|
81
|
-
STEP_URL:
|
|
65
|
+
STEP_URL: ${{steps.jobs.outputs.html_url}}#step:10:1
|
|
82
66
|
- name: Upload artifacts (PR number and benchmarks output)
|
|
83
67
|
uses: actions/upload-artifact@v4
|
|
84
68
|
with:
|
|
85
69
|
name: benchmark-artifacts
|
|
86
|
-
path: ${{
|
|
70
|
+
path: ${{env.ARTIFACTS_DIR}}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
|
|
1
2
|
# This workflow runs pre-commit hooks on pushes and pull requests to main
|
|
2
3
|
# to enforce coding style. To ensure correct configuration, please refer to:
|
|
3
4
|
# https://lincc-ppt.readthedocs.io/en/latest/practices/ci_precommit.html
|
|
@@ -31,5 +32,5 @@ jobs:
|
|
|
31
32
|
extra_args: --all-files --verbose
|
|
32
33
|
env:
|
|
33
34
|
SKIP: "check-lincc-frameworks-template-version,no-commit-to-branch,check-added-large-files,validate-pyproject,sphinx-build,pytest-check"
|
|
34
|
-
- uses: pre-commit-ci/lite-action@v1.0.
|
|
35
|
+
- uses: pre-commit-ci/lite-action@v1.0.3
|
|
35
36
|
if: failure() && github.event_name == 'pull_request' && github.event.pull_request.draft == false
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
|
|
1
2
|
repos:
|
|
2
3
|
# Compare the local template version to the latest remote template version
|
|
3
4
|
# This hook should always pass. It will print a message if the local version
|
|
4
5
|
# is out of date.
|
|
5
6
|
- repo: https://github.com/lincc-frameworks/pre-commit-hooks
|
|
6
|
-
rev: v0.1.
|
|
7
|
+
rev: v0.1.2
|
|
7
8
|
hooks:
|
|
8
9
|
- id: check-lincc-frameworks-template-version
|
|
9
10
|
name: Check template version
|
|
@@ -16,6 +17,7 @@ repos:
|
|
|
16
17
|
name: Clear output from Jupyter notebooks
|
|
17
18
|
description: Clear output from Jupyter notebooks.
|
|
18
19
|
files: \.ipynb$
|
|
20
|
+
exclude: ^docs/pre_executed
|
|
19
21
|
stages: [commit]
|
|
20
22
|
language: system
|
|
21
23
|
entry: jupyter nbconvert --clear-output
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: nested-pandas
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Author-email: LINCC Frameworks <brantd@uw.edu>
|
|
5
5
|
License: MIT License
|
|
6
6
|
|
|
7
|
-
Copyright (c)
|
|
7
|
+
Copyright (c) 2024 LINCC Frameworks
|
|
8
8
|
|
|
9
9
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
10
10
|
of this software and associated documentation files (the "Software"), to deal
|
|
@@ -38,7 +38,7 @@ Requires-Dist: numpy
|
|
|
38
38
|
Requires-Dist: pandas<2.3,>=2.2
|
|
39
39
|
Requires-Dist: pyarrow>=15
|
|
40
40
|
Provides-Extra: dev
|
|
41
|
-
Requires-Dist: asv==0.6.
|
|
41
|
+
Requires-Dist: asv==0.6.4; extra == "dev"
|
|
42
42
|
Requires-Dist: jupyter; extra == "dev"
|
|
43
43
|
Requires-Dist: mypy; extra == "dev"
|
|
44
44
|
Requires-Dist: pre-commit; extra == "dev"
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Benchmarks
|
|
2
|
+
|
|
3
|
+
This directory contains files that will be run via continuous testing either
|
|
4
|
+
nightly or after committing code to a pull request.
|
|
5
|
+
|
|
6
|
+
The runtime and/or memory usage of the functions defined in these files will be
|
|
7
|
+
tracked and reported to give you a sense of the overall performance of your code.
|
|
8
|
+
|
|
9
|
+
You are encouraged to add, update, or remove benchmark functions to suit the needs
|
|
10
|
+
of your project.
|
|
11
|
+
|
|
12
|
+
For more information, see the documentation here: https://lincc-ppt.readthedocs.io/en/latest/practices/ci_benchmarking.html
|
|
@@ -15,7 +15,7 @@ sys.path.insert(0, os.path.abspath("../src/"))
|
|
|
15
15
|
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
|
|
16
16
|
|
|
17
17
|
project = "nested-pandas"
|
|
18
|
-
copyright = "
|
|
18
|
+
copyright = "2024, LINCC Frameworks"
|
|
19
19
|
author = "LINCC Frameworks"
|
|
20
20
|
release = version("nested-pandas")
|
|
21
21
|
# for example take major/minor
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"cell_type": "markdown",
|
|
12
12
|
"metadata": {},
|
|
13
13
|
"source": [
|
|
14
|
-
"With a valid Python environment, nested-pandas and
|
|
14
|
+
"With a valid Python environment, nested-pandas and its dependencies are easy to install using the `pip` package manager. The following command can be used to install it:"
|
|
15
15
|
]
|
|
16
16
|
},
|
|
17
17
|
{
|
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
"\n",
|
|
48
48
|
"We can use the `NestedFrame` constructor to create our base frame from a dictionary of our columns.\n",
|
|
49
49
|
"\n",
|
|
50
|
-
"We can then create an addtional pandas dataframes and pack them into our `NestedFrame` with `NestedFrame.add_nested
|
|
50
|
+
"We can then create an addtional pandas dataframes and pack them into our `NestedFrame` with `NestedFrame.add_nested`."
|
|
51
51
|
]
|
|
52
52
|
},
|
|
53
53
|
{
|
|
@@ -97,7 +97,7 @@
|
|
|
97
97
|
"# Note: that we use the `tempfile` module to create and then cleanup a temporary directory.\n",
|
|
98
98
|
"# You can of course remove this and use your own directory and real files on your system.\n",
|
|
99
99
|
"with tempfile.TemporaryDirectory() as temp_path:\n",
|
|
100
|
-
" # Generates parquet files with random data within our temporary
|
|
100
|
+
" # Generates parquet files with random data within our temporary directory.\n",
|
|
101
101
|
" generate_parquet_file(10, {\"nested1\": 100, \"nested2\": 10}, temp_path, file_per_layer=True)\n",
|
|
102
102
|
"\n",
|
|
103
103
|
" # Read each individual parquet file into its own dataframe.\n",
|
|
@@ -148,7 +148,7 @@
|
|
|
148
148
|
"source": [
|
|
149
149
|
"So inspect `nf`, a `NestedFrame` we created from our call to `read_parquet` with the `to_pack` argument, we're able to pack nested parquet files according to the shared index values with the index in `base.parquet`.\n",
|
|
150
150
|
"\n",
|
|
151
|
-
"The resulting `NestedFrame` having the same number of rows as `base.parquet` and with `nested1.parquet` and `nested2.parquet` packed into the
|
|
151
|
+
"The resulting `NestedFrame` having the same number of rows as `base.parquet` and with `nested1.parquet` and `nested2.parquet` packed into the `nested1` and `nested2` columns respectively."
|
|
152
152
|
]
|
|
153
153
|
},
|
|
154
154
|
{
|
|
@@ -164,7 +164,7 @@
|
|
|
164
164
|
"cell_type": "markdown",
|
|
165
165
|
"metadata": {},
|
|
166
166
|
"source": [
|
|
167
|
-
"Since we loaded each individual parquet file into its own dataframe, we can also verify that using `read_parquet` with the `to_pack` argument is equivalent to the following method of packing the dataframes directly with `NestedFrame.add_nested
|
|
167
|
+
"Since we loaded each individual parquet file into its own dataframe, we can also verify that using `read_parquet` with the `to_pack` argument is equivalent to the following method of packing the dataframes directly with `NestedFrame.add_nested`."
|
|
168
168
|
]
|
|
169
169
|
},
|
|
170
170
|
{
|
|
@@ -189,11 +189,11 @@
|
|
|
189
189
|
"source": [
|
|
190
190
|
"# Saving NestedFrames to Parquet Files\n",
|
|
191
191
|
"\n",
|
|
192
|
-
"Additionally we can save an existing `NestedFrame` as one of more parquet files using `NestedFrame.to_parquet
|
|
192
|
+
"Additionally we can save an existing `NestedFrame` as one of more parquet files using `NestedFrame.to_parquet`.\n",
|
|
193
193
|
"\n",
|
|
194
194
|
"When `by_layer=True` we save each individual layer of the NestedFrame into its own parquet file in a specified output directory.\n",
|
|
195
195
|
"\n",
|
|
196
|
-
"The base layer will be outputted to
|
|
196
|
+
"The base layer will be outputted to `base.parquet`, and each nested layer will be written to a file based on its column name. So the nested layer in column `nested1` will be written to `nested1.parquet`."
|
|
197
197
|
]
|
|
198
198
|
},
|
|
199
199
|
{
|
|
@@ -233,7 +233,7 @@
|
|
|
233
233
|
"source": [
|
|
234
234
|
"We also support saving a `NestedFrame` as a single parquet file where the packed layers are still packed in their respective columns.\n",
|
|
235
235
|
"\n",
|
|
236
|
-
"Here we provide `NestedFrame.to_parquet` with the desired path of the *single* output file (rather than the path of a directory to store *multiple* output files) and use `per_layer=False
|
|
236
|
+
"Here we provide `NestedFrame.to_parquet` with the desired path of the *single* output file (rather than the path of a directory to store *multiple* output files) and use `per_layer=False`.\n",
|
|
237
237
|
"\n",
|
|
238
238
|
"Our `read_parquet` function can load a `NestedFrame` saved in this single file parquet without requiring any additional arguments. "
|
|
239
239
|
]
|
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
"cell_type": "markdown",
|
|
50
50
|
"metadata": {},
|
|
51
51
|
"source": [
|
|
52
|
-
"First, we can directly fetch a column from our nested column (aptly called \"nested\"). For example, below we can fetch the time column, \"t\", by specifying `\"nested.t\"` as the column to retrieve. This returns a \"flat\" view of the nested t column, where all rows from all dataframes are present in one dataframe."
|
|
52
|
+
"First, we can directly fetch a column from our nested column (aptly called \"nested\"). For example, below we can fetch the time column, \"t\", by specifying `\"nested.t\"` as the column to retrieve. This returns a \"flat\" view of the nested `t` column, where all rows from all dataframes are present in one dataframe."
|
|
53
53
|
]
|
|
54
54
|
},
|
|
55
55
|
{
|
|
@@ -170,7 +170,7 @@
|
|
|
170
170
|
"cell_type": "markdown",
|
|
171
171
|
"metadata": {},
|
|
172
172
|
"source": [
|
|
173
|
-
"This is functionally equivalent to using `add_nested
|
|
173
|
+
"This is functionally equivalent to using `add_nested`:"
|
|
174
174
|
]
|
|
175
175
|
},
|
|
176
176
|
{
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
"# Lower-level interface for performance and flexibility\n",
|
|
9
9
|
"## Reveal the hidden power of nested Series\n",
|
|
10
10
|
"\n",
|
|
11
|
-
"This section is for users looking to optimize the
|
|
11
|
+
"This section is for users looking to optimize both the compute and memory performance of their workflows. This section also details a broader suite of data representations usable within `nested-pandas`.\n",
|
|
12
12
|
"It shows how to deal with individual nested columns: add, remove, and modify data using both \"flat-array\" and \"list-array\" representations.\n",
|
|
13
13
|
"It also demonstrates how to convert nested Series to and from different data types, like `pd.ArrowDtype`d Series, flat dataframes, list-array dataframes, and collections of nested elements."
|
|
14
14
|
]
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
"source": [
|
|
37
37
|
"## Generate some data and get a Series of `NestedDtype` type\n",
|
|
38
38
|
"\n",
|
|
39
|
-
"We are going to use built-in data generator to get a `NestedFrame` with a \"nested\" column being a `Series` of `NestedDtype` type.\n",
|
|
39
|
+
"We are going to use the built-in data generator to get a `NestedFrame` with a \"nested\" column being a `Series` of `NestedDtype` type.\n",
|
|
40
40
|
"This column would represent [light curves](https://en.wikipedia.org/wiki/Light_curve) of some astronomical objects. "
|
|
41
41
|
]
|
|
42
42
|
},
|
|
@@ -94,7 +94,7 @@
|
|
|
94
94
|
"id": "33d8caacf0bf042e",
|
|
95
95
|
"metadata": {},
|
|
96
96
|
"source": [
|
|
97
|
-
"You can also get a list of fields with `.fields` attribute"
|
|
97
|
+
"You can also get a list of fields with `.fields` attribute:"
|
|
98
98
|
]
|
|
99
99
|
},
|
|
100
100
|
{
|
|
@@ -130,7 +130,7 @@
|
|
|
130
130
|
"id": "7167f5a9c947d96f",
|
|
131
131
|
"metadata": {},
|
|
132
132
|
"source": [
|
|
133
|
-
"You can also get a subset of nested columns as a new nested Series"
|
|
133
|
+
"You can also get a subset of nested columns as a new nested Series:"
|
|
134
134
|
]
|
|
135
135
|
},
|
|
136
136
|
{
|
|
@@ -479,7 +479,7 @@
|
|
|
479
479
|
"source": [
|
|
480
480
|
"#### pd.Series from an array\n",
|
|
481
481
|
"\n",
|
|
482
|
-
"Construction with `pyarrow` struct arrays is the cheapest way to create a nested Series. It is very
|
|
482
|
+
"Construction with `pyarrow` struct arrays is the cheapest way to create a nested Series. It is very similar to the initialization of a `pd.Series` of `pd.ArrowDtype` type."
|
|
483
483
|
]
|
|
484
484
|
},
|
|
485
485
|
{
|
|
@@ -611,21 +611,21 @@
|
|
|
611
611
|
],
|
|
612
612
|
"metadata": {
|
|
613
613
|
"kernelspec": {
|
|
614
|
-
"display_name": "Python 3",
|
|
614
|
+
"display_name": "Python 3 (ipykernel)",
|
|
615
615
|
"language": "python",
|
|
616
616
|
"name": "python3"
|
|
617
617
|
},
|
|
618
618
|
"language_info": {
|
|
619
619
|
"codemirror_mode": {
|
|
620
620
|
"name": "ipython",
|
|
621
|
-
"version":
|
|
621
|
+
"version": 3
|
|
622
622
|
},
|
|
623
623
|
"file_extension": ".py",
|
|
624
624
|
"mimetype": "text/x-python",
|
|
625
625
|
"name": "python",
|
|
626
626
|
"nbconvert_exporter": "python",
|
|
627
|
-
"pygments_lexer": "
|
|
628
|
-
"version": "
|
|
627
|
+
"pygments_lexer": "ipython3",
|
|
628
|
+
"version": "3.12.6"
|
|
629
629
|
}
|
|
630
630
|
},
|
|
631
631
|
"nbformat": 4,
|
|
@@ -79,7 +79,7 @@
|
|
|
79
79
|
"flux = np.array([])\n",
|
|
80
80
|
"err = np.array([])\n",
|
|
81
81
|
"index = np.array([])\n",
|
|
82
|
-
"# Loop over each spectrum, adding
|
|
82
|
+
"# Loop over each spectrum, adding its data to the arrays\n",
|
|
83
83
|
"for i, hdu in enumerate(sp):\n",
|
|
84
84
|
" wave = np.append(wave, 10 ** hdu[\"COADD\"].data.loglam) # * u.angstrom\n",
|
|
85
85
|
" flux = np.append(flux, hdu[\"COADD\"].data.flux * 1e-17) # * u.erg/u.second/u.centimeter**2/u.angstrom\n",
|
|
@@ -115,7 +115,7 @@
|
|
|
115
115
|
"cell_type": "markdown",
|
|
116
116
|
"metadata": {},
|
|
117
117
|
"source": [
|
|
118
|
-
"And we can see that each object now has the
|
|
118
|
+
"And we can see that each object now has the `coadd_spectrum` nested column with the full spectrum available."
|
|
119
119
|
]
|
|
120
120
|
},
|
|
121
121
|
{
|
|
@@ -161,7 +161,7 @@
|
|
|
161
161
|
],
|
|
162
162
|
"metadata": {
|
|
163
163
|
"kernelspec": {
|
|
164
|
-
"display_name": "Python 3",
|
|
164
|
+
"display_name": "Python 3 (ipykernel)",
|
|
165
165
|
"language": "python",
|
|
166
166
|
"name": "python3"
|
|
167
167
|
},
|
|
@@ -175,9 +175,9 @@
|
|
|
175
175
|
"name": "python",
|
|
176
176
|
"nbconvert_exporter": "python",
|
|
177
177
|
"pygments_lexer": "ipython3",
|
|
178
|
-
"version": "3.
|
|
178
|
+
"version": "3.12.6"
|
|
179
179
|
}
|
|
180
180
|
},
|
|
181
181
|
"nbformat": 4,
|
|
182
|
-
"nbformat_minor":
|
|
182
|
+
"nbformat_minor": 4
|
|
183
183
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
|
|
1
2
|
[project]
|
|
2
3
|
name = "nested-pandas"
|
|
3
4
|
license = {file = "LICENSE"}
|
|
@@ -28,7 +29,7 @@ dependencies = [
|
|
|
28
29
|
# On a mac, install optional dependencies with `pip install '.[dev]'` (include the single quotes)
|
|
29
30
|
[project.optional-dependencies]
|
|
30
31
|
dev = [
|
|
31
|
-
"asv==0.6.
|
|
32
|
+
"asv==0.6.4", # Used to compute performance benchmarks
|
|
32
33
|
"jupyter", # Clears output from Jupyter notebooks
|
|
33
34
|
"mypy", # Used for static type checking of files
|
|
34
35
|
"pre-commit", # Used to run checks before finalizing a git commit
|
|
@@ -92,6 +93,7 @@ select = [
|
|
|
92
93
|
"D300",
|
|
93
94
|
"D417",
|
|
94
95
|
"D419",
|
|
96
|
+
# Numpy v2.0 compatibility
|
|
95
97
|
"NPY201",
|
|
96
98
|
]
|
|
97
99
|
|