nested-pandas 0.3.2__tar.gz → 0.3.3__tar.gz

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/.copier-answers.yml

@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: v2.0.3
+_commit: v2.0.5
 _src_path: gh:lincc-frameworks/python-project-template
 author_email: brantd@uw.edu
 author_name: LINCC Frameworks
@@ -22,3 +22,4 @@ python_versions:
 - '3.10'
 - '3.11'
 - '3.12'
+- '3.13'

nested_pandas-0.3.3/.github/ISSUE_TEMPLATE/README.md

@@ -0,0 +1,9 @@
+# Configurations
+
+Templates for various different issue types are defined in this directory
+and a pull request template is defined as ``../pull_request_template.md``. Adding,
+removing, and modifying these templates to suit the needs of your project is encouraged.
+
+For more information about these templates, look here: https://lincc-ppt.readthedocs.io/en/latest/practices/issue_pr_templating.html
+
+Or if you still have questions contact us: https://lincc-ppt.readthedocs.io/en/latest/source/contact.html

nested_pandas-0.3.3/.github/workflows/README.md

@@ -0,0 +1,9 @@
+# Workflows
+
+The .yml files in this directory are used to define the various continuous
+integration scripts that will be run on your behalf e.g. nightly as a smoke check,
+or when you create a new PR.
+
+For more information about CI and workflows, look here: https://lincc-ppt.readthedocs.io/en/latest/practices/ci.html
+
+Or if you still have questions contact us: https://lincc-ppt.readthedocs.io/en/latest/source/contact.html

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/.github/workflows/asv-main.yml

@@ -8,7 +8,7 @@ on:
     branches: [ main ]
 
 env:
-  PYTHON_VERSION: "3.10"
+  PYTHON_VERSION: "3.11"
   ASV_VERSION: "0.6.4"
   WORKING_DIR: ${{github.workspace}}/benchmarks
 

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/.github/workflows/asv-nightly.yml

@@ -9,7 +9,7 @@ on:
   workflow_dispatch:
   
 env:
-  PYTHON_VERSION: "3.10"
+  PYTHON_VERSION: "3.11"
   ASV_VERSION: "0.6.4"
   WORKING_DIR: ${{github.workspace}}/benchmarks
   NIGHTLY_HASH_FILE: nightly-hash

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/.github/workflows/asv-pr.yml

@@ -15,7 +15,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  PYTHON_VERSION: "3.10"
+  PYTHON_VERSION: "3.11"
   ASV_VERSION: "0.6.4"
   WORKING_DIR: ${{github.workspace}}/benchmarks
   ARTIFACTS_DIR: ${{github.workspace}}/artifacts

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/.github/workflows/build-documentation.yml

@@ -21,10 +21,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python 3.10
+    - name: Set up Python 3.11
       uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
+        python-version: '3.11'
     - name: Install dependencies
       run: |
         sudo apt-get update

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/.github/workflows/pre-commit-ci.yml

@@ -20,7 +20,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
+        python-version: '3.11'
     - name: Install dependencies
       run: |
         sudo apt-get update
@@ -32,5 +32,5 @@ jobs:
         extra_args: --all-files --verbose
       env:
         SKIP: "check-lincc-frameworks-template-version,no-commit-to-branch,check-added-large-files,validate-pyproject,sphinx-build,pytest-check"
-    - uses: pre-commit-ci/lite-action@v1.0.3
+    - uses: pre-commit-ci/lite-action@v1.1.0
       if: failure() && github.event_name == 'pull_request' && github.event.pull_request.draft == false

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/.github/workflows/publish-benchmarks-pr.yml

@@ -27,7 +27,7 @@ jobs:
         echo "Conclusion: ${{ github.event.workflow_run.conclusion }}"
         echo "Event: ${{ github.event.workflow_run.event }}"
     - name: Download artifact
-      uses: dawidd6/action-download-artifact@v6
+      uses: dawidd6/action-download-artifact@v7
       with:
         name: benchmark-artifacts
         run_id: ${{ github.event.workflow_run.id }}

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/.github/workflows/publish-to-pypi.yml

@@ -27,7 +27,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
+        python-version: '3.11'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/.github/workflows/smoke-test.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
 
     steps:
     - uses: actions/checkout@v4

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/.github/workflows/testing-and-coverage.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
 
     steps:
     - uses: actions/checkout@v4
@@ -33,6 +33,6 @@ jobs:
       run: |
         python -m pytest --cov=nested_pandas --cov-report=xml
     - name: Upload coverage report to codecov
-      uses: codecov/codecov-action@v4
+      uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
         description: Clear output from Jupyter notebooks.
         files: \.ipynb$
         exclude: ^docs/pre_executed
-        stages: [commit]
+        stages: [pre-commit]
         language: system
         entry: jupyter nbconvert --clear-output
     # Prevents committing directly branches named 'main' and 'master'.

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/.readthedocs.yml

@@ -9,7 +9,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.10"
+    python: "3.11"
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:

{nested_pandas-0.3.2/src/nested_pandas.egg-info → nested_pandas-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nested-pandas
-Version: 0.3.2
+Version: 0.3.3
 Author-email: LINCC Frameworks <brantd@uw.edu>
 License: MIT License
         

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/benchmarks/asv.conf.json

@@ -37,7 +37,7 @@
     // The Pythons you'd like to test against. If not provided, defaults
     // to the current version of Python used to run `asv`.
     "pythons": [
-        "3.10"
+        "3.11"
     ],
     // The matrix of dependencies to test. Each key is the name of a
     // package (in PyPI) and the values are version numbers.  An empty

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/docs/index.rst

@@ -19,10 +19,15 @@ is used internally to aid in scalability and performance.
   :align: center
   :alt: Example NestedFrame
 
+<<<<<<< before updating
 Nested-Pandas is motivated by time-domain astronomy use cases, where we see
 typically two levels of information, information about astronomical objects and
 then an associated set of `N` measurements of those objects. Nested-Pandas offers
 a performant and memory-efficient package for working with these types of datasets. 
+=======
+   >> conda create env -n <env_name> python=3.11
+   >> conda activate <env_name>
+>>>>>>> after updating
 
 Core advantages being:
 

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/src/nested_pandas/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.2'
-__version_tuple__ = version_tuple = (0, 3, 2)
+__version__ = version = '0.3.3'
+__version_tuple__ = version_tuple = (0, 3, 3)

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/src/nested_pandas/nestedframe/core.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 
 import ast
 import os
+import re
 
 import numpy as np
 import pandas as pd
@@ -18,6 +19,10 @@ from pandas.core.computation.parsing import clean_column_name
 from nested_pandas.series.dtype import NestedDtype
 from nested_pandas.series.packer import pack, pack_lists, pack_sorted_df_into_struct
 
+# Used to identify backtick-protected names in the expressions
+# used in NestedFrame.eval() and NestedFrame.query().
+_backtick_protected_names = re.compile(r"`[^`]+`", re.MULTILINE)
+
 
 class NestedPandasExprVisitor(PandasExprVisitor):
     """
@@ -83,9 +88,9 @@ class _NestResolver(dict):
         super().__init__()
         # Pre-load the field resolvers for all columns which are known at present.
         for column in outer.nested_columns:
-            self._initialize_field_resolver(column, outer)
+            self._initialize_column_resolver(column, outer)
 
-    def _initialize_field_resolver(self, column: str, outer: NestedFrame):
+    def _initialize_column_resolver(self, column: str, outer: NestedFrame):
         """
         Initialize a resolver for the given nested column, and also an alias
         for it, in the case of column names that have spaces or are otherwise
@@ -107,7 +112,7 @@ class _NestResolver(dict):
         if not super().__contains__(top_nest):
             if top_nest not in self._outer.nested_columns:
                 raise KeyError(f"Unknown nest {top_nest}")
-            self._initialize_field_resolver(top_nest, self._outer)
+            self._initialize_column_resolver(top_nest, self._outer)
         return super().__getitem__(top_nest)
 
     def __setitem__(self, item, _):
@@ -139,8 +144,20 @@ class _NestedFieldResolver:
         # Flattened only once for every access of this particular nest
         # within the expression.
         self._flat_nest = outer[nest_name].nest.to_flat()
+        # Save aliases to any columns that are not identifier-like.
+        # If our given frame has aliases for identifiers, use these instead
+        # of generating our own.
+        self._aliases = getattr(outer, "_aliases", None)
+        if self._aliases is None:
+            self._aliases = {}
+            for column in self._flat_nest.columns:
+                clean_id = clean_column_name(column)
+                if clean_id != column:
+                    self._aliases[clean_id] = column
 
     def __getattr__(self, item_name: str):
+        if self._aliases:
+            item_name = self._aliases.get(item_name, item_name)
         if item_name in self._flat_nest:
             result = _SeriesFromNest(self._flat_nest[item_name])
             # Assigning these properties directly in order to avoid any complication
@@ -193,6 +210,25 @@ def _subexprs_by_nest(parents: list, node) -> dict[str, list]:
     return result
 
 
+def _identify_aliases(expr: str) -> tuple[str, dict[str, str]]:
+    """
+    Given an expression string, identify backtick-quoted names
+    and replace them with cleaned names, returning the cleaned
+    expression and a dictionary of aliases, where the keys are
+    clean aliases to the original names.
+    """
+    aliases = {}
+
+    def sub_and_alias(match):
+        original = match.group(0)[1:-1]  # remove backticks
+        alias = clean_column_name(original)
+        if alias != original:
+            aliases[alias] = original
+        return alias
+
+    return _backtick_protected_names.sub(sub_and_alias, expr), aliases
+
+
 class NestedFrame(pd.DataFrame):
     """A Pandas Dataframe extension with support for nested structure.
 
@@ -205,6 +241,12 @@ class NestedFrame(pd.DataFrame):
     # Series produce instances of this class, preserving the type and origin.
     __pandas_priority__ = 4500
 
+    # The "_aliases" attribute is usually None or not even present, but when it is present,
+    # it indicates that an evaluation is in progress, and that columns and fields with names
+    # that are not identifier-like have been aliases to cleaned names, and this attribute
+    # contains those aliases, keyed by the cleaned name.
+    _metadata = ["_aliases"]
+
     @property
     def _constructor(self) -> Self:  # type: ignore[name-defined] # noqa: F821
         return NestedFrame
@@ -232,62 +274,86 @@ class NestedFrame(pd.DataFrame):
                 nest_cols.append(column)
         return nest_cols
 
-    def _is_known_hierarchical_column(self, colname) -> bool:
+    def _parse_hierarchical_components(self, delimited_path: str, delimiter: str = ".") -> list[str]:
+        """
+        Given a string that may be a delimited path, parse it into its components,
+        respecting backticks that are used to protect component names that may contain the delimiter.
+        """
+        aliases = getattr(self, "_aliases", None)
+        if aliases is None:
+            delimited_path, aliases = _identify_aliases(delimited_path)
+        return [aliases.get(x, x) for x in delimited_path.split(delimiter)]
+
+    def _is_known_hierarchical_column(self, components: list[str] | str) -> bool:
         """Determine whether a string is a known hierarchical column name"""
-        if "." in colname:
-            base_name = colname.split(".")[0]
-            if base_name in self.nested_columns:
-                # TODO: only handles one level of nesting for now
-                nested_name = ".".join(colname.split(".")[1:])
-                return nested_name in self.all_columns[base_name]
+        if isinstance(components, str):
+            components = self._parse_hierarchical_components(components)
+        if len(components) < 2:
             return False
+        base_name = components[0]
+        if base_name in self.nested_columns:
+            nested_name = ".".join(components[1:])
+            return nested_name in self.all_columns[base_name]
         return False
 
-    def _is_known_column(self, colname) -> bool:
-        """Determine whether a string is a known column name"""
-        return colname in self.columns or self._is_known_hierarchical_column(colname)
+    def _is_known_column(self, components: list[str] | str) -> bool:
+        """Determine whether a list of field components describes a known column name"""
+        if isinstance(components, str):
+            components = self._parse_hierarchical_components(components)
+        if ".".join(components) in self.columns:
+            return True
+        return self._is_known_hierarchical_column(components)
 
     def __getitem__(self, item):
         """Adds custom __getitem__ behavior for nested columns"""
 
-        if isinstance(item, str):
-            # Preempt the nested check if the item is a base column
-            if item in self.columns:
-                return super().__getitem__(item)
-            # If a nested column name is passed, return a flat series for that column
-            # flat series is chosen over list series for utility
-            # e.g. native ability to do something like ndf["nested.a"] + 3
-            elif self._is_known_hierarchical_column(item):
-                # TODO: only handles one level of nesting for now
-                nested = item.split(".")[0]
-                col = ".".join(item.split(".")[1:])
-                return self[nested].nest.get_flat_series(col)
-            else:
-                raise KeyError(f"Column '{item}' not found in nested columns or base columns")
-        else:
+        if not isinstance(item, str):
             return super().__getitem__(item)
 
+        # Preempt the nested check if the item is a base column, with or without
+        # dots and backticks.
+        if item in self.columns:
+            return super().__getitem__(item)
+        components = self._parse_hierarchical_components(item)
+        # One more check on the entirety of the item name, in case backticks were used
+        # (even if they weren't necessary).
+        cleaned_item = ".".join(components)
+        if cleaned_item in self.columns:
+            return super().__getitem__(cleaned_item)
+
+        # If a nested column name is passed, return a flat series for that column
+        # flat series is chosen over list series for utility
+        # e.g. native ability to do something like ndf["nested.a"] + 3
+        if self._is_known_hierarchical_column(components):
+            nested = components[0]
+            field = ".".join(components[1:])
+            return self[nested].nest.get_flat_series(field)
+        else:
+            raise KeyError(f"Column '{cleaned_item}' not found in nested columns or base columns")
+
     def __setitem__(self, key, value):
         """Adds custom __setitem__ behavior for nested columns"""
-
+        components = self._parse_hierarchical_components(key)
         # Replacing or adding columns to a nested structure
         # Allows statements like ndf["nested.t"] = ndf["nested.t"] - 5
         # Or ndf["nested.base_t"] = ndf["nested.t"] - 5
         # Performance note: This requires building a new nested structure
         # TODO: Support assignment of a new column to an existing nested col from a list series
-        if self._is_known_hierarchical_column(key) or (
-            "." in key and key.split(".")[0] in self.nested_columns
+        if self._is_known_hierarchical_column(components) or (
+            len(components) > 1 and components[0] in self.nested_columns
         ):
-            nested, col = key.split(".")
-            new_nested_series = self[nested].nest.with_flat_field(col, value)
+            if len(components) != 2:
+                raise ValueError(f"Only one level of nesting is supported; given {key}")
+            nested, field = components
+            new_nested_series = self[nested].nest.with_flat_field(field, value)
             return super().__setitem__(nested, new_nested_series)
 
         # Adding a new nested structure from a column
         # Allows statements like ndf["new_nested.t"] = ndf["nested.t"] - 5
-        if "." in key:
-            new_nested, col = key.split(".")
+        if len(components) > 1:
+            new_nested, field = components
             if isinstance(value, pd.Series):
-                value.name = col
+                value.name = field
                 value = value.to_frame()
             new_df = self.add_nested(value, name=new_nested)
             self._update_inplace(new_df)
@@ -512,10 +578,15 @@ class NestedFrame(pd.DataFrame):
         --------
         https://pandas.pydata.org/docs/reference/api/pandas.eval.html
         """
+        _, aliases = _identify_aliases(expr)
+        self._aliases: dict[str, str] | None = aliases
+
         kwargs["resolvers"] = tuple(kwargs.get("resolvers", ())) + (_NestResolver(self),)
         kwargs["inplace"] = inplace
         kwargs["parser"] = "nested-pandas"
-        return super().eval(expr, **kwargs)
+        answer = super().eval(expr, **kwargs)
+        self._aliases = None
+        return answer
 
     def extract_nest_names(
         self,
@@ -801,7 +872,7 @@ class NestedFrame(pd.DataFrame):
         Takes a function and applies it to each top-level row of the NestedFrame.
 
         The user may specify which columns the function is applied to, with
-        columns from the 'base' layer being passsed to the function as
+        columns from the 'base' layer being passed to the function as
         scalars and columns from the nested layers being passed as numpy arrays.
 
         Parameters
@@ -838,12 +909,15 @@ class NestedFrame(pd.DataFrame):
         # Parse through the initial args to determine the columns to apply the function to
         requested_columns = []
         for arg in args:
-            if not isinstance(arg, str) or not self._is_known_column(arg):
-                # We've reached an argument that is not a valid column, so we assume
-                # the remaining args are extra arguments to the function
+            # Stop when we reach an argument that is not a valid column, as we assume
+            # that the remaining args are extra arguments to the function
+            if not isinstance(arg, str):
+                break
+            components = self._parse_hierarchical_components(arg)
+            if not self._is_known_column(components):
                 break
-            layer = "base" if "." not in arg else arg.split(".")[0]
-            col = arg.split(".")[-1]
+            layer = "base" if len(components) < 2 else components[0]
+            col = components[-1]
             requested_columns.append((layer, col))
 
         # We require the first *args to be the columns to apply the function to

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/src/nested_pandas/series/packer.py

@@ -196,10 +196,34 @@ def pack_lists(df: pd.DataFrame, name: str | None = None, *, validate: bool = Tr
     nested_pandas.series.dtype.NestedDtype : The dtype of the output series.
     nested_pandas.series.packer.pack_flat : Pack a "flat" dataframe with repeated indexes.
     """
-    struct_array = pa.StructArray.from_arrays(
-        [df[column] for column in df.columns],
-        names=df.columns,
-    )
+    # When series is converted to pa.array it may be both Array and ChunkedArray
+    # We convert it to chunked for the sake of consistency
+    pa_arrays_maybe_chunked = {column: pa.array(df[column]) for column in df.columns}
+    pa_chunked_arrays = {
+        column: arr if isinstance(arr, pa.ChunkedArray) else pa.chunked_array([arr])
+        for column, arr in pa_arrays_maybe_chunked.items()
+    }
+
+    # If all chunk arrays have the same chunk lengths, we can build a chunked struct array with no
+    # data copying.
+    chunk_lengths = pa.array([[len(chunk) for chunk in arr.chunks] for arr in pa_chunked_arrays.values()])
+    if all(chunk_length == chunk_lengths[0] for chunk_length in chunk_lengths):
+        chunks = []
+        numpy_chunks = next(iter(pa_chunked_arrays.values())).num_chunks
+        for i in range(numpy_chunks):
+            chunks.append(
+                pa.StructArray.from_arrays(
+                    [arr.chunk(i) for arr in pa_chunked_arrays.values()],
+                    names=pa_chunked_arrays.keys(),
+                )
+            )
+        struct_array = pa.chunked_array(chunks)
+    else:  # "flatten" the chunked arrays
+        struct_array = pa.StructArray.from_arrays(
+            [arr.combine_chunks() for arr in pa_chunked_arrays.values()],
+            names=pa_chunked_arrays.keys(),
+        )
+
     ext_array = NestedExtensionArray(struct_array, validate=validate)
     return pd.Series(
         ext_array,

{nested_pandas-0.3.2 → nested_pandas-0.3.3/src/nested_pandas.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nested-pandas
-Version: 0.3.2
+Version: 0.3.3
 Author-email: LINCC Frameworks <brantd@uw.edu>
 License: MIT License
         

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/src/nested_pandas.egg-info/SOURCES.txt

@@ -14,6 +14,8 @@ pyproject.toml
 .github/ISSUE_TEMPLATE/0-general_issue.md
 .github/ISSUE_TEMPLATE/1-bug_report.md
 .github/ISSUE_TEMPLATE/2-feature_request.md
+.github/ISSUE_TEMPLATE/README.md
+.github/workflows/README.md
 .github/workflows/asv-main.yml
 .github/workflows/asv-nightly.yml
 .github/workflows/asv-pr.yml

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/tests/nested_pandas/nestedframe/test_nestedframe.py

@@ -85,6 +85,35 @@ def test_is_known_hierarchical_column():
     assert not base._is_known_hierarchical_column("base.a")
 
 
+def test_is_known_column():
+    """
+    Test that known (non-hierarchical) columns can be identified.  The key
+    point to test is that columns which might look like they are nested,
+    but which are already known to not be, are correctly identified.
+    """
+    base = NestedFrame(data={"R. A.": [1, 2, 3], "nested.b": [2, 4, 6]}, index=[0, 1, 2])
+    nested = pd.DataFrame(
+        data={"c": [0, 2, 4, 1, 4, 3, 1, 4, 1], "d": [5, 4, 7, 5, 3, 1, 9, 3, 4]},
+        index=[0, 0, 0, 1, 1, 1, 2, 2, 2],
+    )
+
+    base = base.add_nested(nested, "nested")
+
+    assert base._is_known_column("R. A.")
+    assert base._is_known_column("`R. A.`")
+    assert base._is_known_column("nested.b")
+
+    # In this context, the "." delimiter matters a lot, so the following, which would be
+    # acceptable in an .eval() context, is not acceptable here.
+    assert not base._is_known_column("nested . b")
+    assert not base._is_known_column("nested. c")
+    assert not base._is_known_column("nested  .d")
+
+    # But hierarchical ones should also work
+    assert base._is_known_column("nested.c")
+    assert base._is_known_column("nested.d")
+
+
 def test_get_nested_column():
     """Test that __getitem__ can retrieve a nested column"""
 
@@ -167,6 +196,21 @@ def test_get_dot_names():
     assert len(nf["nested.R.A."]) == 4
 
 
+def test_nesting_limit():
+    """Test the ability to prevent nesting beyond a depth of 1."""
+    nf = NestedFrame.from_flat(
+        NestedFrame({"a": [1, 2, 3, 4], ".b.": [1, 1, 3, 3], "R.A.": [3, None, 6, 5]}, index=[1, 1, 2, 2]),
+        base_columns=[".b."],
+    )
+    with pytest.raises(ValueError):
+        # The error gets triggered for the attempt to create new nested columns; if the column has
+        # already been created, it should be fine.
+        nf["nested.c.d.e"] = nf[".b."]
+    nf["nested.c"] = nf["nested.R.A."]
+    # Test that the above works with backticks, too, even in cases where they are not strictly necessary.
+    nf["`nested.d`"] = nf["`.b.`"]
+
+
 def test_add_nested_with_flat_df():
     """Test that add_nested correctly adds a nested column to the base df"""
 
@@ -612,6 +656,17 @@ def test_query_on_non_identifier_columns():
     nf3 = nf.query("`bad dog`.a > 2")
     assert nf3["bad dog"].nest["a"].size == 4
 
+    # And also for fields within the nested columns.
+    # Taken from GH#176
+    nf = NestedFrame(data={"dog": [1, 2, 3], "good dog": [2, 4, 6]}, index=[0, 1, 2])
+    nested = pd.DataFrame(
+        data={"n/a": [0, 2, 4, 1, 4, 3, 1, 4, 1], "n/b": [5, 4, 7, 5, 3, 1, 9, 3, 4]},
+        index=[0, 0, 0, 1, 1, 1, 2, 2, 2],
+    )
+    nf = nf.add_nested(nested, "bad dog")
+    nf4 = nf.query("`bad dog`.`n/a` > 2")
+    assert nf4["bad dog"].nest["n/a"].size == 4
+
 
 def test_dropna():
     """Test that dropna works on all layers"""
@@ -815,6 +870,14 @@ def test_reduce():
     for i in range(len(result)):
         assert result["offset_avg"].values[i] == expected_offset_avg[i]
 
+    # Verify that we can understand a string argument to the reduce function,
+    # so long as it isn't a column name.
+    def make_id(col1, prefix_str):
+        return f"{prefix_str}{col1}"
+
+    result = nf.reduce(make_id, "b", "some_id_")
+    assert result[0][1] == "some_id_4"
+
 
 def test_reduce_duplicated_cols():
     """Tests nf.reduce() to correctly handle duplicated column names."""
@@ -1034,6 +1097,20 @@ def test_eval_assignment():
     assert (nf["p2.e"] == nf["packed.d"] * 2 + nf.c).all()
     assert (nf["p2.f"] == nf["p2.e"] + nf.b).all()
 
+    # Verify that assignment can be done to nested columns and fields
+    # having names which are not valid Python identifiers, and must
+    # be quoted with backticks.
+    nf = NestedFrame(data={"dog": [1, 2, 3], "good dog": [2, 4, 6]}, index=[0, 1, 2])
+    nested = pd.DataFrame(
+        data={"n/a": [0, 2, 4, 1, 4, 3, 1, 4, 1], "n/b": [5, 4, 7, 5, 3, 1, 9, 3, 4]},
+        index=[0, 0, 0, 1, 1, 1, 2, 2, 2],
+    )
+    nf = nf.add_nested(nested, "bad dog")
+    nfx = nf.eval("`bad dog`.`n/c` = `bad dog`.`n/b` + 2.5")
+    # The number of columns at the top should not have changed
+    assert len(nfx.columns) == len(nf.columns)
+    assert (nfx["bad dog"].nest["n/c"] == nf["bad dog"].nest["n/b"] + 2.5).all()
+
 
 def test_access_non_existing_column():
     """Test that accessing a non-existing column raises a KeyError"""

{nested_pandas-0.3.2 → nested_pandas-0.3.3}/tests/nested_pandas/series/test_packer.py

@@ -286,6 +286,42 @@ def test_pack_lists():
         assert_series_equal(series.nest.get_list_series(field_name), packed_df[field_name])
 
 
+def test_pack_lists_with_chunked_arrays():
+    """Issue https://github.com/lincc-frameworks/nested-pandas/issues/180"""
+    chunked_a = pd.Series(
+        pa.chunked_array([pa.array([[1, 2, 3], [4, 5]])] * 3),
+        dtype=pd.ArrowDtype(pa.list_(pa.int64())),
+        name="a",
+    )
+    chunked_b = pd.Series(
+        pa.chunked_array([pa.array([[0.0, 1.0, 2.0], [3.0, 4.0]])] * 3),
+        dtype=pd.ArrowDtype(pa.list_(pa.float64())),
+        name="b",
+    )
+    list_df = pd.DataFrame({"a": chunked_a, "b": chunked_b}, index=[0, 1, 2, 3, 4, 5])
+    series = packer.pack_lists(list_df)
+    assert_series_equal(series.nest.get_list_series("a"), chunked_a)
+    assert_series_equal(series.nest.get_list_series("b"), chunked_b)
+
+
+def test_pack_lists_with_uneven_chunked_arrays():
+    """Issue https://github.com/lincc-frameworks/nested-pandas/issues/180"""
+    chunked_a = pd.Series(
+        pa.chunked_array([pa.array([[1, 2, 3], [4, 5]])] * 3),
+        dtype=pd.ArrowDtype(pa.list_(pa.int64())),
+        name="a",
+    )
+    chunked_b = pd.Series(
+        pa.array([[0.0, 1.0, 2.0], [3.0, 4.0]] * 3),
+        dtype=pd.ArrowDtype(pa.list_(pa.float64())),
+        name="b",
+    )
+    list_df = pd.DataFrame({"a": chunked_a, "b": chunked_b}, index=[0, 1, 2, 3, 4, 5])
+    series = packer.pack_lists(list_df)
+    assert_series_equal(series.nest.get_list_series("a"), chunked_a)
+    assert_series_equal(series.nest.get_list_series("b"), chunked_b)
+
+
 def test_pack_seq_with_dfs_and_index():
     """Test pack_seq()."""
     dfs = [