dataiter 1.3.1__tar.gz → 1.4__tar.gz

{dataiter-1.3.1 → dataiter-1.4}/.gitignore

@@ -1,6 +1,4 @@
-*.lprof
-*.prof
-.claude
+.claude/settings.local.json
 .env
 .envrc
 .pytest_cache
@@ -9,7 +7,6 @@ benchmark-head.py
 benchmark-versions.csv
 benchmark-versions.ods
 build
-dataiter.egg-info
 dist
 doc/_build
 doc/comparison/comparison.html

{dataiter-1.3.1 → dataiter-1.4}/PKG-INFO

@@ -1,21 +1,20 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.5
 Name: dataiter
-Version: 1.3.1
+Version: 1.4
 Summary: Simple, light-weight data frames for Python
 Project-URL: Homepage, https://github.com/otsaloma/dataiter
 Author-email: Osmo Salomaa <otsaloma@iki.fi>
-License: MIT
+License-Expression: MIT
 License-File: AUTHORS.md
 License-File: COPYING
 Requires-Python: >=3.11.0
 Requires-Dist: attd>=0.3
 Requires-Dist: numpy<3.0,>=2.4.0
-Requires-Dist: pyarrow>=2.0
+Requires-Dist: pyarrow>=16.0
 Requires-Dist: wcwidth>=0.1
 Description-Content-Type: text/markdown
 
-Simple, Light-Weight Data Frames for Python
-===========================================
+# Simple, Light-Weight Data Frames for Python
 
 [![PyPI](https://img.shields.io/pypi/v/dataiter.svg)](https://pypi.org/project/dataiter)
 [![Downloads](https://pepy.tech/badge/dataiter/month)](https://pepy.tech/project/dataiter)
@@ -81,6 +80,14 @@ the differences and similarities in common operations.
 
 https://dataiter.readthedocs.io/en/stable/comparison.html
 
+For coding agents and LLMs, each documentation page is also available as
+Markdown by replacing `.html` with `.md` in the URL. There's also an
+`llms.txt` index and complete documentation as `llms-full.txt`. See
+
+- https://dataiter.readthedocs.io/en/stable/data-frame.md
+- https://dataiter.readthedocs.io/en/stable/llms.txt
+- https://dataiter.readthedocs.io/en/stable/llms-full.txt
+
 ## Development
 
 To install a virtualenv for development, use

{dataiter-1.3.1 → dataiter-1.4}/README.md

@@ -1,5 +1,4 @@
-Simple, Light-Weight Data Frames for Python
-===========================================
+# Simple, Light-Weight Data Frames for Python
 
 [![PyPI](https://img.shields.io/pypi/v/dataiter.svg)](https://pypi.org/project/dataiter)
 [![Downloads](https://pepy.tech/badge/dataiter/month)](https://pepy.tech/project/dataiter)
@@ -65,6 +64,14 @@ the differences and similarities in common operations.
 
 https://dataiter.readthedocs.io/en/stable/comparison.html
 
+For coding agents and LLMs, each documentation page is also available as
+Markdown by replacing `.html` with `.md` in the URL. There's also an
+`llms.txt` index and complete documentation as `llms-full.txt`. See
+
+- https://dataiter.readthedocs.io/en/stable/data-frame.md
+- https://dataiter.readthedocs.io/en/stable/llms.txt
+- https://dataiter.readthedocs.io/en/stable/llms-full.txt
+
 ## Development
 
 To install a virtualenv for development, use

{dataiter-1.3.1 → dataiter-1.4}/dataiter/__init__.py

@@ -25,7 +25,7 @@ import numpy as np
 
 from dataiter import util
 
-__version__ = "1.3.1"
+__version__ = "1.4"
 
 DEFAULT_PEEK_ELEMENTS = 10
 DEFAULT_PEEK_ITEMS = 3
@@ -87,34 +87,34 @@ globals().pop("np", None)
 globals().pop("numba", None)
 globals().pop("util", None)
 
-from dataiter import dtypes # noqa
-from dataiter.vector import Vector # noqa
-from dataiter.data_frame import DataFrame # noqa
-from dataiter.data_frame import DataFrameColumn # noqa
-from dataiter.geojson import GeoJSON # noqa
-from dataiter.list_of_dicts import ListOfDicts # noqa
-from dataiter import dt # noqa
-from dataiter import regex # noqa
-
-from dataiter.aggregate import all # noqa
-from dataiter.aggregate import any # noqa
-from dataiter.aggregate import count # noqa
-from dataiter.aggregate import count_unique # noqa
-from dataiter.aggregate import first # noqa
-from dataiter.aggregate import last # noqa
-from dataiter.aggregate import max # noqa
-from dataiter.aggregate import mean # noqa
-from dataiter.aggregate import median # noqa
-from dataiter.aggregate import min # noqa
-from dataiter.aggregate import mode # noqa
-from dataiter.aggregate import nth # noqa
-from dataiter.aggregate import quantile # noqa
-from dataiter.aggregate import std # noqa
-from dataiter.aggregate import sum # noqa
-from dataiter.aggregate import var # noqa
-
-from dataiter.io import read_csv # noqa
-from dataiter.io import read_geojson # noqa
-from dataiter.io import read_json # noqa
-from dataiter.io import read_npz # noqa
-from dataiter.io import read_parquet # noqa
+from dataiter import dtypes
+from dataiter.vector import Vector
+from dataiter.data_frame import DataFrame
+from dataiter.data_frame import DataFrameColumn
+from dataiter.geojson import GeoJSON
+from dataiter.list_of_dicts import ListOfDicts
+from dataiter import dt
+from dataiter import regex
+
+from dataiter.aggregate import all
+from dataiter.aggregate import any
+from dataiter.aggregate import count
+from dataiter.aggregate import count_unique
+from dataiter.aggregate import first
+from dataiter.aggregate import last
+from dataiter.aggregate import max
+from dataiter.aggregate import mean
+from dataiter.aggregate import median
+from dataiter.aggregate import min
+from dataiter.aggregate import mode
+from dataiter.aggregate import nth
+from dataiter.aggregate import quantile
+from dataiter.aggregate import std
+from dataiter.aggregate import sum
+from dataiter.aggregate import var
+
+from dataiter.io import read_csv
+from dataiter.io import read_geojson
+from dataiter.io import read_json
+from dataiter.io import read_npz
+from dataiter.io import read_parquet

{dataiter-1.3.1 → dataiter-1.4}/dataiter/aggregate.py

@@ -242,7 +242,6 @@ def is_na_numba(x):
         na[i] = is_na_item_numba(x[i])
     return na
 
-@composite
 def last(x, *, drop_na=False):
     """
     Return the last element of `x`.

{dataiter-1.3.1 → dataiter-1.4}/dataiter/data_frame.py

@@ -22,7 +22,6 @@
 
 import codecs
 import dataiter
-import functools
 import itertools
 import json
 import numpy as np
@@ -103,10 +102,6 @@ class DataFrame(dict):
     # List of names that are actual attributes, not columns
     ATTRIBUTES = ["colnames", "_group_colnames"]
 
-    # Use dummy attributes corresponding to dictionary keys so that
-    # Tab completion of column names at a Python shell would work.
-    COLUMN_PLACEHOLDER = type("COLUMN_PLACEHOLDER", (), {})
-
     def __init__(self, *args, **kwargs):
         """
         Return a new data frame.
@@ -122,9 +117,6 @@ class DataFrame(dict):
                 value.nrow == nrow): continue
             column = DataFrameColumn(value, nrow=nrow)
             super().__setitem__(key, column)
-        for key in self:
-            if not self.__hasattr(key) and key.isidentifier():
-                super().__setattr__(key, self.COLUMN_PLACEHOLDER)
         # Check that we have a uniform table.
         self._check_dimensions()
         self._group_colnames = ()
@@ -140,12 +132,13 @@ class DataFrame(dict):
             return self.__delitem__(name)
         return super().__delattr__(name)
 
-    def __delitem__(self, key):
-        # Note that this is not called for some methods,
-        # at least pop, popitem and clear.
-        if self[key] is self.COLUMN_PLACEHOLDER:
-            super().__delattr__(key)
-        return super().__delitem__(key)
+    def __dir__(self):
+        # Used for tab completion of column names at a Python shell.
+        names = super().__dir__()
+        return names + [x for x in self if
+                        isinstance(x, str) and
+                        x.isidentifier() and
+                        x not in names]
 
     def __eq__(self, other):
         return (isinstance(other, DataFrame) and
@@ -159,27 +152,6 @@ class DataFrame(dict):
             return self.__getitem__(name)
         raise AttributeError(name)
 
-    def __getattribute__(self, name):
-        value = super().__getattribute__(name)
-        if name == "COLUMN_PLACEHOLDER":
-            return value
-        if value is self.COLUMN_PLACEHOLDER and name in self:
-            return self[name]
-        return value
-
-    def __hasattr(self, name):
-        # Return True if attribute exists and is not a column.
-        return hasattr(self, name) and not isinstance(getattr(self, name), DataFrameColumn)
-
-    @classmethod
-    def __is_builtin_attr(cls, name):
-        return name in cls.__list_builtin_attrs()
-
-    @classmethod
-    @functools.lru_cache(None)
-    def __list_builtin_attrs(cls):
-        return set(dir(cls()))
-
     def __setattr__(self, name, value):
         if name in self.ATTRIBUTES:
             return super().__setattr__(name, value)
@@ -187,8 +159,6 @@ class DataFrame(dict):
 
     def __setitem__(self, key, value):
         value = self._reconcile_column(value)
-        if not self.__hasattr(key) and key.isidentifier():
-            super().__setattr__(key, self.COLUMN_PLACEHOLDER)
         return super().__setitem__(key, value)
 
     def __repr__(self):
@@ -220,13 +190,12 @@ class DataFrame(dict):
         group_colnames = self._group_colnames
         data = self.sort(**dict.fromkeys(group_colnames, 1))
         data._index_ = np.arange(data.nrow)
-        stat = data.unique(*group_colnames).select("_index_", *group_colnames)
-        indices = np.split(data._index_, stat._index_[1:])
+        starts = data._get_group_starts(*group_colnames)
+        stat = data.select("_index_", *group_colnames).slice(starts)
         group_aware = [getattr(x, "group_aware", False) for x in colname_function_pairs.values()]
         if any(group_aware):
-            groups = Vector.fast(range(len(indices)), int)
-            n = Vector.fast(map(len, indices), int)
-            data._group_ = np.repeat(groups, n)
+            n = np.diff(starts, append=data.nrow)
+            data._group_ = np.repeat(np.arange(len(starts)), n)
         slices = None
         for colname, function in colname_function_pairs.items():
             if getattr(function, "group_aware", False):
@@ -246,6 +215,7 @@ class DataFrame(dict):
                 # what special values to expect and thus we end up
                 # needing to use the slow Vector.__init__.
                 if slices is None:
+                    indices = np.split(data._index_, starts[1:])
                     slices = [data._view_rows(x) for x in indices]
                 stat[colname] = [function(x) for x in slices]
         return stat.unselect("_index_", "_group_")
@@ -589,6 +559,16 @@ class DataFrame(dict):
                 ba[item[0]] = ba.pop(item[1])
         return ab.rbind(ba).sort(_aid_=1, _bid_=1).unselect("_aid_", "_bid_")
 
+    def _get_group_starts(self, *colnames):
+        # Assumes data is sorted by colnames, i.e. groups are contiguous.
+        change = np.zeros(self.nrow, bool)
+        change[:1] = True
+        for colname in colnames:
+            column = self[colname]
+            na = column.is_na()
+            change[1:] |= (column[1:] != column[:-1]) & ~(na[1:] & na[:-1])
+        return np.flatnonzero(change)
+
     def _get_join_indices(self, other, by1, by2):
         other_ids = list(zip(*[other[x] for x in by2]))
         other_by_id = {other_ids[i]: i for i in range(other.nrow)}
@@ -688,7 +668,7 @@ class DataFrame(dict):
         vectorized form.
 
         >>> data = di.read_csv("data/listings-reviews.csv")
-        >>> data.map(lambda x, i: (x.reviews[i], x.rating[i]))
+        >>> data.map(lambda x, i: (x.reviews[i], x.rating[i]))[:3]
         """
         return [function(self, i) for i in range(self.nrow)]
 
@@ -844,22 +824,6 @@ class DataFrame(dict):
             wide = wide.left_join(part, *ids)
         return wide
 
-    def pop(self, key, *args, **kwargs):
-        """"""
-        value = super().pop(key, *args, **kwargs)
-        if hasattr(self, key):
-            if not self.__is_builtin_attr(key):
-                super().__delattr__(key)
-        return value
-
-    def popitem(self):
-        """"""
-        key, value = super().popitem()
-        if hasattr(self, key):
-            if not self.__is_builtin_attr(key):
-                super().__delattr__(key)
-        return key, value
-
     def print_(self, *, max_rows=None, max_width=None, truncate_width=None):
         """
         Print data frame to ``sys.stdout``.
@@ -1066,7 +1030,7 @@ class DataFrame(dict):
         >>> listings.semi_join(reviews, "id")
         """
         by1, by2 = self._split_join_by(*by)
-        other = other.unique(*by2)
+        other = other.drop_na(*by2).unique(*by2)
         found, src = self._get_join_indices(other, by1, by2)
         for colname, column in self.items():
             yield colname, column[found].copy()
@@ -1129,18 +1093,8 @@ class DataFrame(dict):
             column = self[colname]
             if column._is_string_fixed():
                 column[column.is_na()] = "\uffff"
-            if dir > 0 and any((
-                column._is_string_fixed(),
-                column.is_boolean(),
-                column.is_bytes(),
-                column.is_datetime(),
-                column.is_float(),
-                column.is_integer(),
-                # Segfaults on NumPy < 2.2.1!
-                # https://github.com/numpy/numpy/issues/27984
-                column.is_string(),
-                column.is_timedelta(),
-            )): return column
+            if dir > 0 and not column.is_object():
+                return column
             if not column.is_number():
                 column = column.rank(method="min")
             return column if dir > 0 else -column
@@ -1159,9 +1113,8 @@ class DataFrame(dict):
         data = self.select(*by)
         data._index_ = np.arange(data.nrow)
         data = data.sort(**dict.fromkeys(by, 1))
-        data._sorted_index_ = np.arange(data.nrow)
-        stat = data.unique(*by)
-        return np.split(data._index_, stat._sorted_index_[1:])
+        starts = data._get_group_starts(*by)
+        return np.split(data._index_, starts[1:])
 
     def _split_join_by(self, *by):
         by1 = [x if isinstance(x, str) else x[0] for x in by]
@@ -1277,28 +1230,44 @@ class DataFrame(dict):
         >>> data = di.read_csv("data/listings.csv")
         >>> data.unique("hood")
         """
-        # Strangely, standard Python is a lot faster here than np.unique
-        # with all the extra needed across columns of different type.
-        # We just need to avoid NaN and NaT for the 'in' checks to work.
         colnames = colnames or self.colnames
         columns = [self[x] for x in colnames]
+        if all(x.is_number() or
+               x.is_boolean() or
+               x.is_datetime() or
+               x.is_timedelta() for x in columns):
+            # np.lexsort is stable, so the first row of each group
+            # in sorted order is the first occurrence in self.
+            order = np.lexsort(columns[::-1])
+            starts = self.select(*colnames).slice(order)._get_group_starts(*colnames)
+            keep = np.sort(order[starts])
+            for colname, column in self.items():
+                yield colname, column[keep].copy()
+            return
+        # For strings and objects, a Python set is faster than sorting.
+        # We just need to avoid NaN and NaT for the 'in' checks to work.
         for i, column in enumerate(columns):
             if column.is_datetime():
                 flag = np.nanmin(column) - np.timedelta64(1, "D")
                 if np.isnat(flag):
                     flag = np.datetime64("-001-01-01")
                 columns[i] = column.replace_na(flag)
+            if column.is_timedelta():
+                unit = np.datetime_data(column.dtype)[0]
+                flag = np.nanmin(column) - np.timedelta64(1, unit)
+                if np.isnat(flag):
+                    flag = np.timedelta64(0, unit)
+                columns[i] = column.replace_na(flag)
             if column.is_float():
                 flag = np.nanmin(column) - 1
                 if np.isnan(flag):
                     flag = -1
                 columns[i] = column.replace_na(flag)
-        rows = list(zip(*columns))
         seen = set()
         keep = []
-        for i in range(self.nrow):
-            if rows[i] not in seen:
-                seen.add(rows[i])
+        for i, row in enumerate(zip(*columns)):
+            if row not in seen:
+                seen.add(row)
                 keep.append(i)
         for colname, column in self.items():
             yield colname, column[keep].copy()

{dataiter-1.3.1 → dataiter-1.4}/dataiter/io.py

@@ -51,7 +51,7 @@ def read_npz(path, *, allow_pickle=True):
     return DataFrame.read_npz(path, allow_pickle=allow_pickle)
 
 def read_parquet(path, *, columns=[], dtypes={}):
-    return DataFrame.read_parquet(path, columns=[], dtypes={})
+    return DataFrame.read_parquet(path, columns=columns, dtypes=dtypes)
 
 read_csv.__doc__ = util.format_alias_doc(read_csv, DataFrame.read_csv)
 read_geojson.__doc__ = util.format_alias_doc(read_geojson, GeoJSON.read)

{dataiter-1.3.1 → dataiter-1.4}/dataiter/list_of_dicts.py

@@ -479,7 +479,7 @@ class ListOfDicts(list):
         list of whatever `function` returns.
 
         >>> data = di.read_json("data/listings.json")
-        >>> data.map(lambda x: (x.guests, x.price))
+        >>> data.map(lambda x: (x.guests, x.price))[:3]
         """
         new = list(map(function, self))
         coerce = all(isinstance(x, dict) for x in new)

{dataiter-1.3.1 → dataiter-1.4}/dataiter/regex.py

@@ -28,12 +28,16 @@ from dataiter import util
 from dataiter import Vector
 from numpy.dtypes import StringDType
 
-def _prep(string, dtype, default):
+def _apply(function, string, dtype=object, na_value=None):
+    if util.is_scalar(string):
+        return function(string)
     assert isinstance(string, np.ndarray)
     assert isinstance(string.dtype, StringDType)
-    out = np.full_like(string, default, dtype)
+    out = np.full_like(string, na_value, dtype)
     na = string == dtypes.string.na_object
-    return out, na
+    for i in np.flatnonzero(~na):
+        out[i] = function(string[i])
+    return Vector.fast(out, dtype)
 
 def findall(pattern, string, flags=0):
     """
@@ -44,12 +48,8 @@ def findall(pattern, string, flags=0):
     >>> x = di.Vector(["asdf", "1234"])
     >>> regex.findall(r"[a-z]", x)
     """
-    if util.is_scalar(string):
-        return re.findall(pattern, string, flags=flags)
-    out, na = _prep(string, object, None)
-    for i in np.flatnonzero(~na):
-        out[i] = re.findall(pattern, string[i], flags=flags)
-    return Vector.fast(out, object)
+    f = lambda x: re.findall(pattern, x, flags=flags)
+    return _apply(f, string)
 
 def fullmatch(pattern, string, flags=0):
     """
@@ -60,12 +60,8 @@ def fullmatch(pattern, string, flags=0):
     >>> x = di.Vector(["asdf", "1234"])
     >>> regex.fullmatch(r"[a-z]+", x)
     """
-    if util.is_scalar(string):
-        return re.fullmatch(pattern, string, flags=flags)
-    out, na = _prep(string, object, None)
-    for i in np.flatnonzero(~na):
-        out[i] = re.fullmatch(pattern, string[i], flags=flags)
-    return Vector.fast(out, object)
+    f = lambda x: re.fullmatch(pattern, x, flags=flags)
+    return _apply(f, string)
 
 def match(pattern, string, flags=0):
     """
@@ -76,12 +72,8 @@ def match(pattern, string, flags=0):
     >>> x = di.Vector(["asdf", "1234"])
     >>> regex.match(r"[a-z]", x)
     """
-    if util.is_scalar(string):
-        return re.match(pattern, string, flags=flags)
-    out, na = _prep(string, object, None)
-    for i in np.flatnonzero(~na):
-        out[i] = re.match(pattern, string[i], flags=flags)
-    return Vector.fast(out, object)
+    f = lambda x: re.match(pattern, x, flags=flags)
+    return _apply(f, string)
 
 def search(pattern, string, flags=0):
     """
@@ -92,12 +84,8 @@ def search(pattern, string, flags=0):
     >>> x = di.Vector(["asdf", "1234"])
     >>> regex.search(r"[a-z]", x)
     """
-    if util.is_scalar(string):
-        return re.search(pattern, string, flags=flags)
-    out, na = _prep(string, object, None)
-    for i in np.flatnonzero(~na):
-        out[i] = re.search(pattern, string[i], flags=flags)
-    return Vector.fast(out, object)
+    f = lambda x: re.search(pattern, x, flags=flags)
+    return _apply(f, string)
 
 def split(pattern, string, maxsplit=0, flags=0):
     """
@@ -108,12 +96,8 @@ def split(pattern, string, maxsplit=0, flags=0):
     >>> x = di.Vector(["one two three", "four"])
     >>> regex.split(r" +", x)
     """
-    if util.is_scalar(string):
-        return re.split(pattern, string, maxsplit=maxsplit, flags=flags)
-    out, na = _prep(string, object, None)
-    for i in np.flatnonzero(~na):
-        out[i] = re.split(pattern, string[i], maxsplit=maxsplit, flags=flags)
-    return Vector.fast(out, object)
+    f = lambda x: re.split(pattern, x, maxsplit=maxsplit, flags=flags)
+    return _apply(f, string)
 
 def sub(pattern, repl, string, count=0, flags=0):
     """
@@ -124,12 +108,8 @@ def sub(pattern, repl, string, count=0, flags=0):
     >>> x = di.Vector(["great", "fantastic"])
     >>> regex.sub(r"$", r"!", x)
     """
-    if util.is_scalar(string):
-        return re.sub(pattern, repl, string, count=count, flags=flags)
-    out, na = _prep(string, dtypes.string, dtypes.string.na_object)
-    for i in np.flatnonzero(~na):
-        out[i] = re.sub(pattern, repl, string[i], count=count, flags=flags)
-    return Vector.fast(out, str)
+    f = lambda x: re.sub(pattern, repl, x, count=count, flags=flags)
+    return _apply(f, string, dtypes.string, dtypes.string.na_object)
 
 def subn(pattern, repl, string, count=0, flags=0):
     """
@@ -140,9 +120,5 @@ def subn(pattern, repl, string, count=0, flags=0):
     >>> x = di.Vector(["great", "fantastic"])
     >>> regex.subn(r"$", r"!", x)
     """
-    if util.is_scalar(string):
-        return re.subn(pattern, repl, string, count=count, flags=flags)
-    out, na = _prep(string, object, None)
-    for i in np.flatnonzero(~na):
-        out[i] = re.subn(pattern, repl, string[i], count=count, flags=flags)
-    return Vector.fast(out, object)
+    f = lambda x: re.subn(pattern, repl, x, count=count, flags=flags)
+    return _apply(f, string)

{dataiter-1.3.1 → dataiter-1.4}/dataiter/test/test_aggregate.py

@@ -57,7 +57,7 @@ D6 = D1 + datetime.timedelta(days=5)
 D7 = D1 + datetime.timedelta(days=6)
 
 NaN = np.nan
-NaT = np.datetime64("NaT")
+NaT = np.datetime64("NaT", "D")
 
 EMPTY_VECTOR = Vector([], float)
 GROUPS = [1, 1, 2, 2, 3, 3, 4, 4, 5, 5]

{dataiter-1.3.1 → dataiter-1.4}/dataiter/test/test_data_frame.py

@@ -23,7 +23,6 @@
 import dataiter as di
 import numpy as np
 import pytest
-import tempfile
 
 from dataiter import DataFrame
 from dataiter import DataFrameColumn
@@ -112,12 +111,14 @@ class TestDataFrame:
         data = test.data_frame("vehicles.csv")
         assert data.make is data["make"]
 
-    def test___hasattr(self):
-        data = DataFrame(a=[1, 2, 3])
-        data._DataFrame__hasattr("left_join")
-        data._DataFrame__hasattr("items")
-        assert not data._DataFrame__hasattr("a")
-        assert not data._DataFrame__hasattr("xxx")
+    def test___dir__(self):
+        data = DataFrame(a=[1, 2, 3], items=[1, 2, 3])
+        data["b c"] = 1
+        names = dir(data)
+        assert "a" in names
+        assert "b c" not in names
+        assert names.count("items") == 1
+        assert "left_join" in names
 
     def test___setattr__(self):
         data = test.data_frame("vehicles.csv")
@@ -482,7 +483,7 @@ class TestDataFrame:
         assert data.make.is_object()
         assert data.model.is_object()
 
-    def test_read_csv_missing_blank(self):
+    def test_read_csv_missing_blank(self, tmp_path):
         text = """
 int,float,string,bool,date
 ,2.5,"test",TRUE,2025-01-01
@@ -491,14 +492,14 @@ int,float,string,bool,date
 1,2.5,"test",,2025-01-01
 1,2.5,"test",TRUE,
 """.strip()
-        handle, path = tempfile.mkstemp(".csv")
+        path = str(tmp_path / "test.csv")
         Path(path).write_text(text, "utf-8")
         data = DataFrame.read_csv(path)
         for i, column in enumerate(data.columns):
             assert column.is_na().sum() == 1
             assert column.is_na()[i]
 
-    def test_read_csv_missing_excel(self):
+    def test_read_csv_missing_excel(self, tmp_path):
         text = """
 int,float,string,bool,date
 #N/A,2.5,"test",TRUE,2025-01-01
@@ -507,14 +508,14 @@ int,float,string,bool,date
 1,2.5,"test",#N/A,2025-01-01
 1,2.5,"test",TRUE,#N/A
 """.strip()
-        handle, path = tempfile.mkstemp(".csv")
+        path = str(tmp_path / "test.csv")
         Path(path).write_text(text, "utf-8")
         data = DataFrame.read_csv(path)
         for i, column in enumerate(data.columns):
             assert column.is_na().sum() == 1
             assert column.is_na()[i]
 
-    def test_read_csv_missing_na(self):
+    def test_read_csv_missing_na(self, tmp_path):
         text = """
 int,float,string,bool,date
 NA,2.5,"test",TRUE,2025-01-01
@@ -523,14 +524,14 @@ NA,2.5,"test",TRUE,2025-01-01
 1,2.5,"test",NA,2025-01-01
 1,2.5,"test",TRUE,NA
 """.strip()
-        handle, path = tempfile.mkstemp(".csv")
+        path = str(tmp_path / "test.csv")
         Path(path).write_text(text, "utf-8")
         data = DataFrame.read_csv(path)
         for i, column in enumerate(data.columns):
             assert column.is_na().sum() == 1
             assert column.is_na()[i]
 
-    def test_read_csv_missing_null(self):
+    def test_read_csv_missing_null(self, tmp_path):
         text = """
 int,float,string,bool,date
 NULL,2.5,"test",TRUE,2025-01-01
@@ -539,7 +540,7 @@ NULL,2.5,"test",TRUE,2025-01-01
 1,2.5,"test",NULL,2025-01-01
 1,2.5,"test",TRUE,NULL
 """.strip()
-        handle, path = tempfile.mkstemp(".csv")
+        path = str(tmp_path / "test.csv")
         Path(path).write_text(text, "utf-8")
         data = DataFrame.read_csv(path)
         for i, column in enumerate(data.columns):
@@ -568,37 +569,37 @@ NULL,2.5,"test",TRUE,2025-01-01
         DataFrame.read_json(test.get_data_path("vehicles.json"))
 
     @pytest.mark.filterwarnings(IGNORE_NPZ_PICKLE_WARNING)
-    def test_read_npz(self):
+    def test_read_npz(self, tmp_path):
         orig = test.data_frame("vehicles.csv")
-        handle, path = tempfile.mkstemp(".npz")
+        path = str(tmp_path / "test.npz")
         orig.write_npz(path)
         data = DataFrame.read_npz(path)
         assert data == orig
 
     @pytest.mark.filterwarnings(IGNORE_NPZ_PICKLE_WARNING)
-    def test_read_npz_path(self):
+    def test_read_npz_path(self, tmp_path):
         orig = test.data_frame("vehicles.csv")
-        handle, path = tempfile.mkstemp(".npz")
+        path = str(tmp_path / "test.npz")
         orig.write_npz(path)
         DataFrame.read_npz(Path(path))
 
-    def test_read_parquet(self):
+    def test_read_parquet(self, tmp_path):
         orig = test.data_frame("vehicles.csv")
-        handle, path = tempfile.mkstemp(".parquet")
+        path = str(tmp_path / "test.parquet")
         orig.write_parquet(path)
         data = DataFrame.read_parquet(path)
         assert data == orig
 
-    def test_read_pickle(self):
+    def test_read_pickle(self, tmp_path):
         orig = test.data_frame("vehicles.csv")
-        handle, path = tempfile.mkstemp(".pkl")
+        path = str(tmp_path / "test.pkl")
         orig.write_pickle(path)
         data = DataFrame.read_pickle(path)
         assert data == orig
 
-    def test_read_pickle_path(self):
+    def test_read_pickle_path(self, tmp_path):
         orig = test.data_frame("vehicles.csv")
-        handle, path = tempfile.mkstemp(".pkl")
+        path = str(tmp_path / "test.pkl")
         orig.write_pickle(path)
         DataFrame.read_pickle(Path(path))
 
@@ -785,59 +786,59 @@ NULL,2.5,"test",TRUE,2025-01-01
         assert np.all(data.make == "Talbot")
         assert np.all(data.test == 1)
 
-    def test_write_csv(self):
+    def test_write_csv(self, tmp_path):
         orig = test.data_frame("vehicles.csv")
-        handle, path = tempfile.mkstemp(".csv")
+        path = str(tmp_path / "test.csv")
         orig.write_csv(path)
         data = DataFrame.read_csv(path)
         assert data == orig
 
-    def test_write_csv_path(self):
+    def test_write_csv_path(self, tmp_path):
         orig = test.data_frame("vehicles.csv")
-        handle, path = tempfile.mkstemp(".csv")
+        path = str(tmp_path / "test.csv")
         orig.write_csv(Path(path))
 
-    def test_write_json(self):
+    def test_write_json(self, tmp_path):
         orig = test.data_frame("downloads.json")
-        handle, path = tempfile.mkstemp(".json")
+        path = str(tmp_path / "test.json")
         orig.write_json(path)
         data = DataFrame.read_json(path)
         assert data == orig
 
-    def test_write_json_path(self):
+    def test_write_json_path(self, tmp_path):
         orig = test.data_frame("downloads.json")
-        handle, path = tempfile.mkstemp(".json")
+        path = str(tmp_path / "test.json")
         orig.write_json(Path(path))
 
     @pytest.mark.filterwarnings(IGNORE_NPZ_PICKLE_WARNING)
-    def test_write_npz(self):
+    def test_write_npz(self, tmp_path):
         orig = test.data_frame("vehicles.csv")
-        handle, path = tempfile.mkstemp(".npz")
+        path = str(tmp_path / "test.npz")
         orig.write_npz(path)
         data = DataFrame.read_npz(path)
         assert data == orig
 
     @pytest.mark.filterwarnings(IGNORE_NPZ_PICKLE_WARNING)
-    def test_write_npz_path(self):
+    def test_write_npz_path(self, tmp_path):
         orig = test.data_frame("vehicles.csv")
-        handle, path = tempfile.mkstemp(".npz")
+        path = str(tmp_path / "test.npz")
         orig.write_npz(Path(path))
 
-    def test_write_parquet(self):
+    def test_write_parquet(self, tmp_path):
         orig = test.data_frame("vehicles.csv")
-        handle, path = tempfile.mkstemp(".parquet")
+        path = str(tmp_path / "test.parquet")
         orig.write_parquet(path)
         data = DataFrame.read_parquet(path)
         assert data == orig
 
-    def test_write_pickle(self):
+    def test_write_pickle(self, tmp_path):
         orig = test.data_frame("vehicles.csv")
-        handle, path = tempfile.mkstemp(".pkl")
+        path = str(tmp_path / "test.pkl")
         orig.write_pickle(path)
         data = DataFrame.read_pickle(path)
         assert data == orig
 
-    def test_write_pickle_path(self):
+    def test_write_pickle_path(self, tmp_path):
         orig = test.data_frame("vehicles.csv")
-        handle, path = tempfile.mkstemp(".pkl")
+        path = str(tmp_path / "test.pkl")
         orig.write_pickle(Path(path))

{dataiter-1.3.1 → dataiter-1.4}/dataiter/test/test_dt.py

@@ -25,7 +25,7 @@ import numpy as np
 from dataiter import dt
 from dataiter import Vector
 
-NaT = np.datetime64("NaT")
+NaT = np.datetime64("NaT", "D")
 
 class TestDT:
 

{dataiter-1.3.1 → dataiter-1.4}/dataiter/test/test_geojson.py

@@ -20,8 +20,6 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-import tempfile
-
 from dataiter import GeoJSON
 from dataiter import test
 from pathlib import Path
@@ -74,15 +72,15 @@ class TestGeoJSON:
         assert data.head(0).to_string()
         assert data.head(5).to_string()
 
-    def test_write(self):
+    def test_write(self, tmp_path):
         orig = test.geojson(self.path)
-        handle, path = tempfile.mkstemp(".geojson")
+        path = str(tmp_path / "test.geojson")
         orig.write(path)
         data = GeoJSON.read(path)
         assert data == orig
         assert data.metadata == orig.metadata
 
-    def test_write_path(self):
+    def test_write_path(self, tmp_path):
         orig = test.geojson(self.path)
-        handle, path = tempfile.mkstemp(".geojson")
+        path = str(tmp_path / "test.geojson")
         orig.write(Path(path))

{dataiter-1.3.1 → dataiter-1.4}/dataiter/test/test_list_of_dicts.py

@@ -21,7 +21,6 @@
 # THE SOFTWARE.
 
 import datetime
-import tempfile
 
 from attd import AttributeDict
 from dataiter import ListOfDicts
@@ -393,16 +392,16 @@ class TestListOfDicts:
         assert isinstance(data[100].date, datetime.date)
         assert isinstance(data[100].downloads, int)
 
-    def test_read_pickle(self):
+    def test_read_pickle(self, tmp_path):
         orig = test.list_of_dicts("downloads.json")
-        handle, path = tempfile.mkstemp(".pkl")
+        path = str(tmp_path / "test.pkl")
         orig.write_pickle(path)
         data = ListOfDicts.read_pickle(path)
         assert data == orig
 
-    def test_read_pickle_path(self):
+    def test_read_pickle_path(self, tmp_path):
         orig = test.list_of_dicts("downloads.json")
-        handle, path = tempfile.mkstemp(".pkl")
+        path = str(tmp_path / "test.pkl")
         orig.write_pickle(path)
         ListOfDicts.read_pickle(Path(path))
 
@@ -565,38 +564,38 @@ class TestListOfDicts:
         assert all("downloads" not in x for x in data)
         assert orig._obsolete
 
-    def test_write_csv(self):
+    def test_write_csv(self, tmp_path):
         orig = test.list_of_dicts("vehicles.csv")
-        handle, path = tempfile.mkstemp(".csv")
+        path = str(tmp_path / "test.csv")
         orig.write_csv(path)
         data = ListOfDicts.read_csv(path)
         assert data == orig
 
-    def test_write_csv_path(self):
+    def test_write_csv_path(self, tmp_path):
         orig = test.list_of_dicts("vehicles.csv")
-        handle, path = tempfile.mkstemp(".csv")
+        path = str(tmp_path / "test.csv")
         orig.write_csv(Path(path))
 
-    def test_write_json(self):
+    def test_write_json(self, tmp_path):
         orig = test.list_of_dicts("downloads.json")
-        handle, path = tempfile.mkstemp(".json")
+        path = str(tmp_path / "test.json")
         orig.write_json(path)
         data = ListOfDicts.read_json(path)
         assert data == orig
 
-    def test_write_json_path(self):
+    def test_write_json_path(self, tmp_path):
         orig = test.list_of_dicts("downloads.json")
-        handle, path = tempfile.mkstemp(".json")
+        path = str(tmp_path / "test.json")
         orig.write_json(Path(path))
 
-    def test_write_pickle(self):
+    def test_write_pickle(self, tmp_path):
         orig = test.list_of_dicts("downloads.json")
-        handle, path = tempfile.mkstemp(".pkl")
+        path = str(tmp_path / "test.pkl")
         orig.write_pickle(path)
         data = ListOfDicts.read_pickle(path)
         assert data == orig
 
-    def test_write_pickle_path(self):
+    def test_write_pickle_path(self, tmp_path):
         orig = test.list_of_dicts("downloads.json")
-        handle, path = tempfile.mkstemp(".pkl")
+        path = str(tmp_path / "test.pkl")
         orig.write_pickle(Path(path))

{dataiter-1.3.1 → dataiter-1.4}/dataiter/test/test_util.py

@@ -23,7 +23,6 @@
 import datetime
 import math
 import numpy as np
-import tempfile
 
 from dataiter import util
 
@@ -145,33 +144,33 @@ class TestUtil:
         assert util.utruncate("abc\u200bdef", 4) == "abc\u200bd"
         assert util.utruncate("abc\u200bdef\u200b", 4) == "abc\u200bd"
 
-    def test_xopen_bz2(self):
+    def test_xopen_bz2(self, tmp_path):
         text = "test åäö"
-        handle, path = tempfile.mkstemp(".bz2")
+        path = str(tmp_path / "test.bz2")
         with util.xopen(path, "wt") as f:
             f.write(text)
         with util.xopen(path, "rt") as f:
             assert f.read() == text
 
-    def test_xopen_gz(self):
+    def test_xopen_gz(self, tmp_path):
         text = "test åäö"
-        handle, path = tempfile.mkstemp(".gz")
+        path = str(tmp_path / "test.gz")
         with util.xopen(path, "wt") as f:
             f.write(text)
         with util.xopen(path, "rt") as f:
             assert f.read() == text
 
-    def test_xopen_txt(self):
+    def test_xopen_txt(self, tmp_path):
         text = "test åäö"
-        handle, path = tempfile.mkstemp(".txt")
+        path = str(tmp_path / "test.txt")
         with util.xopen(path, "wt") as f:
             f.write(text)
         with util.xopen(path, "rt") as f:
             assert f.read() == text
 
-    def test_xopen_xz(self):
+    def test_xopen_xz(self, tmp_path):
         text = "test åäö"
-        handle, path = tempfile.mkstemp(".xz")
+        path = str(tmp_path / "test.xz")
         with util.xopen(path, "wt") as f:
             f.write(text)
         with util.xopen(path, "rt") as f:

{dataiter-1.3.1 → dataiter-1.4}/dataiter/test/test_vector.py

@@ -28,7 +28,7 @@ from dataiter import Vector
 from numpy.dtypes import StringDType
 
 NaN = np.nan
-NaT = np.datetime64("NaT")
+NaT = np.datetime64("NaT", "D")
 DATE = datetime.date.today()
 DATETIME = datetime.datetime.now()
 

{dataiter-1.3.1 → dataiter-1.4}/dataiter/util.py

@@ -45,9 +45,9 @@ def count_digits(value):
     return n, m
 
 def format_alias_doc(alias, target):
-    return f"{target.__doc__}\n\n{' '*8}" + (
-        ".. note:: :func:`{}` is a convenience alias for :meth:`{}`."
-        .format(alias.__name__, target.__qualname__))
+    return (f"{target.__doc__}\n\n{' '*8}"
+            f".. note:: :func:`{alias.__name__}` is a convenience "
+            f"alias for :meth:`{target.__qualname__}`.")
 
 def format_floats(seq, ksep=None):
     precision = dataiter.PRINT_FLOAT_PRECISION
@@ -62,8 +62,7 @@ def format_floats(seq, ksep=None):
     n = max(x[0] for x in digits)
     m = max(x[1] for x in digits)
     precision = min(m, max(0, precision - n))
-    return [f"{{:,.{precision}f}}".format(x).replace(",", ksep)
-            for x in seq]
+    return [f"{x:,.{precision}f}".replace(",", ksep) for x in seq]
 
 def generate_colnames(n):
     return list(itertools.islice(yield_colnames(), n))

{dataiter-1.3.1 → dataiter-1.4}/dataiter/vector.py

@@ -186,6 +186,10 @@ class Vector(np.ndarray):
             return array.dtype.type(array)
         return array.view(self.__class__)
 
+    def __iter__(self):
+        # NumPy iterates subclasses ~2x slower than base arrays.
+        return iter(self.view(np.ndarray))
+
     def __repr__(self):
         return self.to_string()
 
@@ -427,6 +431,8 @@ class Vector(np.ndarray):
             return np.isnan(self)
         if self.is_string() or self._is_string_fixed():
             return self == dtypes.string.na_object
+        if self.is_integer() or self.is_boolean():
+            return self.fast(np.zeros(len(self), bool))
         # Can't use np.isin here since elements can be arrays.
         return self.fast([x is None for x in self], bool)
 
@@ -542,10 +548,13 @@ class Vector(np.ndarray):
         false and 1.0 for true. Depending on how you use the data, that might
         work as well as an object vector of ``True``, ``False`` and ``None``.
         """
-        if self.is_datetime():
-            return np.datetime64("NaT")
-        if self.is_timedelta():
-            return np.timedelta64("NaT")
+        if self.is_datetime() or self.is_timedelta():
+            unit = np.datetime_data(self.dtype)
+            # Use a coarse unit for generic dtype so that
+            # a finer unit from actual data wins in upcasting.
+            if unit[0] == "generic":
+                unit = "D"
+            return self.dtype.type("NaT", unit)
         if self.is_float():
             return np.nan
         if self.is_integer():
@@ -701,7 +710,7 @@ class Vector(np.ndarray):
         elif len(types) == 1 and types.copy().pop().__module__ == "numpy":
             # If we have a regular Python list of NumPy scalars,
             # infer type. This should be rare, but can happen.
-            dtype = types.copy().pop()().dtype
+            dtype = np.dtype(types.copy().pop())
             na = Vector.fast([], dtype).na_value
         else:
             # Guess the missing value based on types in seq.
@@ -736,7 +745,7 @@ class Vector(np.ndarray):
             return np.nan
         datetimes = [datetime.date, datetime.datetime, np.datetime64]
         if all(x in datetimes for x in types):
-            return np.datetime64("NaT")
+            return np.datetime64("NaT", "D")
         # Usually causes dtype to be object!
         return None
 
@@ -815,7 +824,7 @@ class Vector(np.ndarray):
             strings = util.format_floats(self, ksep=ksep)
             return self.__class__.fast(pad(strings), str)
         if self.is_integer() and not self.is_timedelta():
-            strings = ["{:,d}".format(x).replace(",", ksep) for x in self]
+            strings = [f"{x:,d}".replace(",", ksep) for x in self]
             return self.__class__.fast(pad(strings), str)
         if self.is_object():
             strings = [str(x) for x in self]

{dataiter-1.3.1 → dataiter-1.4}/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["build==1.2.2.post1", "hatchling==1.21.1"]
+requires = ["hatchling==1.32.4"]
 build-backend = "hatchling.build"
 
 [project]
@@ -7,10 +7,10 @@ name = "dataiter"
 dynamic = ["version"]
 description = "Simple, light-weight data frames for Python"
 readme = "README.md"
-license = { text = "MIT" }
+license = "MIT"
 requires-python = ">=3.11.0"
 authors = [{ name = "Osmo Salomaa", email = "otsaloma@iki.fi" }]
-dependencies = ["attd>=0.3", "numpy>=2.4.0,<3.0", "pyarrow>=2.0", "wcwidth>=0.1"]
+dependencies = ["attd>=0.3", "numpy>=2.4.0,<3.0", "pyarrow>=16.0", "wcwidth>=0.1"]
 
 [project.urls]
 Homepage = "https://github.com/otsaloma/dataiter"