dataiter 0.999__tar.gz → 1.0__tar.gz

{dataiter-0.999 → dataiter-1.0}/COPYING

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2019-2024 Osmo Salomaa
+Copyright (c) 2019-2025 Osmo Salomaa
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

{dataiter-0.999 → dataiter-1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataiter
-Version: 0.999
+Version: 1.0
 Summary: Simple, light-weight data frames for Python
 Project-URL: Homepage, https://github.com/otsaloma/dataiter
 Author-email: Osmo Salomaa <otsaloma@iki.fi>
@@ -10,7 +10,7 @@ License-File: COPYING
 Requires-Python: >=3.9.0
 Requires-Dist: attd>=0.3
 Requires-Dist: numpy<3.0,>=2.0
-Requires-Dist: pandas>=1.0
+Requires-Dist: pyarrow>=2.0
 Requires-Dist: wcwidth>=0.1
 Description-Content-Type: text/markdown
 

{dataiter-0.999 → dataiter-1.0}/dataiter/__init__.py

@@ -25,7 +25,7 @@ import numpy as np
 
 from dataiter import util
 
-__version__ = "0.999"
+__version__ = "1.0"
 
 DEFAULT_PEEK_ELEMENTS = 10
 DEFAULT_PEEK_ITEMS = 3
@@ -94,6 +94,7 @@ from dataiter.data_frame import DataFrameColumn # noqa
 from dataiter.geojson import GeoJSON # noqa
 from dataiter.list_of_dicts import ListOfDicts # noqa
 from dataiter import dt # noqa
+from dataiter import regex # noqa
 
 from dataiter.aggregate import all # noqa
 from dataiter.aggregate import any # noqa

{dataiter-0.999 → dataiter-1.0}/dataiter/data_frame.py

@@ -20,6 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+import codecs
 import dataiter
 import functools
 import itertools
@@ -474,16 +475,29 @@ class DataFrame(dict):
             yield colname, np.delete(column, rows)
 
     @classmethod
+    @deco.new_from_generator
     def from_arrow(cls, data, *, dtypes={}):
         """
         Return a new data frame from ``pyarrow.Table`` `data`.
 
         `dtypes` is an optional dict mapping column names to NumPy datatypes.
         """
-        # Arrow's 'to_numpy' is "limited to primitive types for which NumPy has
-        # the same physical representation as Arrow, and assuming the Arrow
-        # data has no nulls." Using Pandas is easier and probably good enough.
-        return cls.from_pandas(data.to_pandas(), dtypes=dtypes)
+        for name, column in zip(data.column_names, data.columns):
+            req_dtype = dtypes.get(name, None)
+            na = column.is_null(nan_is_null=True).to_numpy()
+            column = column.to_numpy()
+            if np.issubdtype(column.dtype, np.object_):
+                if req_dtype is None or np.dtype(req_dtype) != np.dtype(object):
+                    # Likely to be strings, but not necessarily.
+                    # Force type-guessing in Vector.fast.
+                    column = column.tolist()
+            column = DataFrameColumn.fast(column, req_dtype)
+            if na.any():
+                if column.dtype != column.na_dtype:
+                    # Upcast integer to float if needed.
+                    column = column.astype(column.na_dtype)
+                column[na] = column.na_value
+            yield name, column
 
     @classmethod
     def from_json(cls, string, *, columns=[], dtypes={}, **kwargs):
@@ -508,21 +522,29 @@ class DataFrame(dict):
         return cls(**data)
 
     @classmethod
+    @deco.new_from_generator
     def from_pandas(cls, data, *, dtypes={}):
         """
         Return a new data frame from ``pandas.DataFrame`` `data`.
 
         `dtypes` is an optional dict mapping column names to NumPy datatypes.
         """
-        data = {x: data[x].to_numpy(copy=True) for x in data.columns}
-        for name, value in data.items():
-            # Pandas object columns are likely to be strings,
-            # convert to list to force type guessing in Vector.__init__.
-            if np.issubdtype(value.dtype, np.object_):
-                data[name] = data[name].tolist()
-        for name, dtype in dtypes.items():
-            data[name] = DataFrameColumn(data[name], dtype)
-        return cls(**data)
+        for name in data.columns:
+            req_dtype = dtypes.get(name, None)
+            na = data[name].isna().to_numpy(copy=True)
+            column = data[name].to_numpy(copy=True)
+            if np.issubdtype(column.dtype, np.object_):
+                if req_dtype is None or np.dtype(req_dtype) != np.dtype(object):
+                    # Likely to be strings, but not necessarily.
+                    # Force type-guessing in Vector.fast.
+                    column = column.tolist()
+            column = DataFrameColumn.fast(column, req_dtype)
+            if na.any():
+                if column.dtype != column.na_dtype:
+                    # Upcast integer to float if needed.
+                    column = column.astype(column.na_dtype)
+                column[na] = column.na_value
+            yield name, column
 
     def full_join(self, other, *by):
         """
@@ -720,8 +742,9 @@ class DataFrame(dict):
         self._check_dimensions()
         return len(self)
 
-    def _new(self, *args, **kwargs):
-        return self.__class__(*args, **kwargs)
+    @classmethod
+    def _new(cls, *args, **kwargs):
+        return cls(*args, **kwargs)
 
     @property
     def nrow(self):
@@ -857,19 +880,17 @@ class DataFrame(dict):
         `columns` is an optional list of columns to limit to. `dtypes` is an
         optional dict mapping column names to NumPy datatypes.
         """
-        import pandas as pd
-        data = pd.read_csv(path,
-                           sep=sep,
-                           header=0 if header else None,
-                           usecols=columns or None,
-                           dtype=dtypes,
-                           parse_dates=False,
-                           encoding=encoding,
-                           low_memory=False)
+        from pyarrow import csv
+        table = csv.read_csv(
+            path,
+            read_options=csv.ReadOptions(encoding=encoding, autogenerate_column_names=not header),
+            parse_options=csv.ParseOptions(delimiter=sep, newlines_in_values=True),
+            convert_options=csv.ConvertOptions(include_columns=columns))
 
         if not header:
-            data.columns = util.generate_colnames(len(data.columns))
-        return cls.from_pandas(data, dtypes=dtypes)
+            names = util.generate_colnames(table.shape[1])
+            table = table.rename_columns(names)
+        return cls.from_arrow(table, dtypes=dtypes)
 
     @classmethod
     def read_json(cls, path, *, encoding="utf-8", columns=[], dtypes={}, **kwargs):
@@ -1255,9 +1276,18 @@ class DataFrame(dict):
 
         Will automatically compress if `path` ends in ``.bz2|.gz|.xz``.
         """
-        data = self.to_pandas()
+        from pyarrow import csv
+        table = self.to_arrow()
         util.makedirs_for_file(path)
-        data.to_csv(path, sep=sep, header=header, index=False, encoding=encoding)
+        csv.write_csv(table, path, write_options=csv.WriteOptions(include_header=header,
+                                                                  delimiter=sep,
+                                                                  quoting_style="needed"))
+
+        # XXX: Arrow's WriteOptions doesn't include encoding.
+        # If user requested something else than UTF-8, we need to rewrite!
+        if codecs.lookup(encoding) != codecs.lookup("utf-8"):
+            with util.xopen(path, "rt", encoding="utf-8") as f: text = f.read()
+            with util.xopen(path, "wt", encoding=encoding) as f: f.write(text)
 
     def write_json(self, path, *, encoding="utf-8", **kwargs):
         """

{dataiter-0.999 → dataiter-1.0}/dataiter/dt.py

@@ -50,7 +50,6 @@ def from_string(x, format):
     if util.is_scalar(x):
         x = Vector([x], str)
         return from_string(x, format)[0]
-    x = util.sequencify(x)
     assert isinstance(x, np.ndarray)
     assert isinstance(x.dtype, StringDType)
     out = np.full_like(x, None, object)
@@ -150,7 +149,6 @@ def _pull_datetime(x, function):
     if util.is_scalar(x):
         x = Vector([x], np.datetime64)
         return _pull_datetime(x, function)[0]
-    x = util.sequencify(x)
     assert isinstance(x, np.ndarray)
     assert np.issubdtype(x.dtype, np.datetime64)
     out = np.full_like(x, np.nan)
@@ -165,7 +163,6 @@ def _pull_int(x, function):
     if util.is_scalar(x):
         x = Vector([x], np.datetime64)
         return _pull_int(x, function)[0]
-    x = util.sequencify(x)
     assert isinstance(x, np.ndarray)
     assert np.issubdtype(x.dtype, np.datetime64)
     out = np.full_like(x, np.nan, float)
@@ -180,7 +177,6 @@ def _pull_str(x, function):
     if util.is_scalar(x):
         x = Vector([x], np.datetime64)
         return _pull_str(x, function)[0]
-    x = util.sequencify(x)
     assert isinstance(x, np.ndarray)
     assert np.issubdtype(x.dtype, np.datetime64)
     out = np.full_like(x, dtypes.string.na_object, object)
@@ -216,7 +212,6 @@ def replace(x, year=None, month=None, day=None, hour=None, minute=None, second=N
     scalar_keys = [x for x in kwargs if util.is_scalar(kwargs[x])]
     vector_keys = [x for x in kwargs if x not in scalar_keys]
     # Like _pull_datetime, but no vectorized function.
-    x = util.sequencify(x)
     assert isinstance(x, np.ndarray)
     assert np.issubdtype(x.dtype, np.datetime64)
     out = np.full_like(x, np.nan)

{dataiter-0.999 → dataiter-1.0}/dataiter/dtypes.py

@@ -20,6 +20,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+import warnings
+
 from numpy.dtypes import StringDType
 
 #: Instance of NumPy variable-width StringDType used
@@ -30,3 +32,7 @@ string = StringDType(na_object="")
 # input, a distinction between NA and blank cannot usually be made.
 # TODO: Consider changing this to something like ':NA:'.
 # https://numpy.org/doc/stable/user/basics.strings.html#missing-data-support
+
+# Ignore pointless warnings about using StringDType in numpy.savez.
+_pattern = "Custom dtypes are saved as python objects using the pickle protocol"
+warnings.filterwarnings("ignore", message=_pattern, category=UserWarning)

dataiter-1.0/dataiter/regex.py

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2025 Osmo Salomaa
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import numpy as np
+import re
+
+from dataiter import dtypes
+from dataiter import util
+from dataiter import Vector
+from numpy.dtypes import StringDType
+
+def _prep(string, dtype, default):
+    assert isinstance(string, np.ndarray)
+    assert isinstance(string.dtype, StringDType)
+    out = np.full_like(string, default, dtype)
+    na = string == dtypes.string.na_object
+    return out, na
+
+def findall(pattern, string, flags=0):
+    """
+    Return a list of matches of `pattern` in `string`.
+
+    https://docs.python.org/3/library/re.html#re.findall
+
+    >>> x = di.Vector(["asdf", "1234"])
+    >>> regex.findall(r"[a-z]", x)
+    """
+    if util.is_scalar(string):
+        return re.findall(pattern, string, flags=flags)
+    out, na = _prep(string, object, None)
+    for i in np.flatnonzero(~na):
+        out[i] = re.findall(pattern, string[i], flags=flags)
+    return Vector.fast(out, object)
+
+def fullmatch(pattern, string, flags=0):
+    """
+    Return a ``re.Match`` object or ``None``.
+
+    https://docs.python.org/3/library/re.html#re.fullmatch
+
+    >>> x = di.Vector(["asdf", "1234"])
+    >>> regex.fullmatch(r"[a-z]+", x)
+    """
+    if util.is_scalar(string):
+        return re.fullmatch(pattern, string, flags=flags)
+    out, na = _prep(string, object, None)
+    for i in np.flatnonzero(~na):
+        out[i] = re.fullmatch(pattern, string[i], flags=flags)
+    return Vector.fast(out, object)
+
+def match(pattern, string, flags=0):
+    """
+    Return a ``re.Match`` object or ``None``.
+
+    https://docs.python.org/3/library/re.html#re.match
+
+    >>> x = di.Vector(["asdf", "1234"])
+    >>> regex.match(r"[a-z]", x)
+    """
+    if util.is_scalar(string):
+        return re.match(pattern, string, flags=flags)
+    out, na = _prep(string, object, None)
+    for i in np.flatnonzero(~na):
+        out[i] = re.match(pattern, string[i], flags=flags)
+    return Vector.fast(out, object)
+
+def search(pattern, string, flags=0):
+    """
+    Return a ``re.Match`` object or ``None``.
+
+    https://docs.python.org/3/library/re.html#re.search
+
+    >>> x = di.Vector(["asdf", "1234"])
+    >>> regex.search(r"[a-z]", x)
+    """
+    if util.is_scalar(string):
+        return re.search(pattern, string, flags=flags)
+    out, na = _prep(string, object, None)
+    for i in np.flatnonzero(~na):
+        out[i] = re.search(pattern, string[i], flags=flags)
+    return Vector.fast(out, object)
+
+def split(pattern, string, maxsplit=0, flags=0):
+    """
+    Return a list of `string` split by `pattern`.
+
+    https://docs.python.org/3/library/re.html#re.split
+
+    >>> x = di.Vector(["one two three", "four"])
+    >>> regex.split(r" +", x)
+    """
+    if util.is_scalar(string):
+        return re.split(pattern, string, maxsplit=maxsplit, flags=flags)
+    out, na = _prep(string, object, None)
+    for i in np.flatnonzero(~na):
+        out[i] = re.split(pattern, string[i], maxsplit=maxsplit, flags=flags)
+    return Vector.fast(out, object)
+
+def sub(pattern, repl, string, count=0, flags=0):
+    """
+    Return `string` with instances of `pattern` replaced with `repl`.
+
+    https://docs.python.org/3/library/re.html#re.sub
+
+    >>> x = di.Vector(["great", "fantastic"])
+    >>> regex.sub(r"$", r"!", x)
+    """
+    if util.is_scalar(string):
+        return re.sub(pattern, repl, string, count=count, flags=flags)
+    out, na = _prep(string, dtypes.string, dtypes.string.na_object)
+    for i in np.flatnonzero(~na):
+        out[i] = re.sub(pattern, repl, string[i], count=count, flags=flags)
+    return Vector.fast(out, str)
+
+def subn(pattern, repl, string, count=0, flags=0):
+    """
+    Return `string`, count of instances of `pattern` replaced with `repl`.
+
+    https://docs.python.org/3/library/re.html#re.subn
+
+    >>> x = di.Vector(["great", "fantastic"])
+    >>> regex.subn(r"$", r"!", x)
+    """
+    if util.is_scalar(string):
+        return re.subn(pattern, repl, string, count=count, flags=flags)
+    out, na = _prep(string, object, None)
+    for i in np.flatnonzero(~na):
+        out[i] = re.subn(pattern, repl, string[i], count=count, flags=flags)
+    return Vector.fast(out, object)

{dataiter-0.999 → dataiter-1.0}/dataiter/test/test_data_frame.py

@@ -31,9 +31,8 @@ from dataiter import test
 from pathlib import Path
 from unittest.mock import patch
 
-# This UserWarning seems unavoidable when using StringDtype, which unlike
-# other dtypes must be instantiated and that custom instance used.
-IGNORE_NPZ_PICKLE_WARNING = "ignore:Custom dtypes are saved as python objects using the pickle protocol."
+# See dataiter/dtypes.py for the equivalent non-test ignore.
+IGNORE_NPZ_PICKLE_WARNING = "ignore:Custom dtypes are saved as python objects using the pickle protocol"
 
 class TestDataFrameColumn:
 
@@ -448,6 +447,70 @@ class TestDataFrame:
         assert data.make.is_object()
         assert data.model.is_object()
 
+    def test_read_csv_missing_blank(self):
+        text = """
+int,float,string,bool,date
+,2.5,"test",TRUE,2025-01-01
+1,,"test",TRUE,2025-01-01
+1,2.5,"",TRUE,2025-01-01
+1,2.5,"test",,2025-01-01
+1,2.5,"test",TRUE,
+""".strip()
+        handle, path = tempfile.mkstemp(".csv")
+        Path(path).write_text(text, "utf-8")
+        data = DataFrame.read_csv(path)
+        for i, column in enumerate(data.columns):
+            assert column.is_na().sum() == 1
+            assert column.is_na()[i]
+
+    def test_read_csv_missing_excel(self):
+        text = """
+int,float,string,bool,date
+#N/A,2.5,"test",TRUE,2025-01-01
+1,#N/A,"test",TRUE,2025-01-01
+1,2.5,"",TRUE,2025-01-01
+1,2.5,"test",#N/A,2025-01-01
+1,2.5,"test",TRUE,#N/A
+""".strip()
+        handle, path = tempfile.mkstemp(".csv")
+        Path(path).write_text(text, "utf-8")
+        data = DataFrame.read_csv(path)
+        for i, column in enumerate(data.columns):
+            assert column.is_na().sum() == 1
+            assert column.is_na()[i]
+
+    def test_read_csv_missing_na(self):
+        text = """
+int,float,string,bool,date
+NA,2.5,"test",TRUE,2025-01-01
+1,NA,"test",TRUE,2025-01-01
+1,2.5,"",TRUE,2025-01-01
+1,2.5,"test",NA,2025-01-01
+1,2.5,"test",TRUE,NA
+""".strip()
+        handle, path = tempfile.mkstemp(".csv")
+        Path(path).write_text(text, "utf-8")
+        data = DataFrame.read_csv(path)
+        for i, column in enumerate(data.columns):
+            assert column.is_na().sum() == 1
+            assert column.is_na()[i]
+
+    def test_read_csv_missing_null(self):
+        text = """
+int,float,string,bool,date
+NULL,2.5,"test",TRUE,2025-01-01
+1,NULL,"test",TRUE,2025-01-01
+1,2.5,"",TRUE,2025-01-01
+1,2.5,"test",NULL,2025-01-01
+1,2.5,"test",TRUE,NULL
+""".strip()
+        handle, path = tempfile.mkstemp(".csv")
+        Path(path).write_text(text, "utf-8")
+        data = DataFrame.read_csv(path)
+        for i, column in enumerate(data.columns):
+            assert column.is_na().sum() == 1
+            assert column.is_na()[i]
+
     def test_read_json(self):
         path = str(test.get_data_path("downloads.json"))
         data = DataFrame.read_json(path)

dataiter-1.0/dataiter/test/test_regex.py

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2025 Osmo Salomaa
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import re
+
+from dataiter import regex
+from dataiter import Vector
+
+class TestRegex:
+
+    def test_findall(self):
+        pattern = r"[a-z]"
+        string = Vector(["asdf", "1234", ""])
+        result = regex.findall(pattern, string)
+        expected = [["a", "s", "d", "f"], [], None]
+        assert result.tolist() == expected
+        assert regex.findall(pattern, string[0]) == result[0]
+
+    def test_fullmatch(self):
+        pattern = r"[a-z]+"
+        string = Vector(["asdf", "1234", ""])
+        result = regex.fullmatch(pattern, string)
+        assert isinstance(result[0], re.Match)
+        assert result[1] is None
+        assert result[2] is None
+        match = regex.fullmatch(pattern, string[0])
+        assert isinstance(match, re.Match)
+
+    def test_match(self):
+        pattern = r"[a-z]"
+        string = Vector(["asdf", "1234", ""])
+        result = regex.match(pattern, string)
+        assert isinstance(result[0], re.Match)
+        assert result[1] is None
+        assert result[2] is None
+        match = regex.match(pattern, string[0])
+        assert isinstance(match, re.Match)
+
+    def test_search(self):
+        pattern = r"[a-z]"
+        string = Vector(["asdf", "1234", ""])
+        result = regex.search(pattern, string)
+        assert isinstance(result[0], re.Match)
+        assert result[1] is None
+        assert result[2] is None
+        match = regex.search(pattern, string[0])
+        assert isinstance(match, re.Match)
+
+    def test_split(self):
+        pattern = r" +"
+        string = Vector(["one two three", "four", ""])
+        result = regex.split(pattern, string)
+        expected = [["one", "two", "three"], ["four"], None]
+        assert result.tolist() == expected
+        assert regex.split(pattern, string[0]) == result[0]
+
+    def test_sub(self):
+        pattern = r"$"
+        repl = "!"
+        string = Vector(["great", "fantastic", ""])
+        result = regex.sub(pattern, repl, string)
+        expected = ["great!", "fantastic!", None]
+        assert result.tolist() == expected
+        assert regex.sub(pattern, repl, string[0]) == result[0]
+
+    def test_subn(self):
+        pattern = r"$"
+        repl = "!"
+        string = Vector(["great", "fantastic", ""])
+        result = regex.subn(pattern, repl, string)
+        expected = [("great!", 1), ("fantastic!", 1), None]
+        assert result.tolist() == expected
+        assert regex.subn(pattern, repl, string[0]) == result[0]

{dataiter-0.999 → dataiter-1.0}/dataiter/test/test_vector.py

@@ -234,6 +234,12 @@ class TestVector:
         b = a.drop_na()
         assert b.tolist() == [1, 2, 3]
 
+    def test_dt(self):
+        a = Vector(["2025-01-11"], np.datetime64)
+        assert a.dt.year().tolist() == [2025]
+        assert a.dt.month().tolist() == [1]
+        assert a.dt.day().tolist() == [11]
+
     def test_equal_boolean(self):
         a = Vector([True, False])
         b = Vector([True, False])
@@ -463,6 +469,10 @@ class TestVector:
         assert a.rank(method="min").equal(a.rank(method="max"))
         assert a.rank(method="min").equal(a.rank(method="ordinal"))
 
+    def test_re(self):
+        a = Vector(["great", "fantastic"])
+        assert a.re.sub(r"$", r"!").tolist() == ["great!", "fantastic!"]
+
     def test_replace_na(self):
         a = Vector([1, 2, 3, None])
         assert a.replace_na(0).tolist() == [1, 2, 3, 0]
@@ -492,6 +502,12 @@ class TestVector:
         assert a.sort(dir=1).tolist() == [1, 2, 3, 4, 5, None, None]
         assert a.sort(dir=-1).tolist() == [5, 4, 3, 2, 1, None, None]
 
+    def test_str(self):
+        a = Vector(["asdf", "1234"])
+        assert a.str.isalpha().tolist() == [True, False]
+        assert a.str.str_len().tolist() == [4, 4]
+        assert a.str.upper().tolist() == ["ASDF", "1234"]
+
     def test_tail(self):
         a = Vector([1, 2, 3, 4, 5])
         assert a.tail(3).tolist() == [3, 4, 5]

{dataiter-0.999 → dataiter-1.0}/dataiter/vector.py

@@ -22,6 +22,7 @@
 
 import dataiter
 import datetime
+import functools
 import numpy as np
 import sys
 
@@ -35,6 +36,103 @@ TYPE_CONVERSIONS = {
     datetime.datetime: "datetime64[us]",
 }
 
+def as_vector(function):
+    @functools.wraps(function)
+    def wrapper(*args, **kwargs):
+        array = function(*args, **kwargs)
+        return array.view(Vector)
+    return wrapper
+
+def not_implemented(*args, **kwargs):
+    raise NotImplementedError
+
+class DtProxy:
+    def __init__(self, vector):
+        from dataiter import dt
+        wrap = lambda f: functools.partial(f, vector)
+        self.day = wrap(dt.day)
+        self.from_string = wrap(dt.from_string)
+        self.hour = wrap(dt.hour)
+        self.isoweek = wrap(dt.isoweek)
+        self.isoweekday = wrap(dt.isoweekday)
+        self.microsecond = wrap(dt.microsecond)
+        self.minute = wrap(dt.minute)
+        self.month = wrap(dt.month)
+        self.new = wrap(dt.new)
+        self.quarter = wrap(dt.quarter)
+        self.replace = wrap(dt.replace)
+        self.second = wrap(dt.second)
+        self.to_string = wrap(dt.to_string)
+        self.weekday = wrap(dt.weekday)
+        self.year = wrap(dt.year)
+
+class ReProxy:
+    def __init__(self, vector):
+        from dataiter import regex
+        wrap = lambda f: functools.partial(f, string=vector)
+        self.findall = wrap(regex.findall)
+        self.fullmatch = wrap(regex.fullmatch)
+        self.match = wrap(regex.match)
+        self.search = wrap(regex.search)
+        self.split = wrap(regex.split)
+        self.sub = wrap(regex.sub)
+        self.subn = wrap(regex.subn)
+
+class StrProxy:
+    def __init__(self, vector):
+        wrap = lambda name: as_vector(functools.partial(
+            getattr(np.strings, name, not_implemented),
+            vector))
+
+        # https://numpy.org/doc/stable/reference/routines.strings.html
+        self.add = wrap("add")
+        self.capitalize = wrap("capitalize")
+        self.center = wrap("center")
+        self.count = wrap("count")
+        self.decode = wrap("decode")
+        self.encode = wrap("encode")
+        self.endswith = wrap("endswith")
+        self.equal = wrap("equal")
+        self.expandtabs = wrap("expandtabs")
+        self.find = wrap("find")
+        self.greater = wrap("greater")
+        self.greater_equal = wrap("greater_equal")
+        self.index = wrap("index")
+        self.isalnum = wrap("isalnum")
+        self.isalpha = wrap("isalpha")
+        self.isdecimal = wrap("isdecimal")
+        self.isdigit = wrap("isdigit")
+        self.islower = wrap("islower")
+        self.isnumeric = wrap("isnumeric")
+        self.isspace = wrap("isspace")
+        self.istitle = wrap("istitle")
+        self.isupper = wrap("isupper")
+        self.less = wrap("less")
+        self.less_equal = wrap("less_equal")
+        self.ljust = wrap("ljust")
+        self.lower = wrap("lower")
+        self.lstrip = wrap("lstrip")
+        self.mod = wrap("mod")
+        self.multiply = wrap("multiply")
+        self.not_equal = wrap("not_equal")
+        self.replace = wrap("replace")
+        self.rfind = wrap("rfind")
+        self.rindex = wrap("rindex")
+        self.rjust = wrap("rjust")
+        self.rstrip = wrap("rstrip")
+        self.startswith = wrap("startswith")
+        self.str_len = wrap("str_len")
+        self.strip = wrap("strip")
+        self.swapcase = wrap("swapcase")
+        self.title = wrap("title")
+        self.translate = wrap("translate")
+        self.upper = wrap("upper")
+        self.zfill = wrap("zfill")
+
+        # Skip functions that have a difficult return value.
+        # self.partition = wrap("partition")
+        # self.rpartition = wrap("rpartition")
+
 class Vector(np.ndarray):
 
     """
@@ -76,6 +174,8 @@ class Vector(np.ndarray):
 
         >>> di.Vector([1, 2, 3], int)
         """
+        # Note that when we call array.view(Vector) for a numpy.ndarray,
+        # this will not be called and no attributes added here will exist!
         self._check_dimensions()
 
     def __array_wrap__(self, array, context=None, return_scalar=False):
@@ -108,8 +208,7 @@ class Vector(np.ndarray):
         >>> vector.as_bytes()
         """
         if self.is_string():
-            array = np.strings.encode(self, "utf-8")
-            return array.view(self.__class__)
+            return self.str.encode("utf-8")
         return self.astype(bytes)
 
     def as_date(self):
@@ -192,6 +291,20 @@ class Vector(np.ndarray):
         """
         return self[~self.is_na()].copy()
 
+    @property
+    def dt(self) -> DtProxy:
+        """
+        Proxy object for calling :mod:`dataiter.dt` functions.
+
+        >>> x = di.Vector(["2025-01-11"], np.datetime64)
+        >>> x.dt.year()
+        >>> x.dt.month()
+        >>> x.dt.day()
+        """
+        if not hasattr(self, "_dt"):
+            self._dt = DtProxy(self)
+        return self._dt
+
     @property
     def dtype_label(self):
         """
@@ -457,7 +570,7 @@ class Vector(np.ndarray):
         return array
 
     def _optimize_for_argsort(self):
-        if self.is_string() and (n := np.strings.str_len(self).max()) < 50:
+        if self.is_string() and (n := self.str.str_len().max()) < 50:
             # XXX: We get a huge speed boost often by converting
             # to the old-style fixed-width strings! This is probably
             # temporary and can be removed once StringDType has matured.
@@ -525,6 +638,18 @@ class Vector(np.ndarray):
             return out.view(self.__class__)
         raise ValueError(f"Unexpected method: {method!r}")
 
+    @property
+    def re(self) -> ReProxy:
+        """
+        Proxy object for calling :mod:`dataiter.regex` functions.
+
+        >>> x = di.Vector(["great", "fantastic"])
+        >>> x.re.sub(r"$", r"!")
+        """
+        if not hasattr(self, "_re"):
+            self._re = ReProxy(self)
+        return self._re
+
     def replace_na(self, value):
         """
         Return vector with missing values replaced with `value`.
@@ -623,6 +748,22 @@ class Vector(np.ndarray):
         # Usually causes dtype to be object!
         return None
 
+    @property
+    def str(self) -> StrProxy:
+        """
+        Proxy object for calling ``numpy.strings`` functions.
+
+        https://numpy.org/doc/stable/reference/routines.strings.html
+
+        >>> x = di.Vector(["asdf", "1234"])
+        >>> x.str.isalpha()
+        >>> x.str.str_len()
+        >>> x.str.upper()
+        """
+        if not hasattr(self, "_str"):
+            self._str = StrProxy(self)
+        return self._str
+
     def tail(self, n=None):
         """
         Return the last `n` elements.

{dataiter-0.999 → dataiter-1.0}/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["hatchling==1.21.1"]
+requires = ["build==1.2.2.post1", "hatchling==1.21.1"]
 build-backend = "hatchling.build"
 
 [project]
@@ -10,7 +10,7 @@ readme = "README.md"
 license = "MIT"
 requires-python = ">=3.9.0"
 authors = [{ name = "Osmo Salomaa", email = "otsaloma@iki.fi" }]
-dependencies = ["attd>=0.3", "numpy>=2.0,<3.0", "pandas>=1.0", "wcwidth>=0.1"]
+dependencies = ["attd>=0.3", "numpy>=2.0,<3.0", "pyarrow>=2.0", "wcwidth>=0.1"]
 
 [project.urls]
 Homepage = "https://github.com/otsaloma/dataiter"