dataiter 0.51__tar.gz → 0.999__tar.gz

{dataiter-0.51 → dataiter-0.999}/.gitignore

@@ -4,10 +4,10 @@
 .envrc
 .pytest_cache
 __pycache__
-benchmark*.csv
 benchmark-head.py
+benchmark-versions.csv
+benchmark-versions.ods
 build
-data/orig
 dataiter.egg-info
 dist
 doc/_build

dataiter-0.999/PKG-INFO

@@ -0,0 +1,89 @@
+Metadata-Version: 2.1
+Name: dataiter
+Version: 0.999
+Summary: Simple, light-weight data frames for Python
+Project-URL: Homepage, https://github.com/otsaloma/dataiter
+Author-email: Osmo Salomaa <otsaloma@iki.fi>
+License-Expression: MIT
+License-File: AUTHORS.md
+License-File: COPYING
+Requires-Python: >=3.9.0
+Requires-Dist: attd>=0.3
+Requires-Dist: numpy<3.0,>=2.0
+Requires-Dist: pandas>=1.0
+Requires-Dist: wcwidth>=0.1
+Description-Content-Type: text/markdown
+
+Simple, Light-Weight Data Frames for Python
+===========================================
+
+[![PyPI](https://img.shields.io/pypi/v/dataiter.svg)](https://pypi.org/project/dataiter)
+[![Downloads](https://pepy.tech/badge/dataiter/month)](https://pepy.tech/project/dataiter)
+
+Dataiter's **`DataFrame`** is a class for tabular data similar to R's
+`data.frame`, implementing all common operations to manipulate data. It
+is under the hood a dictionary of NumPy arrays and thus capable of fast
+vectorized operations. You can consider it to be a light-weight
+alternative to Pandas with a simple and consistent API. Performance-wise
+Dataiter relies on NumPy and Numba and is likely to be at best
+comparable to Pandas.
+
+## Installation
+
+```bash
+# Latest stable version
+pip install -U dataiter
+
+# Latest development version
+pip install -U git+https://github.com/otsaloma/dataiter
+
+# Numba (optional)
+pip install -U numba
+```
+
+Dataiter optionally uses **Numba** to speed up certain operations. If
+you have Numba installed, Dataiter will use it automatically. It's
+currently not a hard dependency, so you need to install it separately.
+
+## Quick Start
+
+```python
+>>> import dataiter as di
+>>> data = di.read_csv("data/listings.csv")
+>>> data.filter(hood="Manhattan", guests=2).sort(price=1).head()
+.
+        id      hood zipcode guests    sqft price
+     int64    string  string  int64 float64 int64
+  ──────── ───────── ─────── ────── ─────── ─────
+0 42279170 Manhattan   10013      2     nan     0
+1 42384530 Manhattan   10036      2     nan     0
+2 18835820 Manhattan   10021      2     nan    10
+3 20171179 Manhattan   10027      2     nan    10
+4 14858544 Manhattan              2     nan    15
+5 31397084 Manhattan   10002      2     nan    19
+6 22289683 Manhattan   10031      2     nan    20
+7  7760204 Manhattan   10040      2     nan    22
+8 43292527 Manhattan   10033      2     nan    22
+9 43268040 Manhattan   10033      2     nan    23
+.
+```
+
+## Documentation
+
+https://dataiter.readthedocs.io/
+
+If you're familiar with either dplyr (R) or Pandas (Python), the
+comparison table in the documentation will give you a quick overview of
+the differences and similarities in common operations.
+
+https://dataiter.readthedocs.io/en/stable/comparison.html
+
+## Development
+
+To install a virtualenv for development, use
+
+    make venv
+
+or, for a specific Python version
+
+    make PYTHON=python3.X venv

dataiter-0.999/README.md

@@ -0,0 +1,73 @@
+Simple, Light-Weight Data Frames for Python
+===========================================
+
+[![PyPI](https://img.shields.io/pypi/v/dataiter.svg)](https://pypi.org/project/dataiter)
+[![Downloads](https://pepy.tech/badge/dataiter/month)](https://pepy.tech/project/dataiter)
+
+Dataiter's **`DataFrame`** is a class for tabular data similar to R's
+`data.frame`, implementing all common operations to manipulate data. It
+is under the hood a dictionary of NumPy arrays and thus capable of fast
+vectorized operations. You can consider it to be a light-weight
+alternative to Pandas with a simple and consistent API. Performance-wise
+Dataiter relies on NumPy and Numba and is likely to be at best
+comparable to Pandas.
+
+## Installation
+
+```bash
+# Latest stable version
+pip install -U dataiter
+
+# Latest development version
+pip install -U git+https://github.com/otsaloma/dataiter
+
+# Numba (optional)
+pip install -U numba
+```
+
+Dataiter optionally uses **Numba** to speed up certain operations. If
+you have Numba installed, Dataiter will use it automatically. It's
+currently not a hard dependency, so you need to install it separately.
+
+## Quick Start
+
+```python
+>>> import dataiter as di
+>>> data = di.read_csv("data/listings.csv")
+>>> data.filter(hood="Manhattan", guests=2).sort(price=1).head()
+.
+        id      hood zipcode guests    sqft price
+     int64    string  string  int64 float64 int64
+  ──────── ───────── ─────── ────── ─────── ─────
+0 42279170 Manhattan   10013      2     nan     0
+1 42384530 Manhattan   10036      2     nan     0
+2 18835820 Manhattan   10021      2     nan    10
+3 20171179 Manhattan   10027      2     nan    10
+4 14858544 Manhattan              2     nan    15
+5 31397084 Manhattan   10002      2     nan    19
+6 22289683 Manhattan   10031      2     nan    20
+7  7760204 Manhattan   10040      2     nan    22
+8 43292527 Manhattan   10033      2     nan    22
+9 43268040 Manhattan   10033      2     nan    23
+.
+```
+
+## Documentation
+
+https://dataiter.readthedocs.io/
+
+If you're familiar with either dplyr (R) or Pandas (Python), the
+comparison table in the documentation will give you a quick overview of
+the differences and similarities in common operations.
+
+https://dataiter.readthedocs.io/en/stable/comparison.html
+
+## Development
+
+To install a virtualenv for development, use
+
+    make venv
+
+or, for a specific Python version
+
+    make PYTHON=python3.X venv

{dataiter-0.51 → dataiter-0.999}/dataiter/__init__.py

@@ -21,10 +21,11 @@
 # THE SOFTWARE.
 
 import contextlib
+import numpy as np
 
 from dataiter import util
 
-__version__ = "0.51"
+__version__ = "0.999"
 
 DEFAULT_PEEK_ELEMENTS = 10
 DEFAULT_PEEK_ITEMS = 3
@@ -33,12 +34,33 @@ PRINT_FLOAT_PRECISION = 6
 PRINT_MAX_ELEMENTS = 100
 PRINT_MAX_ITEMS = 10
 PRINT_MAX_ROWS = 100
+
+#: Maximum amount of columns to wrap print output to. This is only a fallback
+#: in case Python's ``shutil.get_terminal_size`` fails to detect the width of
+#: your terminal. By default the detected full width is used.
 PRINT_MAX_WIDTH = 80
+
+#: Thousand separator to use when printing numbers. By default this is blank,
+#: meaning no thousand separators are rendered.
 PRINT_THOUSAND_SEPARATOR = ""
+
+#: Maximum width to truncate string columns to in :class:`DataFrame` print
+#: output. When this is exceeded, strings will be cut and an ellipsis (``…``)
+#: rendered at the cut point.
 PRINT_TRUNCATE_WIDTH = 36
+
+#: ``True`` to use Numba, if available, to speed up :doc:`aggregations
+#: </aggregation>`, ``False`` to only use pure Python code.
 USE_NUMBA = False
+
+#: ``True`` to use Numba cache for JIT-compiled :doc:`aggregations
+#: </aggregation>`, ``False`` to only keep compiled code in memory for the
+#: duration of the session.
 USE_NUMBA_CACHE = True
 
+if not np.__version__.startswith("2."):
+    raise Exception("NumPy 2.x required")
+
 with contextlib.suppress(LookupError):
     USE_NUMBA_CACHE = util.parse_env_boolean("DATAITER_USE_NUMBA_CACHE")
 
@@ -61,9 +83,11 @@ except LookupError:
 
 globals().pop("check", None)
 globals().pop("contextlib", None)
+globals().pop("np", None)
 globals().pop("numba", None)
 globals().pop("util", None)
 
+from dataiter import dtypes # noqa
 from dataiter.vector import Vector # noqa
 from dataiter.data_frame import DataFrame # noqa
 from dataiter.data_frame import DataFrameColumn # noqa

{dataiter-0.51 → dataiter-0.999}/dataiter/data_frame.py

@@ -20,7 +20,6 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-import contextlib
 import dataiter
 import functools
 import itertools
@@ -475,24 +474,16 @@ class DataFrame(dict):
             yield colname, np.delete(column, rows)
 
     @classmethod
-    def from_arrow(cls, data, *, strings_as_object=inf, dtypes={}):
+    def from_arrow(cls, data, *, dtypes={}):
         """
         Return a new data frame from ``pyarrow.Table`` `data`.
 
-        `strings_as_object` is a cutoff point. If any row has more characters
-        than that, the whole column will use the object data type. This is
-        intended to help limit memory use as NumPy strings are fixed-length and
-        can take a huge amount of memory if even a single row is long. If set,
-        `dtypes` overrides this.
-
         `dtypes` is an optional dict mapping column names to NumPy datatypes.
         """
         # Arrow's 'to_numpy' is "limited to primitive types for which NumPy has
         # the same physical representation as Arrow, and assuming the Arrow
         # data has no nulls." Using Pandas is easier and probably good enough.
-        return cls.from_pandas(data.to_pandas(),
-                               strings_as_object=strings_as_object,
-                               dtypes=dtypes)
+        return cls.from_pandas(data.to_pandas(), dtypes=dtypes)
 
     @classmethod
     def from_json(cls, string, *, columns=[], dtypes={}, **kwargs):
@@ -517,29 +508,12 @@ class DataFrame(dict):
         return cls(**data)
 
     @classmethod
-    def from_pandas(cls, data, *, strings_as_object=inf, dtypes={}):
+    def from_pandas(cls, data, *, dtypes={}):
         """
         Return a new data frame from ``pandas.DataFrame`` `data`.
 
-        `strings_as_object` is a cutoff point. If any row has more characters
-        than that, the whole column will use the object data type. This is
-        intended to help limit memory use as NumPy strings are fixed-length and
-        can take a huge amount of memory if even a single row is long. If set,
-        `dtypes` overrides this.
-
         `dtypes` is an optional dict mapping column names to NumPy datatypes.
         """
-        if (not isinstance(strings_as_object, (int, float)) or
-            isinstance(strings_as_object, bool)):
-            raise TypeError("Expected a number for strings_as_object")
-        dtypes = dtypes.copy()
-        from pandas.api.types import is_object_dtype
-        if strings_as_object < inf:
-            for name in data.columns:
-                if name not in dtypes and is_object_dtype(data[name]):
-                    with contextlib.suppress(AttributeError):
-                        if data[name].str.len().max() > strings_as_object:
-                            dtypes[name] = object
         data = {x: data[x].to_numpy(copy=True) for x in data.columns}
         for name, value in data.items():
             # Pandas object columns are likely to be strings,
@@ -875,21 +849,13 @@ class DataFrame(dict):
             yield colname, total
 
     @classmethod
-    def read_csv(cls, path, *, encoding="utf-8", sep=",", header=True, columns=[], strings_as_object=inf, dtypes={}):
+    def read_csv(cls, path, *, encoding="utf-8", sep=",", header=True, columns=[], dtypes={}):
         """
         Return a new data frame from CSV file `path`.
 
         Will automatically decompress if `path` ends in ``.bz2|.gz|.xz``.
-
-        `columns` is an optional list of columns to limit to.
-
-        `strings_as_object` is a cutoff point. If any row has more characters
-        than that, the whole column will use the object data type. This is
-        intended to help limit memory use as NumPy strings are fixed-length and
-        can take a huge amount of memory if even a single row is long. If set,
-        `dtypes` overrides this.
-
-        `dtypes` is an optional dict mapping column names to NumPy datatypes.
+        `columns` is an optional list of columns to limit to. `dtypes` is an
+        optional dict mapping column names to NumPy datatypes.
         """
         import pandas as pd
         data = pd.read_csv(path,
@@ -903,7 +869,7 @@ class DataFrame(dict):
 
         if not header:
             data.columns = util.generate_colnames(len(data.columns))
-        return cls.from_pandas(data, strings_as_object=strings_as_object, dtypes=dtypes)
+        return cls.from_pandas(data, dtypes=dtypes)
 
     @classmethod
     def read_json(cls, path, *, encoding="utf-8", columns=[], dtypes={}, **kwargs):
@@ -911,7 +877,6 @@ class DataFrame(dict):
         Return a new data frame from JSON file `path`.
 
         Will automatically decompress if `path` ends in ``.bz2|.gz|.xz``.
-
         `columns` is an optional list of columns to limit to. `dtypes` is an
         optional dict mapping column names to NumPy datatypes. `kwargs` are
         passed to ``json.load``.
@@ -931,24 +896,17 @@ class DataFrame(dict):
             return cls(**data)
 
     @classmethod
-    def read_parquet(cls, path, *, columns=[], strings_as_object=inf, dtypes={}):
+    def read_parquet(cls, path, *, columns=[], dtypes={}):
         """
         Return a new data frame from Parquet file `path`.
 
-        `columns` is an optional list of columns to limit to.
-
-        `strings_as_object` is a cutoff point. If any row has more characters
-        than that, the whole column will use the object data type. This is
-        intended to help limit memory use as NumPy strings are fixed-length and
-        can take a huge amount of memory if even a single row is long. If set,
-        `dtypes` overrides this.
-
-        `dtypes` is an optional dict mapping column names to NumPy datatypes.
+        `columns` is an optional list of columns to limit to. `dtypes` is an
+        optional dict mapping column names to NumPy datatypes.
         """
         import pyarrow.parquet as pq
         columns = columns or None
         data = pq.read_table(path, columns=columns)
-        return cls.from_arrow(data, strings_as_object=strings_as_object, dtypes=dtypes)
+        return cls.from_arrow(data, dtypes=dtypes)
 
     @classmethod
     def read_pickle(cls, path):
@@ -1076,23 +1034,30 @@ class DataFrame(dict):
         >>> data = di.read_csv("data/listings.csv")
         >>> data.sort(hood=1, zipcode=1)
         """
-        @deco.tuplefy
-        def sort_key():
-            pairs = colname_dir_pairs.items()
-            for colname, dir in reversed(list(pairs)):
-                if dir not in [1, -1]:
-                    raise ValueError("dir should be 1 or -1")
-                column = self[colname]
-                if column.is_object():
-                    # See Vector.sort for comparison.
-                    column = column.as_string()
-                if column.is_string():
-                    column[column.is_na()] = "\uffff"
-                if dir < 0 and not (column.is_boolean() or column.is_number()):
-                    # Use rank for non-numeric so that we can sort descending.
-                    column = column.rank(method="min")
-                yield column if dir > 0 else -column
-        indices = np.lexsort(sort_key())
+        def sort_key(colname, dir):
+            if dir not in [1, -1]:
+                raise ValueError("dir should be 1 or -1")
+            column = self[colname]
+            column = column._optimize_for_argsort()
+            if column._is_string_fixed():
+                column[column.is_na()] = "\uffff"
+            if dir > 0 and any((
+                # No strings here due to lexsort segfault.
+                # Need to require NumPy >= 2.2.1 to fix this.
+                # https://github.com/numpy/numpy/issues/27984
+                column._is_string_fixed(),
+                column.is_boolean(),
+                column.is_bytes(),
+                column.is_datetime(),
+                column.is_float(),
+                column.is_integer(),
+                column.is_timedelta(),
+            )): return column
+            if not column.is_number():
+                column = column.rank(method="min")
+            return column if dir > 0 else -column
+        indices = np.lexsort(tuple(
+            sort_key(*x) for x in reversed(colname_dir_pairs.items())))
         for colname, column in self.items():
             yield colname, column[indices].copy()
 
@@ -1187,7 +1152,7 @@ class DataFrame(dict):
         n = min(self.nrow, max_rows)
         columns = {colname: util.upad(
             [colname] +
-            [str(column.dtype)] +
+            [str(column.dtype_label)] +
             [str(x) for x in column[:n].to_strings(
                 quote=False, pad=True, truncate_width=truncate_width)]
         ) for colname, column in self.items()}
@@ -1224,21 +1189,31 @@ class DataFrame(dict):
         >>> data = di.read_csv("data/listings.csv")
         >>> data.unique("hood")
         """
+        # Strangely, standard Python is a lot faster here than np.unique
+        # with all the extra needed across columns of different type.
+        # We just need to avoid NaN and NaT for the 'in' checks to work.
         colnames = colnames or self.colnames
-        if (len(colnames) == 1 and
-            not self[colnames[0]].is_object()):
-            # Use a single column directly.
-            by = self[colnames[0]]
-        elif (len(set(self[x].dtype for x in colnames)) == 1 and
-              not self[colnames[0]].is_object()):
-            # Stack matching dtypes directly in a new array.
-            by = np.column_stack([self[x] for x in colnames])
-        else:
-            # Use rank for differing dtypes.
-            by = np.column_stack([self[x].rank(method="min") for x in colnames])
-        indices = np.sort(np.unique(by, return_index=True, axis=0)[1])
+        columns = [self[x] for x in colnames]
+        for i, column in enumerate(columns):
+            if column.is_datetime():
+                flag = np.nanmin(column) - np.timedelta64(1, "D")
+                if np.isnat(flag):
+                    flag = np.datetime64("-001-01-01")
+                columns[i] = column.replace_na(flag)
+            if column.is_float():
+                flag = np.nanmin(column) - 1
+                if np.isnan(flag):
+                    flag = -1
+                columns[i] = column.replace_na(flag)
+        rows = list(zip(*columns))
+        seen = set()
+        keep = []
+        for i in range(self.nrow):
+            if rows[i] not in seen:
+                seen.add(rows[i])
+                keep.append(i)
         for colname, column in self.items():
-            yield colname, column[indices].copy()
+            yield colname, column[keep].copy()
 
     @deco.new_from_generator
     def unselect(self, *colnames):
@@ -1289,7 +1264,6 @@ class DataFrame(dict):
         Write data frame to JSON file `path`.
 
         Will automatically compress if `path` ends in ``.bz2|.gz|.xz``.
-
         `kwargs` are passed to ``json.JSONEncoder``.
         """
         return self.to_list_of_dicts().write_json(path, encoding=encoding, **kwargs)

{dataiter-0.51 → dataiter-0.999}/dataiter/dt.py

@@ -23,8 +23,10 @@
 import datetime
 import numpy as np
 
+from dataiter import dtypes
 from dataiter import util
 from dataiter import Vector
+from numpy.dtypes import StringDType
 
 def day(x):
     """
@@ -46,22 +48,22 @@ def from_string(x, format):
     >>> dt.from_string(x, "%d.%m.%Y")
     """
     if util.is_scalar(x):
-        x = np.array([x], str)
+        x = Vector([x], str)
         return from_string(x, format)[0]
     x = util.sequencify(x)
     assert isinstance(x, np.ndarray)
-    assert np.issubdtype(x.dtype, np.unicode_)
+    assert isinstance(x.dtype, StringDType)
     out = np.full_like(x, None, object)
     out = Vector.fast(out, object)
-    na = x == ""
+    na = x == dtypes.string.na_object
     f = np.vectorize(lambda x: datetime.datetime.strptime(x, format))
     out[~na] = f(x[~na].astype(object))
     out = out.as_datetime()
-    if len(out[~na]) > 0:
-        if (hour(out[~na]) == 0).all():
-            if (minute(out[~na]) == 0).all():
-                if (second(out[~na]) == 0).all():
-                    out = out.as_date()
+    if (len(out[~na]) > 0 and
+        (hour(out[~na])   == 0).all() and
+        (minute(out[~na]) == 0).all() and
+        (second(out[~na]) == 0).all()):
+        out = out.as_date()
     return out
 
 def hour(x):
@@ -146,7 +148,7 @@ def now():
 
 def _pull_datetime(x, function):
     if util.is_scalar(x):
-        x = np.array([x], np.datetime64)
+        x = Vector([x], np.datetime64)
         return _pull_datetime(x, function)[0]
     x = util.sequencify(x)
     assert isinstance(x, np.ndarray)
@@ -161,7 +163,7 @@ def _pull_datetime(x, function):
 
 def _pull_int(x, function):
     if util.is_scalar(x):
-        x = np.array([x], np.datetime64)
+        x = Vector([x], np.datetime64)
         return _pull_int(x, function)[0]
     x = util.sequencify(x)
     assert isinstance(x, np.ndarray)
@@ -176,12 +178,12 @@ def _pull_int(x, function):
 
 def _pull_str(x, function):
     if util.is_scalar(x):
-        x = np.array([x], np.datetime64)
+        x = Vector([x], np.datetime64)
         return _pull_str(x, function)[0]
     x = util.sequencify(x)
     assert isinstance(x, np.ndarray)
     assert np.issubdtype(x.dtype, np.datetime64)
-    out = np.full_like(x, "", object)
+    out = np.full_like(x, dtypes.string.na_object, object)
     out = Vector.fast(out, object)
     na = np.isnat(x)
     if na.all(): return out

dataiter-0.999/dataiter/dtypes.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2024 Osmo Salomaa
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from numpy.dtypes import StringDType
+
+#: Instance of NumPy variable-width StringDType used
+string = StringDType(na_object="")
+
+# Use a blank string as missing value sentinel (1) because that's what we used
+# prior to the NumPy 2.0 StringDType and (2) because in many cases, such as CSV
+# input, a distinction between NA and blank cannot usually be made.
+# TODO: Consider changing this to something like ':NA:'.
+# https://numpy.org/doc/stable/user/basics.strings.html#missing-data-support