dataiter 1.0__tar.gz → 1.2__tar.gz

{dataiter-1.0 → dataiter-1.2}/.gitignore

@@ -1,5 +1,6 @@
 *.lprof
 *.prof
+.claude
 .env
 .envrc
 .pytest_cache

{dataiter-1.0 → dataiter-1.2}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: dataiter
-Version: 1.0
+Version: 1.2
 Summary: Simple, light-weight data frames for Python
 Project-URL: Homepage, https://github.com/otsaloma/dataiter
 Author-email: Osmo Salomaa <otsaloma@iki.fi>
-License-Expression: MIT
+License: MIT
 License-File: AUTHORS.md
 License-File: COPYING
 Requires-Python: >=3.9.0

{dataiter-1.0 → dataiter-1.2}/dataiter/__init__.py

@@ -25,7 +25,7 @@ import numpy as np
 
 from dataiter import util
 
-__version__ = "1.0"
+__version__ = "1.2"
 
 DEFAULT_PEEK_ELEMENTS = 10
 DEFAULT_PEEK_ITEMS = 3

{dataiter-1.0 → dataiter-1.2}/dataiter/aggregate.py

@@ -25,7 +25,6 @@ import functools
 import numpy as np
 import statistics
 
-from collections import Counter
 from dataiter import deco
 from dataiter import Vector
 
@@ -403,36 +402,30 @@ def mode(x, *, drop_na=True):
         aggregate.group_aware = True
         return aggregate
     x = handle_na(x, drop_na)
-    return mode1(x) if len(x) >= 1 else x.na_value
+    return statistics.mode(x) if len(x) >= 1 else x.na_value
 
 @deco.listify
 def mode_apply(x, group, drop_na):
     for xg in yield_groups(x, group, drop_na):
-        yield mode1(xg) if len(xg) >= 1 else None
+        yield statistics.mode(xg) if len(xg) >= 1 else None
 
 @njit(cache=dataiter.USE_NUMBA_CACHE)
 def mode_apply_numba(x, group, drop_na):
     out = []
     for xg in yield_groups_numba(x, group, drop_na):
         if len(xg) > 0:
-            ng = np.full(len(xg), 0)
+            best_found = xg[0]
+            best_count = 1
             for i in range(len(xg)):
-                for j in range(len(xg)):
-                    if xg[j] == xg[i]:
-                        ng[i] += 1
-            out.append(xg[np.argmax(ng)])
+                count = np.sum(xg == xg[i])
+                if count > best_count:
+                    best_found = xg[i]
+                    best_count = count
+            out.append(best_found)
         else:
             out.append(None)
     return out
 
-def mode1(x):
-    try:
-        return statistics.mode(x)
-    except statistics.StatisticsError:
-        # Python < 3.8 with several elements tied for mode.
-        # Return the first encountered of the tied elements.
-        return Counter(x).most_common(1)[0][0]
-
 @composite
 def nth(x, index, *, drop_na=False):
     """

{dataiter-1.0 → dataiter-1.2}/dataiter/data_frame.py

@@ -396,7 +396,12 @@ class DataFrame(dict):
         >>> data = di.read_csv("data/listings.csv")
         >>> data.count("hood")
         """
-        return self.copy().group_by(*colnames).aggregate(n=dataiter.count())
+        stat = self.copy().group_by(*colnames).aggregate(_n_=dataiter.count())
+        if (name := "n") in stat:
+            while name in stat: name += "_"
+            stat = stat.rename(**{name: "n"})
+        stat.n = stat.pop("_n_")
+        return stat
 
     def deepcopy(self):
         """
@@ -776,6 +781,69 @@ class DataFrame(dict):
     def _parse_rows_from_integer(self, rows):
         return Vector.fast(rows, int)
 
+    def pivot_longer(self, *, ids=None, names=None, values=None):
+        """
+        Pivot data from wide to long format.
+
+        `ids` should be the name or a list of names of identifier columns. All
+        other columns are considered variable columns and will be pivoted.
+        `names` and `values` are names of columns into which the variable names
+        and values are put in the result.
+
+        >>> wide = di.read_csv("data/listings.csv")
+        >>> wide
+        >>> wide.pivot_longer(ids="id", names="name", values="value")
+        """
+        ids = util.sequencify(ids)
+        if not all(x in self for x in ids):
+            raise ValueError(f"Bad {ids=}")
+        long = self.__class__()
+        for name in (x for x in self if x not in ids):
+            part = self.select(*ids)
+            part[names] = name
+            # Maybe we could avoid using object here,
+            # but it gets complicated.
+            part[values] = self[name].as_object()
+            long = long.rbind(part)
+        return long.sort(**dict.fromkeys(ids, 1))
+
+    def pivot_wider(self, *, ids=None, names=None, values=None, rename=None):
+        """
+        Pivot data from long to wide format.
+
+        `ids` should be the name or a list of names of identifier columns by
+        which the result will be unique. If `ids` is not given, it defaults to
+        all columns except `names` and `values`. `names` should be the name of
+        the column that contains the names of variables, which become new
+        columns. `values` should be the name of the column that contains the
+        corresponding values. `rename` is an optional function that you can use
+        to e.g. lowercase the new column names or add a prefix or suffix.
+
+        >>> long = di.read_csv("data/downloads.csv")
+        >>> long = long.sort(date=1)
+        >>> long
+        >>> long.pivot_wider(ids="date", names="category", values="downloads", rename=str.lower)
+        """
+        if names not in self:
+            raise ValueError(f"Bad {names=}")
+        if values not in self:
+            raise ValueError(f"Bad {values=}")
+        if not ids:
+            ids = [x for x in self if x not in [names, values]]
+        ids = util.sequencify(ids)
+        if not all(x in self for x in ids):
+            raise ValueError(f"Bad {ids=}")
+        long = self.copy()
+        wide = self.select(*ids).unique(*ids)
+        if rename:
+            long[names] = long[names].map(rename)
+        for name in long[names].unique().sort():
+            part = long.filter(long[names] == name)
+            part = part.select(*ids, values)
+            part = part.rename(**{name: values})
+            wide = wide.left_join(part, *ids)
+        return wide
+
     def pop(self, key, *args, **kwargs):
         """"""
         value = super().pop(key, *args, **kwargs)
@@ -1195,9 +1263,9 @@ class DataFrame(dict):
                     batch_rows[i] += " "
                     batch_rows[i] += column[i]
                 del columns[colname]
-            rows_to_print.append("" if rows_to_print else ".")
+            rows_to_print.append("" if rows_to_print else "")
             rows_to_print += batch_rows
-        rows_to_print.append(".")
+        rows_to_print.append("")
         if max_rows < self.nrow:
             rows_to_print.append(f"... {self.nrow} rows total")
         return "\n".join(rows_to_print)

{dataiter-1.0 → dataiter-1.2}/dataiter/dt.py

@@ -74,6 +74,19 @@ def hour(x):
     """
     return _pull_int(x, lambda y: y.hour)
 
+def isoformat(x):
+    """
+    Return datetime `x` as a string in ISO 8601 format.
+
+    For dates, return `YYYY-MM-DD`, for datetimes `YYYY-MM-DDTHH:MM:SS.ffffff`.
+
+    >>> x = dt.new([dt.today()])
+    >>> dt.isoformat(x)
+    >>> y = dt.new([dt.now()])
+    >>> dt.isoformat(y)
+    """
+    return _pull_str(x, lambda x: x.isoformat())
+
 def isoweek(x):
     """
     Extract ISO 8601 week from datetime `x`.

{dataiter-1.0 → dataiter-1.2}/dataiter/test/test_aggregate.py

@@ -171,6 +171,8 @@ class TestAggregate:
                 assert stat.a.equal(expected)
             except AssertionError:
                 print("")
+                print("Function:", function.__name__)
+                print("Numba:", use_numba)
                 print(data)
                 print("Expected:")
                 print(expected)

{dataiter-1.0 → dataiter-1.2}/dataiter/test/test_data_frame.py

@@ -229,6 +229,15 @@ class TestDataFrame:
         assert stat.hood.equal(data.hood.unique().sort())
         assert stat.n.sum() == data.nrow
 
+    def test_count_n_exists(self):
+        data = test.data_frame("listings.csv")
+        stat1 = data.count("zipcode")
+        stat2 = data.count("zipcode").count("n")
+        assert stat1.colnames == ["zipcode", "n"]
+        assert stat2.colnames == ["n_", "n"]
+        assert stat2.nrow == len(stat1.n.unique())
+        assert sum(stat2.n) == stat1.nrow
+
     def test_deepcopy(self):
         orig = test.data_frame("vehicles.csv")
         data = orig.copy()
@@ -375,6 +384,32 @@ class TestDataFrame:
         data = test.data_frame("vehicles.csv")
         assert data.nrow == 33442
 
+    def test_pivot_longer(self):
+        wide = test.data_frame("listings.csv")
+        wide = wide.sort(id=1).head(10)
+        long = wide.pivot_longer(ids="id",
+                                 names="name",
+                                 values="value")
+
+        assert long.nrow == 50
+        assert long.ncol == 3
+        assert all(np.isin(long.name, wide.colnames))
+
+    def test_pivot_wider(self):
+        long = test.data_frame("downloads.csv")
+        long = long.sort(date=1).tail(100)
+        wide = long.pivot_wider(ids="date",
+                                names="category",
+                                values="downloads",
+                                rename=lambda x: f"dl_{x.lower()}")
+
+        assert wide.nrow == 20
+        assert wide.ncol == 6
+        total_downloads = sum(
+            sum(wide[x]) for x in wide.colnames
+            if x.startswith("dl_"))
+        assert total_downloads == sum(long.downloads)
+
     def test_pop(self):
         data = test.data_frame("vehicles.csv")
         data.pop("fuel")

{dataiter-1.0 → dataiter-1.2}/dataiter/test/test_dt.py

@@ -62,6 +62,22 @@ class TestDT:
         a = dt.new(["2022-10-15T12:34:56", NaT])
         assert dt.hour(a).tolist() == [12, None]
 
+    def test_isoformat_date(self):
+        a = np.array(["2022-10-15", NaT], np.datetime64)
+        assert dt.isoformat(a).tolist() == ["2022-10-15", None]
+
+    def test_isoformat_date_scalar(self):
+        x = np.datetime64("2022-10-15")
+        assert dt.isoformat(x) == "2022-10-15"
+
+    def test_isoformat_datetime(self):
+        a = np.array(["2022-10-15T12:34:56.789", NaT], np.datetime64)
+        assert dt.isoformat(a).tolist() == ["2022-10-15T12:34:56.789000", None]
+
+    def test_isoformat_datetime_scalar(self):
+        x = np.datetime64("2022-10-15T12:34:56.789")
+        assert dt.isoformat(x) == "2022-10-15T12:34:56.789000"
+
     def test_isoweek(self):
         a = dt.new(["2022-10-15", NaT])
         assert dt.isoweek(a).tolist() == [41, None]

{dataiter-1.0 → dataiter-1.2}/pyproject.toml

@@ -7,7 +7,7 @@ name = "dataiter"
 dynamic = ["version"]
 description = "Simple, light-weight data frames for Python"
 readme = "README.md"
-license = "MIT"
+license = { text = "MIT" }
 requires-python = ">=3.9.0"
 authors = [{ name = "Osmo Salomaa", email = "otsaloma@iki.fi" }]
 dependencies = ["attd>=0.3", "numpy>=2.0,<3.0", "pyarrow>=2.0", "wcwidth>=0.1"]