dataiter 0.49__tar.gz → 0.50__tar.gz

{dataiter-0.49 → dataiter-0.50}/COPYING

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2019-2023 Osmo Salomaa
+Copyright (c) 2019-2024 Osmo Salomaa
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

{dataiter-0.49 → dataiter-0.50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataiter
-Version: 0.49
+Version: 0.50
 Summary: Classes for data manipulation
 Project-URL: Homepage, https://github.com/otsaloma/dataiter
 Author-email: Osmo Salomaa <otsaloma@iki.fi>
@@ -11,6 +11,7 @@ Requires-Python: >=3.6.0
 Requires-Dist: attd>=0.3
 Requires-Dist: numpy>=1.7
 Requires-Dist: pandas>=1.0
+Requires-Dist: wcwidth>=0.1
 Description-Content-Type: text/markdown
 
 Python Classes for Data Manipulation

{dataiter-0.49 → dataiter-0.50}/dataiter/__init__.py

@@ -24,7 +24,7 @@ import contextlib
 
 from dataiter import util
 
-__version__ = "0.49"
+__version__ = "0.50"
 
 DEFAULT_PEEK_ELEMENTS = 10
 DEFAULT_PEEK_ITEMS = 3

{dataiter-0.49 → dataiter-0.50}/dataiter/data_frame.py

@@ -1185,7 +1185,7 @@ class DataFrame(dict):
         max_width = max_width or util.get_print_width()
         truncate_width = truncate_width or dataiter.PRINT_TRUNCATE_WIDTH
         n = min(self.nrow, max_rows)
-        columns = {colname: util.pad(
+        columns = {colname: util.upad(
             [colname] +
             [str(column.dtype)] +
             [str(x) for x in column[:n].to_strings(
@@ -1194,7 +1194,7 @@ class DataFrame(dict):
         for column in columns.values():
             column.insert(2, "─" * util.ulen(column[0]))
         row_numbers = [str(i) for i in range(n)]
-        row_numbers = util.pad(["", "", ""] + row_numbers)
+        row_numbers = util.upad(["", "", ""] + row_numbers)
         # If the length of rows exceeds max_width, split to
         # batches of columns (like R's print.data.frame).
         rows_to_print = []

{dataiter-0.49 → dataiter-0.50}/dataiter/list_of_dicts.py

@@ -28,6 +28,7 @@ import json
 import operator
 import pickle
 import random
+import sys
 
 from attd import AttributeDict
 from dataiter import deco
@@ -219,6 +220,18 @@ class ListOfDicts(list):
         """
         return self.__deepcopy__()
 
+    @deco.new_from_generator
+    def drop_na(self, *keys):
+        """
+        Return list without items that have missing values in `keys`.
+
+        >>> data = di.read_json("data/listings.json")
+        >>> data.drop_na("sqft")
+        """
+        for item in self:
+            if not any(item.get(x, None) is None for x in keys):
+                yield item
+
     @deco.new_from_generator
     def extend(self, other):
         """
@@ -245,8 +258,7 @@ class ListOfDicts(list):
         >>> data = data.fill_missing_keys()
         """
         if not key_value_pairs:
-            keys = util.unique_keys(itertools.chain(*self))
-            key_value_pairs = dict.fromkeys(keys, None)
+            key_value_pairs = dict.fromkeys(self.keys(), None)
         key_value_pairs = key_value_pairs.items()
         for item in self:
             for key, value in key_value_pairs:
@@ -423,6 +435,15 @@ class ListOfDicts(list):
                 yield item
             yield self[i]
 
+    def keys(self):
+        """
+        Return an iterator over unique keys in all items.
+
+        >>> data = di.read_json("data/listings.json")
+        >>> list(data.keys())
+        """
+        yield from dict.fromkeys(itertools.chain(*self))
+
     @deco.obsoletes
     @deco.new_from_generator
     def left_join(self, other, *by):
@@ -539,6 +560,36 @@ class ListOfDicts(list):
         """
         print(self.to_string(max_items=max_items))
 
+    def print_memory_use(self):
+        """
+        Print memory use by key and total.
+
+        >>> data = di.read_json("data/listings.json")
+        >>> data.print_memory_use()
+        """
+        from dataiter import DataFrame
+        mem = DataFrame()
+        for key in self.keys():
+            new = DataFrame(key=key)
+            values = self.pluck(key)
+            values_real = list(filter(None, values))
+            first = values_real[0] if values_real else None
+            total = sum(sys.getsizeof(x) for x in values)
+            new.type = first.__class__.__name__
+            new.item_size = int(round(total / len(values)))
+            new.total_size = total
+            mem = mem.rbind(new)
+        new = DataFrame(key="TOTAL")
+        new.type = "--"
+        new.item_size = mem.item_size.sum()
+        new.total_size = mem.total_size.sum()
+        mem = mem.rbind(new)
+        # Format sizes into sensible values for display.
+        mem.item_size = [f"{x:.0f} B" for x in mem.item_size]
+        mem.total_size = [f"{x/1024**2:,.0f} MB" for x in mem.total_size]
+        mem.colnames = [x.upper() for x in mem.colnames]
+        print(mem)
+
     def print_na_counts(self):
         """
         Print counts of missing values by key.
@@ -550,7 +601,7 @@ class ListOfDicts(list):
         >>> data.print_na_counts()
         """
         print("Missing counts:")
-        for key in util.unique_keys(itertools.chain(*self)):
+        for key in self.keys():
             n = sum(x.get(key, None) is None for x in self)
             if n == 0: continue
             pc = 100 * n / len(self)
@@ -838,7 +889,7 @@ class ListOfDicts(list):
         if not self:
             raise ValueError("Cannot write empty CSV file")
         # Take a superset of all keys.
-        keys = util.unique_keys(itertools.chain(*self))
+        keys = list(self.keys())
         util.makedirs_for_file(path)
         with util.xopen(path, "wt", encoding=encoding) as f:
             writer = csv.DictWriter(f,

{dataiter-0.49 → dataiter-0.50}/dataiter/test/test_list_of_dicts.py

@@ -171,6 +171,12 @@ class TestListOfDicts:
         assert b._predecessor is a
         assert c._predecessor is None
 
+    def test_drop_na(self):
+        data = test.list_of_dicts("listings.json")
+        data = data.drop_na("sqft")
+        assert len(data) == 396
+        assert None not in data.pluck("sqft")
+
     def test_extend(self):
         orig = test.list_of_dicts("downloads.json")
         data = orig.extend(orig)
@@ -260,6 +266,10 @@ class TestListOfDicts:
         assert isinstance(data[100], AttributeDict)
         assert data[100] == item
 
+    def test_keys(self):
+        data = test.list_of_dicts("listings.json")
+        assert list(data.keys()) == ["id", "hood", "zipcode", "guests", "sqft", "price"]
+
     def test_left_join(self):
         orig = test.list_of_dicts("downloads.json")
         holidays = test.list_of_dicts("holidays.json")
@@ -321,6 +331,12 @@ class TestListOfDicts:
         assert len(downloads) == len(data)
         assert sum(downloads) == 541300639
 
+    @patch("builtins.print")
+    def test_print_memory_use(self, mock_print):
+        data = test.list_of_dicts("vehicles.json")
+        data.print_memory_use()
+        mock_print.assert_called()
+
     @patch("builtins.print")
     def test_print_na_counts(self, mock_print):
         data = test.list_of_dicts("vehicles.json")

{dataiter-0.49 → dataiter-0.50}/dataiter/test/test_util.py

@@ -112,10 +112,6 @@ class TestUtil:
         assert util.length([1]) == 1
         assert util.length([1, 2]) == 2
 
-    def test_pad(self):
-        assert util.pad(["a", "aa", "aaa"], align="right") == ["  a", " aa", "aaa"]
-        assert util.pad(["a", "aa", "aaa"], align="left")  == ["a  ", "aa ", "aaa"]
-
     def test_quote(self):
         assert util.quote("hello") == '"hello"'
         assert util.quote('"hello"') == '"\\"hello\\""'
@@ -140,6 +136,15 @@ class TestUtil:
     def test_unique_types(self):
         assert util.unique_types([1, 2, 3.3, np.nan, None]) == {int, float}
 
+    def test_upad(self):
+        assert util.upad(["a", "aa", "aaa"], align="right") == ["  a", " aa", "aaa"]
+        assert util.upad(["a", "aa", "aaa"], align="left")  == ["a  ", "aa ", "aaa"]
+
+    def test_utruncate(self):
+        assert util.utruncate("abcdef", 4) == "abcd"
+        assert util.utruncate("abc\u200bdef", 4) == "abc\u200bd"
+        assert util.utruncate("abc\u200bdef\u200b", 4) == "abc\u200bd"
+
     def test_xopen_bz2(self):
         text = "test åäö"
         handle, path = tempfile.mkstemp(".bz2")

{dataiter-0.49 → dataiter-0.50}/dataiter/util.py

@@ -31,6 +31,7 @@ import numpy as np
 import os
 import shutil
 import string
+import wcwidth
 
 from dataiter import deco
 from pathlib import Path
@@ -91,15 +92,6 @@ def length(value):
 def makedirs_for_file(path):
     return Path(path).parent.mkdir(parents=True, exist_ok=True)
 
-@deco.listify
-def pad(strings, *, align="right"):
-    width = max(ulen(x) for x in strings)
-    for value in strings:
-        padding = " " * (width - ulen(value))
-        yield (padding + value
-               if align == "right"
-               else value + padding)
-
 def parse_env_boolean(name):
     return {
         "1":     True,
@@ -132,19 +124,34 @@ def unique_keys(keys):
 
 def ulen(string):
     # Return the display length of string accounting for
-    # Unicode characters that have a display width of zero.
-    return len(string
-               .replace("\u200b", "") # zero width space
-               .replace("\u200c", "") # zero width non-joiner
-               .replace("\u200d", "") # zero width joiner
-               .replace("\u2060", "") # word joiner
-               )
+    # Unicode characters that have a display width != 1.
+    length = wcwidth.wcswidth(string)
+    return length if length >= 0 else 0
 
 def unique_types(seq):
     return set(x.__class__ for x in seq if
                x is not None and
                not (isinstance(x, float) and np.isnan(x)))
 
+@deco.listify
+def upad(strings, *, align="right"):
+    # Pad strings for display accounting for
+    # Unicode characters that have a display width != 1.
+    width = max(ulen(x) for x in strings)
+    for value in strings:
+        padding = " " * (width - ulen(value))
+        yield (padding + value
+               if align == "right"
+               else value + padding)
+
+def utruncate(string, width):
+    # Truncate string to display width accounting for
+    # Unicode characters that have a display width != 1.
+    for i in range(1, len(string)):
+        if ulen(string[:i]) > width:
+            return string[:(i-1)]
+    return string
+
 def xopen(path, mode="r", **kwargs):
     if "b" not in mode:
         kwargs.setdefault("encoding", "utf-8")

{dataiter-0.49 → dataiter-0.50}/dataiter/vector.py

@@ -647,7 +647,7 @@ class Vector(np.ndarray):
         if ksep is None:
             ksep = dataiter.PRINT_THOUSAND_SEPARATOR
         quote = util.quote if quote else identity
-        pad = util.pad if pad else identity
+        pad = util.upad if pad else identity
         if self.is_float():
             strings = util.format_floats(self, ksep=ksep)
             return self.__class__.fast(pad(strings), str)
@@ -660,7 +660,7 @@ class Vector(np.ndarray):
                 lines = strings[i].splitlines()
                 if (util.ulen(strings[i]) > truncate_width or
                     (len(lines) > 1 and truncate_width < inf)):
-                    strings[i] = lines[0][:(truncate_width-1)] + "…"
+                    strings[i] = util.utruncate(lines[0], truncate_width-1) + "…"
             return self.__class__.fast(pad(strings), str)
         if self.is_string():
             strings = [quote(x) for x in self]
@@ -668,7 +668,7 @@ class Vector(np.ndarray):
                 lines = strings[i].splitlines()
                 if (util.ulen(strings[i]) > truncate_width or
                     (len(lines) > 1 and truncate_width < inf)):
-                    strings[i] = lines[0][:(truncate_width-1)] + "…"
+                    strings[i] = util.utruncate(lines[0], truncate_width-1) + "…"
             return self.__class__.fast(pad(strings), str)
         strings = [str(x) for x in self]
         return self.__class__.fast(pad(strings), str)

{dataiter-0.49 → dataiter-0.50}/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["hatchling"]
+requires = ["hatchling==1.21.1"]
 build-backend = "hatchling.build"
 
 [project]
@@ -10,7 +10,7 @@ readme = "README.md"
 license = "MIT"
 requires-python = ">=3.6.0"
 authors = [{ name = "Osmo Salomaa", email = "otsaloma@iki.fi" }]
-dependencies = ["attd>=0.3", "numpy>=1.7", "pandas>=1.0"]
+dependencies = ["attd>=0.3", "numpy>=1.7", "pandas>=1.0", "wcwidth>=0.1"]
 
 [project.urls]
 Homepage = "https://github.com/otsaloma/dataiter"