dataiter 0.99__tar.gz → 0.9999__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {dataiter-0.99 → dataiter-0.9999}/.gitignore +2 -2
  2. {dataiter-0.99 → dataiter-0.9999}/COPYING +1 -1
  3. dataiter-0.9999/PKG-INFO +89 -0
  4. dataiter-0.9999/README.md +73 -0
  5. {dataiter-0.99 → dataiter-0.9999}/dataiter/__init__.py +21 -1
  6. {dataiter-0.99 → dataiter-0.9999}/dataiter/data_frame.py +115 -111
  7. {dataiter-0.99 → dataiter-0.9999}/dataiter/dt.py +14 -17
  8. dataiter-0.9999/dataiter/dtypes.py +32 -0
  9. {dataiter-0.99 → dataiter-0.9999}/dataiter/geojson.py +13 -38
  10. {dataiter-0.99 → dataiter-0.9999}/dataiter/io.py +4 -7
  11. {dataiter-0.99 → dataiter-0.9999}/dataiter/list_of_dicts.py +1 -17
  12. dataiter-0.9999/dataiter/regex.py +148 -0
  13. {dataiter-0.99 → dataiter-0.9999}/dataiter/test/__init__.py +11 -9
  14. {dataiter-0.99 → dataiter-0.9999}/dataiter/test/test_aggregate.py +2 -2
  15. {dataiter-0.99 → dataiter-0.9999}/dataiter/test/test_data_frame.py +72 -5
  16. {dataiter-0.99 → dataiter-0.9999}/dataiter/test/test_geojson.py +0 -6
  17. {dataiter-0.99 → dataiter-0.9999}/dataiter/test/test_io.py +5 -0
  18. {dataiter-0.99 → dataiter-0.9999}/dataiter/test/test_list_of_dicts.py +0 -6
  19. dataiter-0.9999/dataiter/test/test_regex.py +92 -0
  20. {dataiter-0.99 → dataiter-0.9999}/dataiter/test/test_vector.py +35 -33
  21. {dataiter-0.99 → dataiter-0.9999}/dataiter/util.py +1 -3
  22. {dataiter-0.99 → dataiter-0.9999}/dataiter/vector.py +244 -74
  23. {dataiter-0.99 → dataiter-0.9999}/pyproject.toml +2 -2
  24. dataiter-0.99/PKG-INFO +0 -72
  25. dataiter-0.99/README.md +0 -56
  26. {dataiter-0.99 → dataiter-0.9999}/AUTHORS.md +0 -0
  27. {dataiter-0.99 → dataiter-0.9999}/dataiter/aggregate.py +0 -0
  28. {dataiter-0.99 → dataiter-0.9999}/dataiter/deco.py +0 -0
  29. {dataiter-0.99 → dataiter-0.9999}/dataiter/test/test_dt.py +0 -0
  30. {dataiter-0.99 → dataiter-0.9999}/dataiter/test/test_util.py +0 -0
@@ -4,10 +4,10 @@
4
4
  .envrc
5
5
  .pytest_cache
6
6
  __pycache__
7
- benchmark*.csv
8
7
  benchmark-head.py
8
+ benchmark-versions.csv
9
+ benchmark-versions.ods
9
10
  build
10
- data/orig
11
11
  dataiter.egg-info
12
12
  dist
13
13
  doc/_build
@@ -1,6 +1,6 @@
1
1
  The MIT License (MIT)
2
2
 
3
- Copyright (c) 2019-2024 Osmo Salomaa
3
+ Copyright (c) 2019-2025 Osmo Salomaa
4
4
 
5
5
  Permission is hereby granted, free of charge, to any person obtaining a copy
6
6
  of this software and associated documentation files (the "Software"), to deal
@@ -0,0 +1,89 @@
1
+ Metadata-Version: 2.1
2
+ Name: dataiter
3
+ Version: 0.9999
4
+ Summary: Simple, light-weight data frames for Python
5
+ Project-URL: Homepage, https://github.com/otsaloma/dataiter
6
+ Author-email: Osmo Salomaa <otsaloma@iki.fi>
7
+ License-Expression: MIT
8
+ License-File: AUTHORS.md
9
+ License-File: COPYING
10
+ Requires-Python: >=3.9.0
11
+ Requires-Dist: attd>=0.3
12
+ Requires-Dist: numpy<3.0,>=2.0
13
+ Requires-Dist: pyarrow>=2.0
14
+ Requires-Dist: wcwidth>=0.1
15
+ Description-Content-Type: text/markdown
16
+
17
+ Simple, Light-Weight Data Frames for Python
18
+ ===========================================
19
+
20
+ [![PyPI](https://img.shields.io/pypi/v/dataiter.svg)](https://pypi.org/project/dataiter)
21
+ [![Downloads](https://pepy.tech/badge/dataiter/month)](https://pepy.tech/project/dataiter)
22
+
23
+ Dataiter's **`DataFrame`** is a class for tabular data similar to R's
24
+ `data.frame`, implementing all common operations to manipulate data. It
25
+ is under the hood a dictionary of NumPy arrays and thus capable of fast
26
+ vectorized operations. You can consider it to be a light-weight
27
+ alternative to Pandas with a simple and consistent API. Performance-wise
28
+ Dataiter relies on NumPy and Numba and is likely to be at best
29
+ comparable to Pandas.
30
+
31
+ ## Installation
32
+
33
+ ```bash
34
+ # Latest stable version
35
+ pip install -U dataiter
36
+
37
+ # Latest development version
38
+ pip install -U git+https://github.com/otsaloma/dataiter
39
+
40
+ # Numba (optional)
41
+ pip install -U numba
42
+ ```
43
+
44
+ Dataiter optionally uses **Numba** to speed up certain operations. If
45
+ you have Numba installed, Dataiter will use it automatically. It's
46
+ currently not a hard dependency, so you need to install it separately.
47
+
48
+ ## Quick Start
49
+
50
+ ```python
51
+ >>> import dataiter as di
52
+ >>> data = di.read_csv("data/listings.csv")
53
+ >>> data.filter(hood="Manhattan", guests=2).sort(price=1).head()
54
+ .
55
+ id hood zipcode guests sqft price
56
+ int64 string string int64 float64 int64
57
+ ──────── ───────── ─────── ────── ─────── ─────
58
+ 0 42279170 Manhattan 10013 2 nan 0
59
+ 1 42384530 Manhattan 10036 2 nan 0
60
+ 2 18835820 Manhattan 10021 2 nan 10
61
+ 3 20171179 Manhattan 10027 2 nan 10
62
+ 4 14858544 Manhattan 2 nan 15
63
+ 5 31397084 Manhattan 10002 2 nan 19
64
+ 6 22289683 Manhattan 10031 2 nan 20
65
+ 7 7760204 Manhattan 10040 2 nan 22
66
+ 8 43292527 Manhattan 10033 2 nan 22
67
+ 9 43268040 Manhattan 10033 2 nan 23
68
+ .
69
+ ```
70
+
71
+ ## Documentation
72
+
73
+ https://dataiter.readthedocs.io/
74
+
75
+ If you're familiar with either dplyr (R) or Pandas (Python), the
76
+ comparison table in the documentation will give you a quick overview of
77
+ the differences and similarities in common operations.
78
+
79
+ https://dataiter.readthedocs.io/en/stable/comparison.html
80
+
81
+ ## Development
82
+
83
+ To install a virtualenv for development, use
84
+
85
+ make venv
86
+
87
+ or, for a specific Python version
88
+
89
+ make PYTHON=python3.X venv
@@ -0,0 +1,73 @@
1
+ Simple, Light-Weight Data Frames for Python
2
+ ===========================================
3
+
4
+ [![PyPI](https://img.shields.io/pypi/v/dataiter.svg)](https://pypi.org/project/dataiter)
5
+ [![Downloads](https://pepy.tech/badge/dataiter/month)](https://pepy.tech/project/dataiter)
6
+
7
+ Dataiter's **`DataFrame`** is a class for tabular data similar to R's
8
+ `data.frame`, implementing all common operations to manipulate data. It
9
+ is under the hood a dictionary of NumPy arrays and thus capable of fast
10
+ vectorized operations. You can consider it to be a light-weight
11
+ alternative to Pandas with a simple and consistent API. Performance-wise
12
+ Dataiter relies on NumPy and Numba and is likely to be at best
13
+ comparable to Pandas.
14
+
15
+ ## Installation
16
+
17
+ ```bash
18
+ # Latest stable version
19
+ pip install -U dataiter
20
+
21
+ # Latest development version
22
+ pip install -U git+https://github.com/otsaloma/dataiter
23
+
24
+ # Numba (optional)
25
+ pip install -U numba
26
+ ```
27
+
28
+ Dataiter optionally uses **Numba** to speed up certain operations. If
29
+ you have Numba installed, Dataiter will use it automatically. It's
30
+ currently not a hard dependency, so you need to install it separately.
31
+
32
+ ## Quick Start
33
+
34
+ ```python
35
+ >>> import dataiter as di
36
+ >>> data = di.read_csv("data/listings.csv")
37
+ >>> data.filter(hood="Manhattan", guests=2).sort(price=1).head()
38
+ .
39
+ id hood zipcode guests sqft price
40
+ int64 string string int64 float64 int64
41
+ ──────── ───────── ─────── ────── ─────── ─────
42
+ 0 42279170 Manhattan 10013 2 nan 0
43
+ 1 42384530 Manhattan 10036 2 nan 0
44
+ 2 18835820 Manhattan 10021 2 nan 10
45
+ 3 20171179 Manhattan 10027 2 nan 10
46
+ 4 14858544 Manhattan 2 nan 15
47
+ 5 31397084 Manhattan 10002 2 nan 19
48
+ 6 22289683 Manhattan 10031 2 nan 20
49
+ 7 7760204 Manhattan 10040 2 nan 22
50
+ 8 43292527 Manhattan 10033 2 nan 22
51
+ 9 43268040 Manhattan 10033 2 nan 23
52
+ .
53
+ ```
54
+
55
+ ## Documentation
56
+
57
+ https://dataiter.readthedocs.io/
58
+
59
+ If you're familiar with either dplyr (R) or Pandas (Python), the
60
+ comparison table in the documentation will give you a quick overview of
61
+ the differences and similarities in common operations.
62
+
63
+ https://dataiter.readthedocs.io/en/stable/comparison.html
64
+
65
+ ## Development
66
+
67
+ To install a virtualenv for development, use
68
+
69
+ make venv
70
+
71
+ or, for a specific Python version
72
+
73
+ make PYTHON=python3.X venv
@@ -25,7 +25,7 @@ import numpy as np
25
25
 
26
26
  from dataiter import util
27
27
 
28
- __version__ = "0.99"
28
+ __version__ = "0.9999"
29
29
 
30
30
  DEFAULT_PEEK_ELEMENTS = 10
31
31
  DEFAULT_PEEK_ITEMS = 3
@@ -34,10 +34,28 @@ PRINT_FLOAT_PRECISION = 6
34
34
  PRINT_MAX_ELEMENTS = 100
35
35
  PRINT_MAX_ITEMS = 10
36
36
  PRINT_MAX_ROWS = 100
37
+
38
+ #: Maximum amount of columns to wrap print output to. This is only a fallback
39
+ #: in case Python's ``shutil.get_terminal_size`` fails to detect the width of
40
+ #: your terminal. By default the detected full width is used.
37
41
  PRINT_MAX_WIDTH = 80
42
+
43
+ #: Thousand separator to use when printing numbers. By default this is blank,
44
+ #: meaning no thousand separators are rendered.
38
45
  PRINT_THOUSAND_SEPARATOR = ""
46
+
47
+ #: Maximum width to truncate string columns to in :class:`DataFrame` print
48
+ #: output. When this is exceeded, strings will be cut and an ellipsis (``…``)
49
+ #: rendered at the cut point.
39
50
  PRINT_TRUNCATE_WIDTH = 36
51
+
52
+ #: ``True`` to use Numba, if available, to speed up :doc:`aggregations
53
+ #: </aggregation>`, ``False`` to only use pure Python code.
40
54
  USE_NUMBA = False
55
+
56
+ #: ``True`` to use Numba cache for JIT-compiled :doc:`aggregations
57
+ #: </aggregation>`, ``False`` to only keep compiled code in memory for the
58
+ #: duration of the session.
41
59
  USE_NUMBA_CACHE = True
42
60
 
43
61
  if not np.__version__.startswith("2."):
@@ -69,12 +87,14 @@ globals().pop("np", None)
69
87
  globals().pop("numba", None)
70
88
  globals().pop("util", None)
71
89
 
90
+ from dataiter import dtypes # noqa
72
91
  from dataiter.vector import Vector # noqa
73
92
  from dataiter.data_frame import DataFrame # noqa
74
93
  from dataiter.data_frame import DataFrameColumn # noqa
75
94
  from dataiter.geojson import GeoJSON # noqa
76
95
  from dataiter.list_of_dicts import ListOfDicts # noqa
77
96
  from dataiter import dt # noqa
97
+ from dataiter import regex # noqa
78
98
 
79
99
  from dataiter.aggregate import all # noqa
80
100
  from dataiter.aggregate import any # noqa
@@ -20,7 +20,7 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- import contextlib
23
+ import codecs
24
24
  import dataiter
25
25
  import functools
26
26
  import itertools
@@ -475,24 +475,29 @@ class DataFrame(dict):
475
475
  yield colname, np.delete(column, rows)
476
476
 
477
477
  @classmethod
478
- def from_arrow(cls, data, *, strings_as_object=inf, dtypes={}):
478
+ @deco.new_from_generator
479
+ def from_arrow(cls, data, *, dtypes={}):
479
480
  """
480
481
  Return a new data frame from ``pyarrow.Table`` `data`.
481
482
 
482
- `strings_as_object` is a cutoff point. If any row has more characters
483
- than that, the whole column will use the object data type. This is
484
- intended to help limit memory use as NumPy strings are fixed-length and
485
- can take a huge amount of memory if even a single row is long. If set,
486
- `dtypes` overrides this.
487
-
488
483
  `dtypes` is an optional dict mapping column names to NumPy datatypes.
489
484
  """
490
- # Arrow's 'to_numpy' is "limited to primitive types for which NumPy has
491
- # the same physical representation as Arrow, and assuming the Arrow
492
- # data has no nulls." Using Pandas is easier and probably good enough.
493
- return cls.from_pandas(data.to_pandas(),
494
- strings_as_object=strings_as_object,
495
- dtypes=dtypes)
485
+ for name, column in zip(data.column_names, data.columns):
486
+ req_dtype = dtypes.get(name, None)
487
+ na = column.is_null(nan_is_null=True).to_numpy()
488
+ column = column.to_numpy()
489
+ if np.issubdtype(column.dtype, np.object_):
490
+ if req_dtype is None or np.dtype(req_dtype) != np.dtype(object):
491
+ # Likely to be strings, but not necessarily.
492
+ # Force type-guessing in Vector.fast.
493
+ column = column.tolist()
494
+ column = DataFrameColumn.fast(column, req_dtype)
495
+ if na.any():
496
+ if column.dtype != column.na_dtype:
497
+ # Upcast integer to float if needed.
498
+ column = column.astype(column.na_dtype)
499
+ column[na] = column.na_value
500
+ yield name, column
496
501
 
497
502
  @classmethod
498
503
  def from_json(cls, string, *, columns=[], dtypes={}, **kwargs):
@@ -517,38 +522,29 @@ class DataFrame(dict):
517
522
  return cls(**data)
518
523
 
519
524
  @classmethod
520
- def from_pandas(cls, data, *, strings_as_object=inf, dtypes={}):
525
+ @deco.new_from_generator
526
+ def from_pandas(cls, data, *, dtypes={}):
521
527
  """
522
528
  Return a new data frame from ``pandas.DataFrame`` `data`.
523
529
 
524
- `strings_as_object` is a cutoff point. If any row has more characters
525
- than that, the whole column will use the object data type. This is
526
- intended to help limit memory use as NumPy strings are fixed-length and
527
- can take a huge amount of memory if even a single row is long. If set,
528
- `dtypes` overrides this.
529
-
530
530
  `dtypes` is an optional dict mapping column names to NumPy datatypes.
531
531
  """
532
- if (not isinstance(strings_as_object, (int, float)) or
533
- isinstance(strings_as_object, bool)):
534
- raise TypeError("Expected a number for strings_as_object")
535
- dtypes = dtypes.copy()
536
- from pandas.api.types import is_object_dtype
537
- if strings_as_object < inf:
538
- for name in data.columns:
539
- if name not in dtypes and is_object_dtype(data[name]):
540
- with contextlib.suppress(AttributeError):
541
- if data[name].str.len().max() > strings_as_object:
542
- dtypes[name] = object
543
- data = {x: data[x].to_numpy(copy=True) for x in data.columns}
544
- for name, value in data.items():
545
- # Pandas object columns are likely to be strings,
546
- # convert to list to force type guessing in Vector.__init__.
547
- if np.issubdtype(value.dtype, np.object_):
548
- data[name] = data[name].tolist()
549
- for name, dtype in dtypes.items():
550
- data[name] = DataFrameColumn(data[name], dtype)
551
- return cls(**data)
532
+ for name in data.columns:
533
+ req_dtype = dtypes.get(name, None)
534
+ na = data[name].isna().to_numpy(copy=True)
535
+ column = data[name].to_numpy(copy=True)
536
+ if np.issubdtype(column.dtype, np.object_):
537
+ if req_dtype is None or np.dtype(req_dtype) != np.dtype(object):
538
+ # Likely to be strings, but not necessarily.
539
+ # Force type-guessing in Vector.fast.
540
+ column = column.tolist()
541
+ column = DataFrameColumn.fast(column, req_dtype)
542
+ if na.any():
543
+ if column.dtype != column.na_dtype:
544
+ # Upcast integer to float if needed.
545
+ column = column.astype(column.na_dtype)
546
+ column[na] = column.na_value
547
+ yield name, column
552
548
 
553
549
  def full_join(self, other, *by):
554
550
  """
@@ -746,8 +742,9 @@ class DataFrame(dict):
746
742
  self._check_dimensions()
747
743
  return len(self)
748
744
 
749
- def _new(self, *args, **kwargs):
750
- return self.__class__(*args, **kwargs)
745
+ @classmethod
746
+ def _new(cls, *args, **kwargs):
747
+ return cls(*args, **kwargs)
751
748
 
752
749
  @property
753
750
  def nrow(self):
@@ -875,35 +872,25 @@ class DataFrame(dict):
875
872
  yield colname, total
876
873
 
877
874
  @classmethod
878
- def read_csv(cls, path, *, encoding="utf-8", sep=",", header=True, columns=[], strings_as_object=inf, dtypes={}):
875
+ def read_csv(cls, path, *, encoding="utf-8", sep=",", header=True, columns=[], dtypes={}):
879
876
  """
880
877
  Return a new data frame from CSV file `path`.
881
878
 
882
879
  Will automatically decompress if `path` ends in ``.bz2|.gz|.xz``.
883
-
884
- `columns` is an optional list of columns to limit to.
885
-
886
- `strings_as_object` is a cutoff point. If any row has more characters
887
- than that, the whole column will use the object data type. This is
888
- intended to help limit memory use as NumPy strings are fixed-length and
889
- can take a huge amount of memory if even a single row is long. If set,
890
- `dtypes` overrides this.
891
-
892
- `dtypes` is an optional dict mapping column names to NumPy datatypes.
880
+ `columns` is an optional list of columns to limit to. `dtypes` is an
881
+ optional dict mapping column names to NumPy datatypes.
893
882
  """
894
- import pandas as pd
895
- data = pd.read_csv(path,
896
- sep=sep,
897
- header=0 if header else None,
898
- usecols=columns or None,
899
- dtype=dtypes,
900
- parse_dates=False,
901
- encoding=encoding,
902
- low_memory=False)
883
+ from pyarrow import csv
884
+ table = csv.read_csv(
885
+ path,
886
+ read_options=csv.ReadOptions(encoding=encoding, autogenerate_column_names=not header),
887
+ parse_options=csv.ParseOptions(delimiter=sep, newlines_in_values=True),
888
+ convert_options=csv.ConvertOptions(include_columns=columns))
903
889
 
904
890
  if not header:
905
- data.columns = util.generate_colnames(len(data.columns))
906
- return cls.from_pandas(data, strings_as_object=strings_as_object, dtypes=dtypes)
891
+ names = util.generate_colnames(table.shape[1])
892
+ table = table.rename_columns(names)
893
+ return cls.from_arrow(table, dtypes=dtypes)
907
894
 
908
895
  @classmethod
909
896
  def read_json(cls, path, *, encoding="utf-8", columns=[], dtypes={}, **kwargs):
@@ -911,7 +898,6 @@ class DataFrame(dict):
911
898
  Return a new data frame from JSON file `path`.
912
899
 
913
900
  Will automatically decompress if `path` ends in ``.bz2|.gz|.xz``.
914
-
915
901
  `columns` is an optional list of columns to limit to. `dtypes` is an
916
902
  optional dict mapping column names to NumPy datatypes. `kwargs` are
917
903
  passed to ``json.load``.
@@ -931,24 +917,17 @@ class DataFrame(dict):
931
917
  return cls(**data)
932
918
 
933
919
  @classmethod
934
- def read_parquet(cls, path, *, columns=[], strings_as_object=inf, dtypes={}):
920
+ def read_parquet(cls, path, *, columns=[], dtypes={}):
935
921
  """
936
922
  Return a new data frame from Parquet file `path`.
937
923
 
938
- `columns` is an optional list of columns to limit to.
939
-
940
- `strings_as_object` is a cutoff point. If any row has more characters
941
- than that, the whole column will use the object data type. This is
942
- intended to help limit memory use as NumPy strings are fixed-length and
943
- can take a huge amount of memory if even a single row is long. If set,
944
- `dtypes` overrides this.
945
-
946
- `dtypes` is an optional dict mapping column names to NumPy datatypes.
924
+ `columns` is an optional list of columns to limit to. `dtypes` is an
925
+ optional dict mapping column names to NumPy datatypes.
947
926
  """
948
927
  import pyarrow.parquet as pq
949
928
  columns = columns or None
950
929
  data = pq.read_table(path, columns=columns)
951
- return cls.from_arrow(data, strings_as_object=strings_as_object, dtypes=dtypes)
930
+ return cls.from_arrow(data, dtypes=dtypes)
952
931
 
953
932
  @classmethod
954
933
  def read_pickle(cls, path):
@@ -1076,23 +1055,30 @@ class DataFrame(dict):
1076
1055
  >>> data = di.read_csv("data/listings.csv")
1077
1056
  >>> data.sort(hood=1, zipcode=1)
1078
1057
  """
1079
- @deco.tuplefy
1080
- def sort_key():
1081
- pairs = colname_dir_pairs.items()
1082
- for colname, dir in reversed(list(pairs)):
1083
- if dir not in [1, -1]:
1084
- raise ValueError("dir should be 1 or -1")
1085
- column = self[colname]
1086
- if column.is_object():
1087
- # See Vector.sort for comparison.
1088
- column = column.as_string()
1089
- if column.is_string():
1090
- column[column.is_na()] = "\uffff"
1091
- if dir < 0 and not (column.is_boolean() or column.is_number()):
1092
- # Use rank for non-numeric so that we can sort descending.
1093
- column = column.rank(method="min")
1094
- yield column if dir > 0 else -column
1095
- indices = np.lexsort(sort_key())
1058
+ def sort_key(colname, dir):
1059
+ if dir not in [1, -1]:
1060
+ raise ValueError("dir should be 1 or -1")
1061
+ column = self[colname]
1062
+ column = column._optimize_for_argsort()
1063
+ if column._is_string_fixed():
1064
+ column[column.is_na()] = "\uffff"
1065
+ if dir > 0 and any((
1066
+ # No strings here due to lexsort segfault.
1067
+ # Need to require NumPy >= 2.2.1 to fix this.
1068
+ # https://github.com/numpy/numpy/issues/27984
1069
+ column._is_string_fixed(),
1070
+ column.is_boolean(),
1071
+ column.is_bytes(),
1072
+ column.is_datetime(),
1073
+ column.is_float(),
1074
+ column.is_integer(),
1075
+ column.is_timedelta(),
1076
+ )): return column
1077
+ if not column.is_number():
1078
+ column = column.rank(method="min")
1079
+ return column if dir > 0 else -column
1080
+ indices = np.lexsort(tuple(
1081
+ sort_key(*x) for x in reversed(colname_dir_pairs.items())))
1096
1082
  for colname, column in self.items():
1097
1083
  yield colname, column[indices].copy()
1098
1084
 
@@ -1187,7 +1173,7 @@ class DataFrame(dict):
1187
1173
  n = min(self.nrow, max_rows)
1188
1174
  columns = {colname: util.upad(
1189
1175
  [colname] +
1190
- [str(column.dtype)] +
1176
+ [str(column.dtype_label)] +
1191
1177
  [str(x) for x in column[:n].to_strings(
1192
1178
  quote=False, pad=True, truncate_width=truncate_width)]
1193
1179
  ) for colname, column in self.items()}
@@ -1224,21 +1210,31 @@ class DataFrame(dict):
1224
1210
  >>> data = di.read_csv("data/listings.csv")
1225
1211
  >>> data.unique("hood")
1226
1212
  """
1213
+ # Strangely, standard Python is a lot faster here than np.unique
1214
+ # with all the extra needed across columns of different type.
1215
+ # We just need to avoid NaN and NaT for the 'in' checks to work.
1227
1216
  colnames = colnames or self.colnames
1228
- if (len(colnames) == 1 and
1229
- not self[colnames[0]].is_object()):
1230
- # Use a single column directly.
1231
- by = self[colnames[0]]
1232
- elif (len(set(self[x].dtype for x in colnames)) == 1 and
1233
- not self[colnames[0]].is_object()):
1234
- # Stack matching dtypes directly in a new array.
1235
- by = np.column_stack([self[x] for x in colnames])
1236
- else:
1237
- # Use rank for differing dtypes.
1238
- by = np.column_stack([self[x].rank(method="min") for x in colnames])
1239
- indices = np.sort(np.unique(by, return_index=True, axis=0)[1])
1217
+ columns = [self[x] for x in colnames]
1218
+ for i, column in enumerate(columns):
1219
+ if column.is_datetime():
1220
+ flag = np.nanmin(column) - np.timedelta64(1, "D")
1221
+ if np.isnat(flag):
1222
+ flag = np.datetime64("-001-01-01")
1223
+ columns[i] = column.replace_na(flag)
1224
+ if column.is_float():
1225
+ flag = np.nanmin(column) - 1
1226
+ if np.isnan(flag):
1227
+ flag = -1
1228
+ columns[i] = column.replace_na(flag)
1229
+ rows = list(zip(*columns))
1230
+ seen = set()
1231
+ keep = []
1232
+ for i in range(self.nrow):
1233
+ if rows[i] not in seen:
1234
+ seen.add(rows[i])
1235
+ keep.append(i)
1240
1236
  for colname, column in self.items():
1241
- yield colname, column[indices].copy()
1237
+ yield colname, column[keep].copy()
1242
1238
 
1243
1239
  @deco.new_from_generator
1244
1240
  def unselect(self, *colnames):
@@ -1280,16 +1276,24 @@ class DataFrame(dict):
1280
1276
 
1281
1277
  Will automatically compress if `path` ends in ``.bz2|.gz|.xz``.
1282
1278
  """
1283
- data = self.to_pandas()
1279
+ from pyarrow import csv
1280
+ table = self.to_arrow()
1284
1281
  util.makedirs_for_file(path)
1285
- data.to_csv(path, sep=sep, header=header, index=False, encoding=encoding)
1282
+ csv.write_csv(table, path, write_options=csv.WriteOptions(include_header=header,
1283
+ delimiter=sep,
1284
+ quoting_style="needed"))
1285
+
1286
+ # XXX: Arrow's WriteOptions doesn't include encoding.
1287
+ # If user requested something else than UTF-8, we need to rewrite!
1288
+ if codecs.lookup(encoding) != codecs.lookup("utf-8"):
1289
+ with util.xopen(path, "rt", encoding="utf-8") as f: text = f.read()
1290
+ with util.xopen(path, "wt", encoding=encoding) as f: f.write(text)
1286
1291
 
1287
1292
  def write_json(self, path, *, encoding="utf-8", **kwargs):
1288
1293
  """
1289
1294
  Write data frame to JSON file `path`.
1290
1295
 
1291
1296
  Will automatically compress if `path` ends in ``.bz2|.gz|.xz``.
1292
-
1293
1297
  `kwargs` are passed to ``json.JSONEncoder``.
1294
1298
  """
1295
1299
  return self.to_list_of_dicts().write_json(path, encoding=encoding, **kwargs)
@@ -23,8 +23,10 @@
23
23
  import datetime
24
24
  import numpy as np
25
25
 
26
+ from dataiter import dtypes
26
27
  from dataiter import util
27
28
  from dataiter import Vector
29
+ from numpy.dtypes import StringDType
28
30
 
29
31
  def day(x):
30
32
  """
@@ -46,22 +48,21 @@ def from_string(x, format):
46
48
  >>> dt.from_string(x, "%d.%m.%Y")
47
49
  """
48
50
  if util.is_scalar(x):
49
- x = np.array([x], str)
51
+ x = Vector([x], str)
50
52
  return from_string(x, format)[0]
51
- x = util.sequencify(x)
52
53
  assert isinstance(x, np.ndarray)
53
- assert np.issubdtype(x.dtype, np.str_)
54
+ assert isinstance(x.dtype, StringDType)
54
55
  out = np.full_like(x, None, object)
55
56
  out = Vector.fast(out, object)
56
- na = x == ""
57
+ na = x == dtypes.string.na_object
57
58
  f = np.vectorize(lambda x: datetime.datetime.strptime(x, format))
58
59
  out[~na] = f(x[~na].astype(object))
59
60
  out = out.as_datetime()
60
- if len(out[~na]) > 0:
61
- if (hour(out[~na]) == 0).all():
62
- if (minute(out[~na]) == 0).all():
63
- if (second(out[~na]) == 0).all():
64
- out = out.as_date()
61
+ if (len(out[~na]) > 0 and
62
+ (hour(out[~na]) == 0).all() and
63
+ (minute(out[~na]) == 0).all() and
64
+ (second(out[~na]) == 0).all()):
65
+ out = out.as_date()
65
66
  return out
66
67
 
67
68
  def hour(x):
@@ -146,9 +147,8 @@ def now():
146
147
 
147
148
  def _pull_datetime(x, function):
148
149
  if util.is_scalar(x):
149
- x = np.array([x], np.datetime64)
150
+ x = Vector([x], np.datetime64)
150
151
  return _pull_datetime(x, function)[0]
151
- x = util.sequencify(x)
152
152
  assert isinstance(x, np.ndarray)
153
153
  assert np.issubdtype(x.dtype, np.datetime64)
154
154
  out = np.full_like(x, np.nan)
@@ -161,9 +161,8 @@ def _pull_datetime(x, function):
161
161
 
162
162
  def _pull_int(x, function):
163
163
  if util.is_scalar(x):
164
- x = np.array([x], np.datetime64)
164
+ x = Vector([x], np.datetime64)
165
165
  return _pull_int(x, function)[0]
166
- x = util.sequencify(x)
167
166
  assert isinstance(x, np.ndarray)
168
167
  assert np.issubdtype(x.dtype, np.datetime64)
169
168
  out = np.full_like(x, np.nan, float)
@@ -176,12 +175,11 @@ def _pull_int(x, function):
176
175
 
177
176
  def _pull_str(x, function):
178
177
  if util.is_scalar(x):
179
- x = np.array([x], np.datetime64)
178
+ x = Vector([x], np.datetime64)
180
179
  return _pull_str(x, function)[0]
181
- x = util.sequencify(x)
182
180
  assert isinstance(x, np.ndarray)
183
181
  assert np.issubdtype(x.dtype, np.datetime64)
184
- out = np.full_like(x, "", object)
182
+ out = np.full_like(x, dtypes.string.na_object, object)
185
183
  out = Vector.fast(out, object)
186
184
  na = np.isnat(x)
187
185
  if na.all(): return out
@@ -214,7 +212,6 @@ def replace(x, year=None, month=None, day=None, hour=None, minute=None, second=N
214
212
  scalar_keys = [x for x in kwargs if util.is_scalar(kwargs[x])]
215
213
  vector_keys = [x for x in kwargs if x not in scalar_keys]
216
214
  # Like _pull_datetime, but no vectorized function.
217
- x = util.sequencify(x)
218
215
  assert isinstance(x, np.ndarray)
219
216
  assert np.issubdtype(x.dtype, np.datetime64)
220
217
  out = np.full_like(x, np.nan)