dataiter 0.99__tar.gz → 0.9999__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataiter-0.99 → dataiter-0.9999}/.gitignore +2 -2
- {dataiter-0.99 → dataiter-0.9999}/COPYING +1 -1
- dataiter-0.9999/PKG-INFO +89 -0
- dataiter-0.9999/README.md +73 -0
- {dataiter-0.99 → dataiter-0.9999}/dataiter/__init__.py +21 -1
- {dataiter-0.99 → dataiter-0.9999}/dataiter/data_frame.py +115 -111
- {dataiter-0.99 → dataiter-0.9999}/dataiter/dt.py +14 -17
- dataiter-0.9999/dataiter/dtypes.py +32 -0
- {dataiter-0.99 → dataiter-0.9999}/dataiter/geojson.py +13 -38
- {dataiter-0.99 → dataiter-0.9999}/dataiter/io.py +4 -7
- {dataiter-0.99 → dataiter-0.9999}/dataiter/list_of_dicts.py +1 -17
- dataiter-0.9999/dataiter/regex.py +148 -0
- {dataiter-0.99 → dataiter-0.9999}/dataiter/test/__init__.py +11 -9
- {dataiter-0.99 → dataiter-0.9999}/dataiter/test/test_aggregate.py +2 -2
- {dataiter-0.99 → dataiter-0.9999}/dataiter/test/test_data_frame.py +72 -5
- {dataiter-0.99 → dataiter-0.9999}/dataiter/test/test_geojson.py +0 -6
- {dataiter-0.99 → dataiter-0.9999}/dataiter/test/test_io.py +5 -0
- {dataiter-0.99 → dataiter-0.9999}/dataiter/test/test_list_of_dicts.py +0 -6
- dataiter-0.9999/dataiter/test/test_regex.py +92 -0
- {dataiter-0.99 → dataiter-0.9999}/dataiter/test/test_vector.py +35 -33
- {dataiter-0.99 → dataiter-0.9999}/dataiter/util.py +1 -3
- {dataiter-0.99 → dataiter-0.9999}/dataiter/vector.py +244 -74
- {dataiter-0.99 → dataiter-0.9999}/pyproject.toml +2 -2
- dataiter-0.99/PKG-INFO +0 -72
- dataiter-0.99/README.md +0 -56
- {dataiter-0.99 → dataiter-0.9999}/AUTHORS.md +0 -0
- {dataiter-0.99 → dataiter-0.9999}/dataiter/aggregate.py +0 -0
- {dataiter-0.99 → dataiter-0.9999}/dataiter/deco.py +0 -0
- {dataiter-0.99 → dataiter-0.9999}/dataiter/test/test_dt.py +0 -0
- {dataiter-0.99 → dataiter-0.9999}/dataiter/test/test_util.py +0 -0
dataiter-0.9999/PKG-INFO
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: dataiter
|
|
3
|
+
Version: 0.9999
|
|
4
|
+
Summary: Simple, light-weight data frames for Python
|
|
5
|
+
Project-URL: Homepage, https://github.com/otsaloma/dataiter
|
|
6
|
+
Author-email: Osmo Salomaa <otsaloma@iki.fi>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: AUTHORS.md
|
|
9
|
+
License-File: COPYING
|
|
10
|
+
Requires-Python: >=3.9.0
|
|
11
|
+
Requires-Dist: attd>=0.3
|
|
12
|
+
Requires-Dist: numpy<3.0,>=2.0
|
|
13
|
+
Requires-Dist: pyarrow>=2.0
|
|
14
|
+
Requires-Dist: wcwidth>=0.1
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
Simple, Light-Weight Data Frames for Python
|
|
18
|
+
===========================================
|
|
19
|
+
|
|
20
|
+
[](https://pypi.org/project/dataiter)
|
|
21
|
+
[](https://pepy.tech/project/dataiter)
|
|
22
|
+
|
|
23
|
+
Dataiter's **`DataFrame`** is a class for tabular data similar to R's
|
|
24
|
+
`data.frame`, implementing all common operations to manipulate data. It
|
|
25
|
+
is under the hood a dictionary of NumPy arrays and thus capable of fast
|
|
26
|
+
vectorized operations. You can consider it to be a light-weight
|
|
27
|
+
alternative to Pandas with a simple and consistent API. Performance-wise
|
|
28
|
+
Dataiter relies on NumPy and Numba and is likely to be at best
|
|
29
|
+
comparable to Pandas.
|
|
30
|
+
|
|
31
|
+
## Installation
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
# Latest stable version
|
|
35
|
+
pip install -U dataiter
|
|
36
|
+
|
|
37
|
+
# Latest development version
|
|
38
|
+
pip install -U git+https://github.com/otsaloma/dataiter
|
|
39
|
+
|
|
40
|
+
# Numba (optional)
|
|
41
|
+
pip install -U numba
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Dataiter optionally uses **Numba** to speed up certain operations. If
|
|
45
|
+
you have Numba installed, Dataiter will use it automatically. It's
|
|
46
|
+
currently not a hard dependency, so you need to install it separately.
|
|
47
|
+
|
|
48
|
+
## Quick Start
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
>>> import dataiter as di
|
|
52
|
+
>>> data = di.read_csv("data/listings.csv")
|
|
53
|
+
>>> data.filter(hood="Manhattan", guests=2).sort(price=1).head()
|
|
54
|
+
.
|
|
55
|
+
id hood zipcode guests sqft price
|
|
56
|
+
int64 string string int64 float64 int64
|
|
57
|
+
──────── ───────── ─────── ────── ─────── ─────
|
|
58
|
+
0 42279170 Manhattan 10013 2 nan 0
|
|
59
|
+
1 42384530 Manhattan 10036 2 nan 0
|
|
60
|
+
2 18835820 Manhattan 10021 2 nan 10
|
|
61
|
+
3 20171179 Manhattan 10027 2 nan 10
|
|
62
|
+
4 14858544 Manhattan 2 nan 15
|
|
63
|
+
5 31397084 Manhattan 10002 2 nan 19
|
|
64
|
+
6 22289683 Manhattan 10031 2 nan 20
|
|
65
|
+
7 7760204 Manhattan 10040 2 nan 22
|
|
66
|
+
8 43292527 Manhattan 10033 2 nan 22
|
|
67
|
+
9 43268040 Manhattan 10033 2 nan 23
|
|
68
|
+
.
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Documentation
|
|
72
|
+
|
|
73
|
+
https://dataiter.readthedocs.io/
|
|
74
|
+
|
|
75
|
+
If you're familiar with either dplyr (R) or Pandas (Python), the
|
|
76
|
+
comparison table in the documentation will give you a quick overview of
|
|
77
|
+
the differences and similarities in common operations.
|
|
78
|
+
|
|
79
|
+
https://dataiter.readthedocs.io/en/stable/comparison.html
|
|
80
|
+
|
|
81
|
+
## Development
|
|
82
|
+
|
|
83
|
+
To install a virtualenv for development, use
|
|
84
|
+
|
|
85
|
+
make venv
|
|
86
|
+
|
|
87
|
+
or, for a specific Python version
|
|
88
|
+
|
|
89
|
+
make PYTHON=python3.X venv
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
Simple, Light-Weight Data Frames for Python
|
|
2
|
+
===========================================
|
|
3
|
+
|
|
4
|
+
[](https://pypi.org/project/dataiter)
|
|
5
|
+
[](https://pepy.tech/project/dataiter)
|
|
6
|
+
|
|
7
|
+
Dataiter's **`DataFrame`** is a class for tabular data similar to R's
|
|
8
|
+
`data.frame`, implementing all common operations to manipulate data. It
|
|
9
|
+
is under the hood a dictionary of NumPy arrays and thus capable of fast
|
|
10
|
+
vectorized operations. You can consider it to be a light-weight
|
|
11
|
+
alternative to Pandas with a simple and consistent API. Performance-wise
|
|
12
|
+
Dataiter relies on NumPy and Numba and is likely to be at best
|
|
13
|
+
comparable to Pandas.
|
|
14
|
+
|
|
15
|
+
## Installation
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
# Latest stable version
|
|
19
|
+
pip install -U dataiter
|
|
20
|
+
|
|
21
|
+
# Latest development version
|
|
22
|
+
pip install -U git+https://github.com/otsaloma/dataiter
|
|
23
|
+
|
|
24
|
+
# Numba (optional)
|
|
25
|
+
pip install -U numba
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Dataiter optionally uses **Numba** to speed up certain operations. If
|
|
29
|
+
you have Numba installed, Dataiter will use it automatically. It's
|
|
30
|
+
currently not a hard dependency, so you need to install it separately.
|
|
31
|
+
|
|
32
|
+
## Quick Start
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
>>> import dataiter as di
|
|
36
|
+
>>> data = di.read_csv("data/listings.csv")
|
|
37
|
+
>>> data.filter(hood="Manhattan", guests=2).sort(price=1).head()
|
|
38
|
+
.
|
|
39
|
+
id hood zipcode guests sqft price
|
|
40
|
+
int64 string string int64 float64 int64
|
|
41
|
+
──────── ───────── ─────── ────── ─────── ─────
|
|
42
|
+
0 42279170 Manhattan 10013 2 nan 0
|
|
43
|
+
1 42384530 Manhattan 10036 2 nan 0
|
|
44
|
+
2 18835820 Manhattan 10021 2 nan 10
|
|
45
|
+
3 20171179 Manhattan 10027 2 nan 10
|
|
46
|
+
4 14858544 Manhattan 2 nan 15
|
|
47
|
+
5 31397084 Manhattan 10002 2 nan 19
|
|
48
|
+
6 22289683 Manhattan 10031 2 nan 20
|
|
49
|
+
7 7760204 Manhattan 10040 2 nan 22
|
|
50
|
+
8 43292527 Manhattan 10033 2 nan 22
|
|
51
|
+
9 43268040 Manhattan 10033 2 nan 23
|
|
52
|
+
.
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Documentation
|
|
56
|
+
|
|
57
|
+
https://dataiter.readthedocs.io/
|
|
58
|
+
|
|
59
|
+
If you're familiar with either dplyr (R) or Pandas (Python), the
|
|
60
|
+
comparison table in the documentation will give you a quick overview of
|
|
61
|
+
the differences and similarities in common operations.
|
|
62
|
+
|
|
63
|
+
https://dataiter.readthedocs.io/en/stable/comparison.html
|
|
64
|
+
|
|
65
|
+
## Development
|
|
66
|
+
|
|
67
|
+
To install a virtualenv for development, use
|
|
68
|
+
|
|
69
|
+
make venv
|
|
70
|
+
|
|
71
|
+
or, for a specific Python version
|
|
72
|
+
|
|
73
|
+
make PYTHON=python3.X venv
|
|
@@ -25,7 +25,7 @@ import numpy as np
|
|
|
25
25
|
|
|
26
26
|
from dataiter import util
|
|
27
27
|
|
|
28
|
-
__version__ = "0.
|
|
28
|
+
__version__ = "0.9999"
|
|
29
29
|
|
|
30
30
|
DEFAULT_PEEK_ELEMENTS = 10
|
|
31
31
|
DEFAULT_PEEK_ITEMS = 3
|
|
@@ -34,10 +34,28 @@ PRINT_FLOAT_PRECISION = 6
|
|
|
34
34
|
PRINT_MAX_ELEMENTS = 100
|
|
35
35
|
PRINT_MAX_ITEMS = 10
|
|
36
36
|
PRINT_MAX_ROWS = 100
|
|
37
|
+
|
|
38
|
+
#: Maximum amount of columns to wrap print output to. This is only a fallback
|
|
39
|
+
#: in case Python's ``shutil.get_terminal_size`` fails to detect the width of
|
|
40
|
+
#: your terminal. By default the detected full width is used.
|
|
37
41
|
PRINT_MAX_WIDTH = 80
|
|
42
|
+
|
|
43
|
+
#: Thousand separator to use when printing numbers. By default this is blank,
|
|
44
|
+
#: meaning no thousand separators are rendered.
|
|
38
45
|
PRINT_THOUSAND_SEPARATOR = ""
|
|
46
|
+
|
|
47
|
+
#: Maximum width to truncate string columns to in :class:`DataFrame` print
|
|
48
|
+
#: output. When this is exceeded, strings will be cut and an ellipsis (``…``)
|
|
49
|
+
#: rendered at the cut point.
|
|
39
50
|
PRINT_TRUNCATE_WIDTH = 36
|
|
51
|
+
|
|
52
|
+
#: ``True`` to use Numba, if available, to speed up :doc:`aggregations
|
|
53
|
+
#: </aggregation>`, ``False`` to only use pure Python code.
|
|
40
54
|
USE_NUMBA = False
|
|
55
|
+
|
|
56
|
+
#: ``True`` to use Numba cache for JIT-compiled :doc:`aggregations
|
|
57
|
+
#: </aggregation>`, ``False`` to only keep compiled code in memory for the
|
|
58
|
+
#: duration of the session.
|
|
41
59
|
USE_NUMBA_CACHE = True
|
|
42
60
|
|
|
43
61
|
if not np.__version__.startswith("2."):
|
|
@@ -69,12 +87,14 @@ globals().pop("np", None)
|
|
|
69
87
|
globals().pop("numba", None)
|
|
70
88
|
globals().pop("util", None)
|
|
71
89
|
|
|
90
|
+
from dataiter import dtypes # noqa
|
|
72
91
|
from dataiter.vector import Vector # noqa
|
|
73
92
|
from dataiter.data_frame import DataFrame # noqa
|
|
74
93
|
from dataiter.data_frame import DataFrameColumn # noqa
|
|
75
94
|
from dataiter.geojson import GeoJSON # noqa
|
|
76
95
|
from dataiter.list_of_dicts import ListOfDicts # noqa
|
|
77
96
|
from dataiter import dt # noqa
|
|
97
|
+
from dataiter import regex # noqa
|
|
78
98
|
|
|
79
99
|
from dataiter.aggregate import all # noqa
|
|
80
100
|
from dataiter.aggregate import any # noqa
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
21
|
# THE SOFTWARE.
|
|
22
22
|
|
|
23
|
-
import
|
|
23
|
+
import codecs
|
|
24
24
|
import dataiter
|
|
25
25
|
import functools
|
|
26
26
|
import itertools
|
|
@@ -475,24 +475,29 @@ class DataFrame(dict):
|
|
|
475
475
|
yield colname, np.delete(column, rows)
|
|
476
476
|
|
|
477
477
|
@classmethod
|
|
478
|
-
|
|
478
|
+
@deco.new_from_generator
|
|
479
|
+
def from_arrow(cls, data, *, dtypes={}):
|
|
479
480
|
"""
|
|
480
481
|
Return a new data frame from ``pyarrow.Table`` `data`.
|
|
481
482
|
|
|
482
|
-
`strings_as_object` is a cutoff point. If any row has more characters
|
|
483
|
-
than that, the whole column will use the object data type. This is
|
|
484
|
-
intended to help limit memory use as NumPy strings are fixed-length and
|
|
485
|
-
can take a huge amount of memory if even a single row is long. If set,
|
|
486
|
-
`dtypes` overrides this.
|
|
487
|
-
|
|
488
483
|
`dtypes` is an optional dict mapping column names to NumPy datatypes.
|
|
489
484
|
"""
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
485
|
+
for name, column in zip(data.column_names, data.columns):
|
|
486
|
+
req_dtype = dtypes.get(name, None)
|
|
487
|
+
na = column.is_null(nan_is_null=True).to_numpy()
|
|
488
|
+
column = column.to_numpy()
|
|
489
|
+
if np.issubdtype(column.dtype, np.object_):
|
|
490
|
+
if req_dtype is None or np.dtype(req_dtype) != np.dtype(object):
|
|
491
|
+
# Likely to be strings, but not necessarily.
|
|
492
|
+
# Force type-guessing in Vector.fast.
|
|
493
|
+
column = column.tolist()
|
|
494
|
+
column = DataFrameColumn.fast(column, req_dtype)
|
|
495
|
+
if na.any():
|
|
496
|
+
if column.dtype != column.na_dtype:
|
|
497
|
+
# Upcast integer to float if needed.
|
|
498
|
+
column = column.astype(column.na_dtype)
|
|
499
|
+
column[na] = column.na_value
|
|
500
|
+
yield name, column
|
|
496
501
|
|
|
497
502
|
@classmethod
|
|
498
503
|
def from_json(cls, string, *, columns=[], dtypes={}, **kwargs):
|
|
@@ -517,38 +522,29 @@ class DataFrame(dict):
|
|
|
517
522
|
return cls(**data)
|
|
518
523
|
|
|
519
524
|
@classmethod
|
|
520
|
-
|
|
525
|
+
@deco.new_from_generator
|
|
526
|
+
def from_pandas(cls, data, *, dtypes={}):
|
|
521
527
|
"""
|
|
522
528
|
Return a new data frame from ``pandas.DataFrame`` `data`.
|
|
523
529
|
|
|
524
|
-
`strings_as_object` is a cutoff point. If any row has more characters
|
|
525
|
-
than that, the whole column will use the object data type. This is
|
|
526
|
-
intended to help limit memory use as NumPy strings are fixed-length and
|
|
527
|
-
can take a huge amount of memory if even a single row is long. If set,
|
|
528
|
-
`dtypes` overrides this.
|
|
529
|
-
|
|
530
530
|
`dtypes` is an optional dict mapping column names to NumPy datatypes.
|
|
531
531
|
"""
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
data[name] = data[name].tolist()
|
|
549
|
-
for name, dtype in dtypes.items():
|
|
550
|
-
data[name] = DataFrameColumn(data[name], dtype)
|
|
551
|
-
return cls(**data)
|
|
532
|
+
for name in data.columns:
|
|
533
|
+
req_dtype = dtypes.get(name, None)
|
|
534
|
+
na = data[name].isna().to_numpy(copy=True)
|
|
535
|
+
column = data[name].to_numpy(copy=True)
|
|
536
|
+
if np.issubdtype(column.dtype, np.object_):
|
|
537
|
+
if req_dtype is None or np.dtype(req_dtype) != np.dtype(object):
|
|
538
|
+
# Likely to be strings, but not necessarily.
|
|
539
|
+
# Force type-guessing in Vector.fast.
|
|
540
|
+
column = column.tolist()
|
|
541
|
+
column = DataFrameColumn.fast(column, req_dtype)
|
|
542
|
+
if na.any():
|
|
543
|
+
if column.dtype != column.na_dtype:
|
|
544
|
+
# Upcast integer to float if needed.
|
|
545
|
+
column = column.astype(column.na_dtype)
|
|
546
|
+
column[na] = column.na_value
|
|
547
|
+
yield name, column
|
|
552
548
|
|
|
553
549
|
def full_join(self, other, *by):
|
|
554
550
|
"""
|
|
@@ -746,8 +742,9 @@ class DataFrame(dict):
|
|
|
746
742
|
self._check_dimensions()
|
|
747
743
|
return len(self)
|
|
748
744
|
|
|
749
|
-
|
|
750
|
-
|
|
745
|
+
@classmethod
|
|
746
|
+
def _new(cls, *args, **kwargs):
|
|
747
|
+
return cls(*args, **kwargs)
|
|
751
748
|
|
|
752
749
|
@property
|
|
753
750
|
def nrow(self):
|
|
@@ -875,35 +872,25 @@ class DataFrame(dict):
|
|
|
875
872
|
yield colname, total
|
|
876
873
|
|
|
877
874
|
@classmethod
|
|
878
|
-
def read_csv(cls, path, *, encoding="utf-8", sep=",", header=True, columns=[],
|
|
875
|
+
def read_csv(cls, path, *, encoding="utf-8", sep=",", header=True, columns=[], dtypes={}):
|
|
879
876
|
"""
|
|
880
877
|
Return a new data frame from CSV file `path`.
|
|
881
878
|
|
|
882
879
|
Will automatically decompress if `path` ends in ``.bz2|.gz|.xz``.
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
`strings_as_object` is a cutoff point. If any row has more characters
|
|
887
|
-
than that, the whole column will use the object data type. This is
|
|
888
|
-
intended to help limit memory use as NumPy strings are fixed-length and
|
|
889
|
-
can take a huge amount of memory if even a single row is long. If set,
|
|
890
|
-
`dtypes` overrides this.
|
|
891
|
-
|
|
892
|
-
`dtypes` is an optional dict mapping column names to NumPy datatypes.
|
|
880
|
+
`columns` is an optional list of columns to limit to. `dtypes` is an
|
|
881
|
+
optional dict mapping column names to NumPy datatypes.
|
|
893
882
|
"""
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
parse_dates=False,
|
|
901
|
-
encoding=encoding,
|
|
902
|
-
low_memory=False)
|
|
883
|
+
from pyarrow import csv
|
|
884
|
+
table = csv.read_csv(
|
|
885
|
+
path,
|
|
886
|
+
read_options=csv.ReadOptions(encoding=encoding, autogenerate_column_names=not header),
|
|
887
|
+
parse_options=csv.ParseOptions(delimiter=sep, newlines_in_values=True),
|
|
888
|
+
convert_options=csv.ConvertOptions(include_columns=columns))
|
|
903
889
|
|
|
904
890
|
if not header:
|
|
905
|
-
|
|
906
|
-
|
|
891
|
+
names = util.generate_colnames(table.shape[1])
|
|
892
|
+
table = table.rename_columns(names)
|
|
893
|
+
return cls.from_arrow(table, dtypes=dtypes)
|
|
907
894
|
|
|
908
895
|
@classmethod
|
|
909
896
|
def read_json(cls, path, *, encoding="utf-8", columns=[], dtypes={}, **kwargs):
|
|
@@ -911,7 +898,6 @@ class DataFrame(dict):
|
|
|
911
898
|
Return a new data frame from JSON file `path`.
|
|
912
899
|
|
|
913
900
|
Will automatically decompress if `path` ends in ``.bz2|.gz|.xz``.
|
|
914
|
-
|
|
915
901
|
`columns` is an optional list of columns to limit to. `dtypes` is an
|
|
916
902
|
optional dict mapping column names to NumPy datatypes. `kwargs` are
|
|
917
903
|
passed to ``json.load``.
|
|
@@ -931,24 +917,17 @@ class DataFrame(dict):
|
|
|
931
917
|
return cls(**data)
|
|
932
918
|
|
|
933
919
|
@classmethod
|
|
934
|
-
def read_parquet(cls, path, *, columns=[],
|
|
920
|
+
def read_parquet(cls, path, *, columns=[], dtypes={}):
|
|
935
921
|
"""
|
|
936
922
|
Return a new data frame from Parquet file `path`.
|
|
937
923
|
|
|
938
|
-
`columns` is an optional list of columns to limit to.
|
|
939
|
-
|
|
940
|
-
`strings_as_object` is a cutoff point. If any row has more characters
|
|
941
|
-
than that, the whole column will use the object data type. This is
|
|
942
|
-
intended to help limit memory use as NumPy strings are fixed-length and
|
|
943
|
-
can take a huge amount of memory if even a single row is long. If set,
|
|
944
|
-
`dtypes` overrides this.
|
|
945
|
-
|
|
946
|
-
`dtypes` is an optional dict mapping column names to NumPy datatypes.
|
|
924
|
+
`columns` is an optional list of columns to limit to. `dtypes` is an
|
|
925
|
+
optional dict mapping column names to NumPy datatypes.
|
|
947
926
|
"""
|
|
948
927
|
import pyarrow.parquet as pq
|
|
949
928
|
columns = columns or None
|
|
950
929
|
data = pq.read_table(path, columns=columns)
|
|
951
|
-
return cls.from_arrow(data,
|
|
930
|
+
return cls.from_arrow(data, dtypes=dtypes)
|
|
952
931
|
|
|
953
932
|
@classmethod
|
|
954
933
|
def read_pickle(cls, path):
|
|
@@ -1076,23 +1055,30 @@ class DataFrame(dict):
|
|
|
1076
1055
|
>>> data = di.read_csv("data/listings.csv")
|
|
1077
1056
|
>>> data.sort(hood=1, zipcode=1)
|
|
1078
1057
|
"""
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
column =
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1058
|
+
def sort_key(colname, dir):
|
|
1059
|
+
if dir not in [1, -1]:
|
|
1060
|
+
raise ValueError("dir should be 1 or -1")
|
|
1061
|
+
column = self[colname]
|
|
1062
|
+
column = column._optimize_for_argsort()
|
|
1063
|
+
if column._is_string_fixed():
|
|
1064
|
+
column[column.is_na()] = "\uffff"
|
|
1065
|
+
if dir > 0 and any((
|
|
1066
|
+
# No strings here due to lexsort segfault.
|
|
1067
|
+
# Need to require NumPy >= 2.2.1 to fix this.
|
|
1068
|
+
# https://github.com/numpy/numpy/issues/27984
|
|
1069
|
+
column._is_string_fixed(),
|
|
1070
|
+
column.is_boolean(),
|
|
1071
|
+
column.is_bytes(),
|
|
1072
|
+
column.is_datetime(),
|
|
1073
|
+
column.is_float(),
|
|
1074
|
+
column.is_integer(),
|
|
1075
|
+
column.is_timedelta(),
|
|
1076
|
+
)): return column
|
|
1077
|
+
if not column.is_number():
|
|
1078
|
+
column = column.rank(method="min")
|
|
1079
|
+
return column if dir > 0 else -column
|
|
1080
|
+
indices = np.lexsort(tuple(
|
|
1081
|
+
sort_key(*x) for x in reversed(colname_dir_pairs.items())))
|
|
1096
1082
|
for colname, column in self.items():
|
|
1097
1083
|
yield colname, column[indices].copy()
|
|
1098
1084
|
|
|
@@ -1187,7 +1173,7 @@ class DataFrame(dict):
|
|
|
1187
1173
|
n = min(self.nrow, max_rows)
|
|
1188
1174
|
columns = {colname: util.upad(
|
|
1189
1175
|
[colname] +
|
|
1190
|
-
[str(column.
|
|
1176
|
+
[str(column.dtype_label)] +
|
|
1191
1177
|
[str(x) for x in column[:n].to_strings(
|
|
1192
1178
|
quote=False, pad=True, truncate_width=truncate_width)]
|
|
1193
1179
|
) for colname, column in self.items()}
|
|
@@ -1224,21 +1210,31 @@ class DataFrame(dict):
|
|
|
1224
1210
|
>>> data = di.read_csv("data/listings.csv")
|
|
1225
1211
|
>>> data.unique("hood")
|
|
1226
1212
|
"""
|
|
1213
|
+
# Strangely, standard Python is a lot faster here than np.unique
|
|
1214
|
+
# with all the extra needed across columns of different type.
|
|
1215
|
+
# We just need to avoid NaN and NaT for the 'in' checks to work.
|
|
1227
1216
|
colnames = colnames or self.colnames
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1217
|
+
columns = [self[x] for x in colnames]
|
|
1218
|
+
for i, column in enumerate(columns):
|
|
1219
|
+
if column.is_datetime():
|
|
1220
|
+
flag = np.nanmin(column) - np.timedelta64(1, "D")
|
|
1221
|
+
if np.isnat(flag):
|
|
1222
|
+
flag = np.datetime64("-001-01-01")
|
|
1223
|
+
columns[i] = column.replace_na(flag)
|
|
1224
|
+
if column.is_float():
|
|
1225
|
+
flag = np.nanmin(column) - 1
|
|
1226
|
+
if np.isnan(flag):
|
|
1227
|
+
flag = -1
|
|
1228
|
+
columns[i] = column.replace_na(flag)
|
|
1229
|
+
rows = list(zip(*columns))
|
|
1230
|
+
seen = set()
|
|
1231
|
+
keep = []
|
|
1232
|
+
for i in range(self.nrow):
|
|
1233
|
+
if rows[i] not in seen:
|
|
1234
|
+
seen.add(rows[i])
|
|
1235
|
+
keep.append(i)
|
|
1240
1236
|
for colname, column in self.items():
|
|
1241
|
-
yield colname, column[
|
|
1237
|
+
yield colname, column[keep].copy()
|
|
1242
1238
|
|
|
1243
1239
|
@deco.new_from_generator
|
|
1244
1240
|
def unselect(self, *colnames):
|
|
@@ -1280,16 +1276,24 @@ class DataFrame(dict):
|
|
|
1280
1276
|
|
|
1281
1277
|
Will automatically compress if `path` ends in ``.bz2|.gz|.xz``.
|
|
1282
1278
|
"""
|
|
1283
|
-
|
|
1279
|
+
from pyarrow import csv
|
|
1280
|
+
table = self.to_arrow()
|
|
1284
1281
|
util.makedirs_for_file(path)
|
|
1285
|
-
|
|
1282
|
+
csv.write_csv(table, path, write_options=csv.WriteOptions(include_header=header,
|
|
1283
|
+
delimiter=sep,
|
|
1284
|
+
quoting_style="needed"))
|
|
1285
|
+
|
|
1286
|
+
# XXX: Arrow's WriteOptions doesn't include encoding.
|
|
1287
|
+
# If user requested something else than UTF-8, we need to rewrite!
|
|
1288
|
+
if codecs.lookup(encoding) != codecs.lookup("utf-8"):
|
|
1289
|
+
with util.xopen(path, "rt", encoding="utf-8") as f: text = f.read()
|
|
1290
|
+
with util.xopen(path, "wt", encoding=encoding) as f: f.write(text)
|
|
1286
1291
|
|
|
1287
1292
|
def write_json(self, path, *, encoding="utf-8", **kwargs):
|
|
1288
1293
|
"""
|
|
1289
1294
|
Write data frame to JSON file `path`.
|
|
1290
1295
|
|
|
1291
1296
|
Will automatically compress if `path` ends in ``.bz2|.gz|.xz``.
|
|
1292
|
-
|
|
1293
1297
|
`kwargs` are passed to ``json.JSONEncoder``.
|
|
1294
1298
|
"""
|
|
1295
1299
|
return self.to_list_of_dicts().write_json(path, encoding=encoding, **kwargs)
|
|
@@ -23,8 +23,10 @@
|
|
|
23
23
|
import datetime
|
|
24
24
|
import numpy as np
|
|
25
25
|
|
|
26
|
+
from dataiter import dtypes
|
|
26
27
|
from dataiter import util
|
|
27
28
|
from dataiter import Vector
|
|
29
|
+
from numpy.dtypes import StringDType
|
|
28
30
|
|
|
29
31
|
def day(x):
|
|
30
32
|
"""
|
|
@@ -46,22 +48,21 @@ def from_string(x, format):
|
|
|
46
48
|
>>> dt.from_string(x, "%d.%m.%Y")
|
|
47
49
|
"""
|
|
48
50
|
if util.is_scalar(x):
|
|
49
|
-
x =
|
|
51
|
+
x = Vector([x], str)
|
|
50
52
|
return from_string(x, format)[0]
|
|
51
|
-
x = util.sequencify(x)
|
|
52
53
|
assert isinstance(x, np.ndarray)
|
|
53
|
-
assert
|
|
54
|
+
assert isinstance(x.dtype, StringDType)
|
|
54
55
|
out = np.full_like(x, None, object)
|
|
55
56
|
out = Vector.fast(out, object)
|
|
56
|
-
na = x ==
|
|
57
|
+
na = x == dtypes.string.na_object
|
|
57
58
|
f = np.vectorize(lambda x: datetime.datetime.strptime(x, format))
|
|
58
59
|
out[~na] = f(x[~na].astype(object))
|
|
59
60
|
out = out.as_datetime()
|
|
60
|
-
if len(out[~na]) > 0
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
61
|
+
if (len(out[~na]) > 0 and
|
|
62
|
+
(hour(out[~na]) == 0).all() and
|
|
63
|
+
(minute(out[~na]) == 0).all() and
|
|
64
|
+
(second(out[~na]) == 0).all()):
|
|
65
|
+
out = out.as_date()
|
|
65
66
|
return out
|
|
66
67
|
|
|
67
68
|
def hour(x):
|
|
@@ -146,9 +147,8 @@ def now():
|
|
|
146
147
|
|
|
147
148
|
def _pull_datetime(x, function):
|
|
148
149
|
if util.is_scalar(x):
|
|
149
|
-
x =
|
|
150
|
+
x = Vector([x], np.datetime64)
|
|
150
151
|
return _pull_datetime(x, function)[0]
|
|
151
|
-
x = util.sequencify(x)
|
|
152
152
|
assert isinstance(x, np.ndarray)
|
|
153
153
|
assert np.issubdtype(x.dtype, np.datetime64)
|
|
154
154
|
out = np.full_like(x, np.nan)
|
|
@@ -161,9 +161,8 @@ def _pull_datetime(x, function):
|
|
|
161
161
|
|
|
162
162
|
def _pull_int(x, function):
|
|
163
163
|
if util.is_scalar(x):
|
|
164
|
-
x =
|
|
164
|
+
x = Vector([x], np.datetime64)
|
|
165
165
|
return _pull_int(x, function)[0]
|
|
166
|
-
x = util.sequencify(x)
|
|
167
166
|
assert isinstance(x, np.ndarray)
|
|
168
167
|
assert np.issubdtype(x.dtype, np.datetime64)
|
|
169
168
|
out = np.full_like(x, np.nan, float)
|
|
@@ -176,12 +175,11 @@ def _pull_int(x, function):
|
|
|
176
175
|
|
|
177
176
|
def _pull_str(x, function):
|
|
178
177
|
if util.is_scalar(x):
|
|
179
|
-
x =
|
|
178
|
+
x = Vector([x], np.datetime64)
|
|
180
179
|
return _pull_str(x, function)[0]
|
|
181
|
-
x = util.sequencify(x)
|
|
182
180
|
assert isinstance(x, np.ndarray)
|
|
183
181
|
assert np.issubdtype(x.dtype, np.datetime64)
|
|
184
|
-
out = np.full_like(x,
|
|
182
|
+
out = np.full_like(x, dtypes.string.na_object, object)
|
|
185
183
|
out = Vector.fast(out, object)
|
|
186
184
|
na = np.isnat(x)
|
|
187
185
|
if na.all(): return out
|
|
@@ -214,7 +212,6 @@ def replace(x, year=None, month=None, day=None, hour=None, minute=None, second=N
|
|
|
214
212
|
scalar_keys = [x for x in kwargs if util.is_scalar(kwargs[x])]
|
|
215
213
|
vector_keys = [x for x in kwargs if x not in scalar_keys]
|
|
216
214
|
# Like _pull_datetime, but no vectorized function.
|
|
217
|
-
x = util.sequencify(x)
|
|
218
215
|
assert isinstance(x, np.ndarray)
|
|
219
216
|
assert np.issubdtype(x.dtype, np.datetime64)
|
|
220
217
|
out = np.full_like(x, np.nan)
|