dataiter 0.99__tar.gz → 0.999__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataiter-0.99 → dataiter-0.999}/.gitignore +2 -2
- dataiter-0.999/PKG-INFO +89 -0
- dataiter-0.999/README.md +73 -0
- {dataiter-0.99 → dataiter-0.999}/dataiter/__init__.py +20 -1
- {dataiter-0.99 → dataiter-0.999}/dataiter/data_frame.py +59 -85
- {dataiter-0.99 → dataiter-0.999}/dataiter/dt.py +14 -12
- dataiter-0.999/dataiter/dtypes.py +32 -0
- {dataiter-0.99 → dataiter-0.999}/dataiter/geojson.py +13 -38
- {dataiter-0.99 → dataiter-0.999}/dataiter/io.py +4 -7
- {dataiter-0.99 → dataiter-0.999}/dataiter/list_of_dicts.py +1 -17
- {dataiter-0.99 → dataiter-0.999}/dataiter/test/__init__.py +11 -9
- {dataiter-0.99 → dataiter-0.999}/dataiter/test/test_aggregate.py +2 -2
- {dataiter-0.99 → dataiter-0.999}/dataiter/test/test_data_frame.py +9 -6
- {dataiter-0.99 → dataiter-0.999}/dataiter/test/test_geojson.py +0 -6
- {dataiter-0.99 → dataiter-0.999}/dataiter/test/test_io.py +5 -0
- {dataiter-0.99 → dataiter-0.999}/dataiter/test/test_list_of_dicts.py +0 -6
- {dataiter-0.99 → dataiter-0.999}/dataiter/test/test_vector.py +19 -33
- {dataiter-0.99 → dataiter-0.999}/dataiter/util.py +1 -3
- {dataiter-0.99 → dataiter-0.999}/dataiter/vector.py +102 -73
- {dataiter-0.99 → dataiter-0.999}/pyproject.toml +1 -1
- dataiter-0.99/PKG-INFO +0 -72
- dataiter-0.99/README.md +0 -56
- {dataiter-0.99 → dataiter-0.999}/AUTHORS.md +0 -0
- {dataiter-0.99 → dataiter-0.999}/COPYING +0 -0
- {dataiter-0.99 → dataiter-0.999}/dataiter/aggregate.py +0 -0
- {dataiter-0.99 → dataiter-0.999}/dataiter/deco.py +0 -0
- {dataiter-0.99 → dataiter-0.999}/dataiter/test/test_dt.py +0 -0
- {dataiter-0.99 → dataiter-0.999}/dataiter/test/test_util.py +0 -0
dataiter-0.999/PKG-INFO
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: dataiter
|
|
3
|
+
Version: 0.999
|
|
4
|
+
Summary: Simple, light-weight data frames for Python
|
|
5
|
+
Project-URL: Homepage, https://github.com/otsaloma/dataiter
|
|
6
|
+
Author-email: Osmo Salomaa <otsaloma@iki.fi>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: AUTHORS.md
|
|
9
|
+
License-File: COPYING
|
|
10
|
+
Requires-Python: >=3.9.0
|
|
11
|
+
Requires-Dist: attd>=0.3
|
|
12
|
+
Requires-Dist: numpy<3.0,>=2.0
|
|
13
|
+
Requires-Dist: pandas>=1.0
|
|
14
|
+
Requires-Dist: wcwidth>=0.1
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
Simple, Light-Weight Data Frames for Python
|
|
18
|
+
===========================================
|
|
19
|
+
|
|
20
|
+
[](https://pypi.org/project/dataiter)
|
|
21
|
+
[](https://pepy.tech/project/dataiter)
|
|
22
|
+
|
|
23
|
+
Dataiter's **`DataFrame`** is a class for tabular data similar to R's
|
|
24
|
+
`data.frame`, implementing all common operations to manipulate data. It
|
|
25
|
+
is under the hood a dictionary of NumPy arrays and thus capable of fast
|
|
26
|
+
vectorized operations. You can consider it to be a light-weight
|
|
27
|
+
alternative to Pandas with a simple and consistent API. Performance-wise
|
|
28
|
+
Dataiter relies on NumPy and Numba and is likely to be at best
|
|
29
|
+
comparable to Pandas.
|
|
30
|
+
|
|
31
|
+
## Installation
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
# Latest stable version
|
|
35
|
+
pip install -U dataiter
|
|
36
|
+
|
|
37
|
+
# Latest development version
|
|
38
|
+
pip install -U git+https://github.com/otsaloma/dataiter
|
|
39
|
+
|
|
40
|
+
# Numba (optional)
|
|
41
|
+
pip install -U numba
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Dataiter optionally uses **Numba** to speed up certain operations. If
|
|
45
|
+
you have Numba installed, Dataiter will use it automatically. It's
|
|
46
|
+
currently not a hard dependency, so you need to install it separately.
|
|
47
|
+
|
|
48
|
+
## Quick Start
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
>>> import dataiter as di
|
|
52
|
+
>>> data = di.read_csv("data/listings.csv")
|
|
53
|
+
>>> data.filter(hood="Manhattan", guests=2).sort(price=1).head()
|
|
54
|
+
.
|
|
55
|
+
id hood zipcode guests sqft price
|
|
56
|
+
int64 string string int64 float64 int64
|
|
57
|
+
──────── ───────── ─────── ────── ─────── ─────
|
|
58
|
+
0 42279170 Manhattan 10013 2 nan 0
|
|
59
|
+
1 42384530 Manhattan 10036 2 nan 0
|
|
60
|
+
2 18835820 Manhattan 10021 2 nan 10
|
|
61
|
+
3 20171179 Manhattan 10027 2 nan 10
|
|
62
|
+
4 14858544 Manhattan 2 nan 15
|
|
63
|
+
5 31397084 Manhattan 10002 2 nan 19
|
|
64
|
+
6 22289683 Manhattan 10031 2 nan 20
|
|
65
|
+
7 7760204 Manhattan 10040 2 nan 22
|
|
66
|
+
8 43292527 Manhattan 10033 2 nan 22
|
|
67
|
+
9 43268040 Manhattan 10033 2 nan 23
|
|
68
|
+
.
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Documentation
|
|
72
|
+
|
|
73
|
+
https://dataiter.readthedocs.io/
|
|
74
|
+
|
|
75
|
+
If you're familiar with either dplyr (R) or Pandas (Python), the
|
|
76
|
+
comparison table in the documentation will give you a quick overview of
|
|
77
|
+
the differences and similarities in common operations.
|
|
78
|
+
|
|
79
|
+
https://dataiter.readthedocs.io/en/stable/comparison.html
|
|
80
|
+
|
|
81
|
+
## Development
|
|
82
|
+
|
|
83
|
+
To install a virtualenv for development, use
|
|
84
|
+
|
|
85
|
+
make venv
|
|
86
|
+
|
|
87
|
+
or, for a specific Python version
|
|
88
|
+
|
|
89
|
+
make PYTHON=python3.X venv
|
dataiter-0.999/README.md
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
Simple, Light-Weight Data Frames for Python
|
|
2
|
+
===========================================
|
|
3
|
+
|
|
4
|
+
[](https://pypi.org/project/dataiter)
|
|
5
|
+
[](https://pepy.tech/project/dataiter)
|
|
6
|
+
|
|
7
|
+
Dataiter's **`DataFrame`** is a class for tabular data similar to R's
|
|
8
|
+
`data.frame`, implementing all common operations to manipulate data. It
|
|
9
|
+
is under the hood a dictionary of NumPy arrays and thus capable of fast
|
|
10
|
+
vectorized operations. You can consider it to be a light-weight
|
|
11
|
+
alternative to Pandas with a simple and consistent API. Performance-wise
|
|
12
|
+
Dataiter relies on NumPy and Numba and is likely to be at best
|
|
13
|
+
comparable to Pandas.
|
|
14
|
+
|
|
15
|
+
## Installation
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
# Latest stable version
|
|
19
|
+
pip install -U dataiter
|
|
20
|
+
|
|
21
|
+
# Latest development version
|
|
22
|
+
pip install -U git+https://github.com/otsaloma/dataiter
|
|
23
|
+
|
|
24
|
+
# Numba (optional)
|
|
25
|
+
pip install -U numba
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Dataiter optionally uses **Numba** to speed up certain operations. If
|
|
29
|
+
you have Numba installed, Dataiter will use it automatically. It's
|
|
30
|
+
currently not a hard dependency, so you need to install it separately.
|
|
31
|
+
|
|
32
|
+
## Quick Start
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
>>> import dataiter as di
|
|
36
|
+
>>> data = di.read_csv("data/listings.csv")
|
|
37
|
+
>>> data.filter(hood="Manhattan", guests=2).sort(price=1).head()
|
|
38
|
+
.
|
|
39
|
+
id hood zipcode guests sqft price
|
|
40
|
+
int64 string string int64 float64 int64
|
|
41
|
+
──────── ───────── ─────── ────── ─────── ─────
|
|
42
|
+
0 42279170 Manhattan 10013 2 nan 0
|
|
43
|
+
1 42384530 Manhattan 10036 2 nan 0
|
|
44
|
+
2 18835820 Manhattan 10021 2 nan 10
|
|
45
|
+
3 20171179 Manhattan 10027 2 nan 10
|
|
46
|
+
4 14858544 Manhattan 2 nan 15
|
|
47
|
+
5 31397084 Manhattan 10002 2 nan 19
|
|
48
|
+
6 22289683 Manhattan 10031 2 nan 20
|
|
49
|
+
7 7760204 Manhattan 10040 2 nan 22
|
|
50
|
+
8 43292527 Manhattan 10033 2 nan 22
|
|
51
|
+
9 43268040 Manhattan 10033 2 nan 23
|
|
52
|
+
.
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Documentation
|
|
56
|
+
|
|
57
|
+
https://dataiter.readthedocs.io/
|
|
58
|
+
|
|
59
|
+
If you're familiar with either dplyr (R) or Pandas (Python), the
|
|
60
|
+
comparison table in the documentation will give you a quick overview of
|
|
61
|
+
the differences and similarities in common operations.
|
|
62
|
+
|
|
63
|
+
https://dataiter.readthedocs.io/en/stable/comparison.html
|
|
64
|
+
|
|
65
|
+
## Development
|
|
66
|
+
|
|
67
|
+
To install a virtualenv for development, use
|
|
68
|
+
|
|
69
|
+
make venv
|
|
70
|
+
|
|
71
|
+
or, for a specific Python version
|
|
72
|
+
|
|
73
|
+
make PYTHON=python3.X venv
|
|
@@ -25,7 +25,7 @@ import numpy as np
|
|
|
25
25
|
|
|
26
26
|
from dataiter import util
|
|
27
27
|
|
|
28
|
-
__version__ = "0.
|
|
28
|
+
__version__ = "0.999"
|
|
29
29
|
|
|
30
30
|
DEFAULT_PEEK_ELEMENTS = 10
|
|
31
31
|
DEFAULT_PEEK_ITEMS = 3
|
|
@@ -34,10 +34,28 @@ PRINT_FLOAT_PRECISION = 6
|
|
|
34
34
|
PRINT_MAX_ELEMENTS = 100
|
|
35
35
|
PRINT_MAX_ITEMS = 10
|
|
36
36
|
PRINT_MAX_ROWS = 100
|
|
37
|
+
|
|
38
|
+
#: Maximum amount of columns to wrap print output to. This is only a fallback
|
|
39
|
+
#: in case Python's ``shutil.get_terminal_size`` fails to detect the width of
|
|
40
|
+
#: your terminal. By default the detected full width is used.
|
|
37
41
|
PRINT_MAX_WIDTH = 80
|
|
42
|
+
|
|
43
|
+
#: Thousand separator to use when printing numbers. By default this is blank,
|
|
44
|
+
#: meaning no thousand separators are rendered.
|
|
38
45
|
PRINT_THOUSAND_SEPARATOR = ""
|
|
46
|
+
|
|
47
|
+
#: Maximum width to truncate string columns to in :class:`DataFrame` print
|
|
48
|
+
#: output. When this is exceeded, strings will be cut and an ellipsis (``…``)
|
|
49
|
+
#: rendered at the cut point.
|
|
39
50
|
PRINT_TRUNCATE_WIDTH = 36
|
|
51
|
+
|
|
52
|
+
#: ``True`` to use Numba, if available, to speed up :doc:`aggregations
|
|
53
|
+
#: </aggregation>`, ``False`` to only use pure Python code.
|
|
40
54
|
USE_NUMBA = False
|
|
55
|
+
|
|
56
|
+
#: ``True`` to use Numba cache for JIT-compiled :doc:`aggregations
|
|
57
|
+
#: </aggregation>`, ``False`` to only keep compiled code in memory for the
|
|
58
|
+
#: duration of the session.
|
|
41
59
|
USE_NUMBA_CACHE = True
|
|
42
60
|
|
|
43
61
|
if not np.__version__.startswith("2."):
|
|
@@ -69,6 +87,7 @@ globals().pop("np", None)
|
|
|
69
87
|
globals().pop("numba", None)
|
|
70
88
|
globals().pop("util", None)
|
|
71
89
|
|
|
90
|
+
from dataiter import dtypes # noqa
|
|
72
91
|
from dataiter.vector import Vector # noqa
|
|
73
92
|
from dataiter.data_frame import DataFrame # noqa
|
|
74
93
|
from dataiter.data_frame import DataFrameColumn # noqa
|
|
@@ -20,7 +20,6 @@
|
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
21
|
# THE SOFTWARE.
|
|
22
22
|
|
|
23
|
-
import contextlib
|
|
24
23
|
import dataiter
|
|
25
24
|
import functools
|
|
26
25
|
import itertools
|
|
@@ -475,24 +474,16 @@ class DataFrame(dict):
|
|
|
475
474
|
yield colname, np.delete(column, rows)
|
|
476
475
|
|
|
477
476
|
@classmethod
|
|
478
|
-
def from_arrow(cls, data, *,
|
|
477
|
+
def from_arrow(cls, data, *, dtypes={}):
|
|
479
478
|
"""
|
|
480
479
|
Return a new data frame from ``pyarrow.Table`` `data`.
|
|
481
480
|
|
|
482
|
-
`strings_as_object` is a cutoff point. If any row has more characters
|
|
483
|
-
than that, the whole column will use the object data type. This is
|
|
484
|
-
intended to help limit memory use as NumPy strings are fixed-length and
|
|
485
|
-
can take a huge amount of memory if even a single row is long. If set,
|
|
486
|
-
`dtypes` overrides this.
|
|
487
|
-
|
|
488
481
|
`dtypes` is an optional dict mapping column names to NumPy datatypes.
|
|
489
482
|
"""
|
|
490
483
|
# Arrow's 'to_numpy' is "limited to primitive types for which NumPy has
|
|
491
484
|
# the same physical representation as Arrow, and assuming the Arrow
|
|
492
485
|
# data has no nulls." Using Pandas is easier and probably good enough.
|
|
493
|
-
return cls.from_pandas(data.to_pandas(),
|
|
494
|
-
strings_as_object=strings_as_object,
|
|
495
|
-
dtypes=dtypes)
|
|
486
|
+
return cls.from_pandas(data.to_pandas(), dtypes=dtypes)
|
|
496
487
|
|
|
497
488
|
@classmethod
|
|
498
489
|
def from_json(cls, string, *, columns=[], dtypes={}, **kwargs):
|
|
@@ -517,29 +508,12 @@ class DataFrame(dict):
|
|
|
517
508
|
return cls(**data)
|
|
518
509
|
|
|
519
510
|
@classmethod
|
|
520
|
-
def from_pandas(cls, data, *,
|
|
511
|
+
def from_pandas(cls, data, *, dtypes={}):
|
|
521
512
|
"""
|
|
522
513
|
Return a new data frame from ``pandas.DataFrame`` `data`.
|
|
523
514
|
|
|
524
|
-
`strings_as_object` is a cutoff point. If any row has more characters
|
|
525
|
-
than that, the whole column will use the object data type. This is
|
|
526
|
-
intended to help limit memory use as NumPy strings are fixed-length and
|
|
527
|
-
can take a huge amount of memory if even a single row is long. If set,
|
|
528
|
-
`dtypes` overrides this.
|
|
529
|
-
|
|
530
515
|
`dtypes` is an optional dict mapping column names to NumPy datatypes.
|
|
531
516
|
"""
|
|
532
|
-
if (not isinstance(strings_as_object, (int, float)) or
|
|
533
|
-
isinstance(strings_as_object, bool)):
|
|
534
|
-
raise TypeError("Expected a number for strings_as_object")
|
|
535
|
-
dtypes = dtypes.copy()
|
|
536
|
-
from pandas.api.types import is_object_dtype
|
|
537
|
-
if strings_as_object < inf:
|
|
538
|
-
for name in data.columns:
|
|
539
|
-
if name not in dtypes and is_object_dtype(data[name]):
|
|
540
|
-
with contextlib.suppress(AttributeError):
|
|
541
|
-
if data[name].str.len().max() > strings_as_object:
|
|
542
|
-
dtypes[name] = object
|
|
543
517
|
data = {x: data[x].to_numpy(copy=True) for x in data.columns}
|
|
544
518
|
for name, value in data.items():
|
|
545
519
|
# Pandas object columns are likely to be strings,
|
|
@@ -875,21 +849,13 @@ class DataFrame(dict):
|
|
|
875
849
|
yield colname, total
|
|
876
850
|
|
|
877
851
|
@classmethod
|
|
878
|
-
def read_csv(cls, path, *, encoding="utf-8", sep=",", header=True, columns=[],
|
|
852
|
+
def read_csv(cls, path, *, encoding="utf-8", sep=",", header=True, columns=[], dtypes={}):
|
|
879
853
|
"""
|
|
880
854
|
Return a new data frame from CSV file `path`.
|
|
881
855
|
|
|
882
856
|
Will automatically decompress if `path` ends in ``.bz2|.gz|.xz``.
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
`strings_as_object` is a cutoff point. If any row has more characters
|
|
887
|
-
than that, the whole column will use the object data type. This is
|
|
888
|
-
intended to help limit memory use as NumPy strings are fixed-length and
|
|
889
|
-
can take a huge amount of memory if even a single row is long. If set,
|
|
890
|
-
`dtypes` overrides this.
|
|
891
|
-
|
|
892
|
-
`dtypes` is an optional dict mapping column names to NumPy datatypes.
|
|
857
|
+
`columns` is an optional list of columns to limit to. `dtypes` is an
|
|
858
|
+
optional dict mapping column names to NumPy datatypes.
|
|
893
859
|
"""
|
|
894
860
|
import pandas as pd
|
|
895
861
|
data = pd.read_csv(path,
|
|
@@ -903,7 +869,7 @@ class DataFrame(dict):
|
|
|
903
869
|
|
|
904
870
|
if not header:
|
|
905
871
|
data.columns = util.generate_colnames(len(data.columns))
|
|
906
|
-
return cls.from_pandas(data,
|
|
872
|
+
return cls.from_pandas(data, dtypes=dtypes)
|
|
907
873
|
|
|
908
874
|
@classmethod
|
|
909
875
|
def read_json(cls, path, *, encoding="utf-8", columns=[], dtypes={}, **kwargs):
|
|
@@ -911,7 +877,6 @@ class DataFrame(dict):
|
|
|
911
877
|
Return a new data frame from JSON file `path`.
|
|
912
878
|
|
|
913
879
|
Will automatically decompress if `path` ends in ``.bz2|.gz|.xz``.
|
|
914
|
-
|
|
915
880
|
`columns` is an optional list of columns to limit to. `dtypes` is an
|
|
916
881
|
optional dict mapping column names to NumPy datatypes. `kwargs` are
|
|
917
882
|
passed to ``json.load``.
|
|
@@ -931,24 +896,17 @@ class DataFrame(dict):
|
|
|
931
896
|
return cls(**data)
|
|
932
897
|
|
|
933
898
|
@classmethod
|
|
934
|
-
def read_parquet(cls, path, *, columns=[],
|
|
899
|
+
def read_parquet(cls, path, *, columns=[], dtypes={}):
|
|
935
900
|
"""
|
|
936
901
|
Return a new data frame from Parquet file `path`.
|
|
937
902
|
|
|
938
|
-
`columns` is an optional list of columns to limit to.
|
|
939
|
-
|
|
940
|
-
`strings_as_object` is a cutoff point. If any row has more characters
|
|
941
|
-
than that, the whole column will use the object data type. This is
|
|
942
|
-
intended to help limit memory use as NumPy strings are fixed-length and
|
|
943
|
-
can take a huge amount of memory if even a single row is long. If set,
|
|
944
|
-
`dtypes` overrides this.
|
|
945
|
-
|
|
946
|
-
`dtypes` is an optional dict mapping column names to NumPy datatypes.
|
|
903
|
+
`columns` is an optional list of columns to limit to. `dtypes` is an
|
|
904
|
+
optional dict mapping column names to NumPy datatypes.
|
|
947
905
|
"""
|
|
948
906
|
import pyarrow.parquet as pq
|
|
949
907
|
columns = columns or None
|
|
950
908
|
data = pq.read_table(path, columns=columns)
|
|
951
|
-
return cls.from_arrow(data,
|
|
909
|
+
return cls.from_arrow(data, dtypes=dtypes)
|
|
952
910
|
|
|
953
911
|
@classmethod
|
|
954
912
|
def read_pickle(cls, path):
|
|
@@ -1076,23 +1034,30 @@ class DataFrame(dict):
|
|
|
1076
1034
|
>>> data = di.read_csv("data/listings.csv")
|
|
1077
1035
|
>>> data.sort(hood=1, zipcode=1)
|
|
1078
1036
|
"""
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
column =
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1037
|
+
def sort_key(colname, dir):
|
|
1038
|
+
if dir not in [1, -1]:
|
|
1039
|
+
raise ValueError("dir should be 1 or -1")
|
|
1040
|
+
column = self[colname]
|
|
1041
|
+
column = column._optimize_for_argsort()
|
|
1042
|
+
if column._is_string_fixed():
|
|
1043
|
+
column[column.is_na()] = "\uffff"
|
|
1044
|
+
if dir > 0 and any((
|
|
1045
|
+
# No strings here due to lexsort segfault.
|
|
1046
|
+
# Need to require NumPy >= 2.2.1 to fix this.
|
|
1047
|
+
# https://github.com/numpy/numpy/issues/27984
|
|
1048
|
+
column._is_string_fixed(),
|
|
1049
|
+
column.is_boolean(),
|
|
1050
|
+
column.is_bytes(),
|
|
1051
|
+
column.is_datetime(),
|
|
1052
|
+
column.is_float(),
|
|
1053
|
+
column.is_integer(),
|
|
1054
|
+
column.is_timedelta(),
|
|
1055
|
+
)): return column
|
|
1056
|
+
if not column.is_number():
|
|
1057
|
+
column = column.rank(method="min")
|
|
1058
|
+
return column if dir > 0 else -column
|
|
1059
|
+
indices = np.lexsort(tuple(
|
|
1060
|
+
sort_key(*x) for x in reversed(colname_dir_pairs.items())))
|
|
1096
1061
|
for colname, column in self.items():
|
|
1097
1062
|
yield colname, column[indices].copy()
|
|
1098
1063
|
|
|
@@ -1187,7 +1152,7 @@ class DataFrame(dict):
|
|
|
1187
1152
|
n = min(self.nrow, max_rows)
|
|
1188
1153
|
columns = {colname: util.upad(
|
|
1189
1154
|
[colname] +
|
|
1190
|
-
[str(column.
|
|
1155
|
+
[str(column.dtype_label)] +
|
|
1191
1156
|
[str(x) for x in column[:n].to_strings(
|
|
1192
1157
|
quote=False, pad=True, truncate_width=truncate_width)]
|
|
1193
1158
|
) for colname, column in self.items()}
|
|
@@ -1224,21 +1189,31 @@ class DataFrame(dict):
|
|
|
1224
1189
|
>>> data = di.read_csv("data/listings.csv")
|
|
1225
1190
|
>>> data.unique("hood")
|
|
1226
1191
|
"""
|
|
1192
|
+
# Strangely, standard Python is a lot faster here than np.unique
|
|
1193
|
+
# with all the extra needed across columns of different type.
|
|
1194
|
+
# We just need to avoid NaN and NaT for the 'in' checks to work.
|
|
1227
1195
|
colnames = colnames or self.colnames
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1196
|
+
columns = [self[x] for x in colnames]
|
|
1197
|
+
for i, column in enumerate(columns):
|
|
1198
|
+
if column.is_datetime():
|
|
1199
|
+
flag = np.nanmin(column) - np.timedelta64(1, "D")
|
|
1200
|
+
if np.isnat(flag):
|
|
1201
|
+
flag = np.datetime64("-001-01-01")
|
|
1202
|
+
columns[i] = column.replace_na(flag)
|
|
1203
|
+
if column.is_float():
|
|
1204
|
+
flag = np.nanmin(column) - 1
|
|
1205
|
+
if np.isnan(flag):
|
|
1206
|
+
flag = -1
|
|
1207
|
+
columns[i] = column.replace_na(flag)
|
|
1208
|
+
rows = list(zip(*columns))
|
|
1209
|
+
seen = set()
|
|
1210
|
+
keep = []
|
|
1211
|
+
for i in range(self.nrow):
|
|
1212
|
+
if rows[i] not in seen:
|
|
1213
|
+
seen.add(rows[i])
|
|
1214
|
+
keep.append(i)
|
|
1240
1215
|
for colname, column in self.items():
|
|
1241
|
-
yield colname, column[
|
|
1216
|
+
yield colname, column[keep].copy()
|
|
1242
1217
|
|
|
1243
1218
|
@deco.new_from_generator
|
|
1244
1219
|
def unselect(self, *colnames):
|
|
@@ -1289,7 +1264,6 @@ class DataFrame(dict):
|
|
|
1289
1264
|
Write data frame to JSON file `path`.
|
|
1290
1265
|
|
|
1291
1266
|
Will automatically compress if `path` ends in ``.bz2|.gz|.xz``.
|
|
1292
|
-
|
|
1293
1267
|
`kwargs` are passed to ``json.JSONEncoder``.
|
|
1294
1268
|
"""
|
|
1295
1269
|
return self.to_list_of_dicts().write_json(path, encoding=encoding, **kwargs)
|
|
@@ -23,8 +23,10 @@
|
|
|
23
23
|
import datetime
|
|
24
24
|
import numpy as np
|
|
25
25
|
|
|
26
|
+
from dataiter import dtypes
|
|
26
27
|
from dataiter import util
|
|
27
28
|
from dataiter import Vector
|
|
29
|
+
from numpy.dtypes import StringDType
|
|
28
30
|
|
|
29
31
|
def day(x):
|
|
30
32
|
"""
|
|
@@ -46,22 +48,22 @@ def from_string(x, format):
|
|
|
46
48
|
>>> dt.from_string(x, "%d.%m.%Y")
|
|
47
49
|
"""
|
|
48
50
|
if util.is_scalar(x):
|
|
49
|
-
x =
|
|
51
|
+
x = Vector([x], str)
|
|
50
52
|
return from_string(x, format)[0]
|
|
51
53
|
x = util.sequencify(x)
|
|
52
54
|
assert isinstance(x, np.ndarray)
|
|
53
|
-
assert
|
|
55
|
+
assert isinstance(x.dtype, StringDType)
|
|
54
56
|
out = np.full_like(x, None, object)
|
|
55
57
|
out = Vector.fast(out, object)
|
|
56
|
-
na = x ==
|
|
58
|
+
na = x == dtypes.string.na_object
|
|
57
59
|
f = np.vectorize(lambda x: datetime.datetime.strptime(x, format))
|
|
58
60
|
out[~na] = f(x[~na].astype(object))
|
|
59
61
|
out = out.as_datetime()
|
|
60
|
-
if len(out[~na]) > 0
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
62
|
+
if (len(out[~na]) > 0 and
|
|
63
|
+
(hour(out[~na]) == 0).all() and
|
|
64
|
+
(minute(out[~na]) == 0).all() and
|
|
65
|
+
(second(out[~na]) == 0).all()):
|
|
66
|
+
out = out.as_date()
|
|
65
67
|
return out
|
|
66
68
|
|
|
67
69
|
def hour(x):
|
|
@@ -146,7 +148,7 @@ def now():
|
|
|
146
148
|
|
|
147
149
|
def _pull_datetime(x, function):
|
|
148
150
|
if util.is_scalar(x):
|
|
149
|
-
x =
|
|
151
|
+
x = Vector([x], np.datetime64)
|
|
150
152
|
return _pull_datetime(x, function)[0]
|
|
151
153
|
x = util.sequencify(x)
|
|
152
154
|
assert isinstance(x, np.ndarray)
|
|
@@ -161,7 +163,7 @@ def _pull_datetime(x, function):
|
|
|
161
163
|
|
|
162
164
|
def _pull_int(x, function):
|
|
163
165
|
if util.is_scalar(x):
|
|
164
|
-
x =
|
|
166
|
+
x = Vector([x], np.datetime64)
|
|
165
167
|
return _pull_int(x, function)[0]
|
|
166
168
|
x = util.sequencify(x)
|
|
167
169
|
assert isinstance(x, np.ndarray)
|
|
@@ -176,12 +178,12 @@ def _pull_int(x, function):
|
|
|
176
178
|
|
|
177
179
|
def _pull_str(x, function):
|
|
178
180
|
if util.is_scalar(x):
|
|
179
|
-
x =
|
|
181
|
+
x = Vector([x], np.datetime64)
|
|
180
182
|
return _pull_str(x, function)[0]
|
|
181
183
|
x = util.sequencify(x)
|
|
182
184
|
assert isinstance(x, np.ndarray)
|
|
183
185
|
assert np.issubdtype(x.dtype, np.datetime64)
|
|
184
|
-
out = np.full_like(x,
|
|
186
|
+
out = np.full_like(x, dtypes.string.na_object, object)
|
|
185
187
|
out = Vector.fast(out, object)
|
|
186
188
|
na = np.isnat(x)
|
|
187
189
|
if na.all(): return out
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
|
|
3
|
+
# Copyright (c) 2024 Osmo Salomaa
|
|
4
|
+
#
|
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
# furnished to do so, subject to the following conditions:
|
|
11
|
+
#
|
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
|
13
|
+
# all copies or substantial portions of the Software.
|
|
14
|
+
#
|
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
# THE SOFTWARE.
|
|
22
|
+
|
|
23
|
+
from numpy.dtypes import StringDType
|
|
24
|
+
|
|
25
|
+
#: Instance of NumPy variable-width StringDType used
|
|
26
|
+
string = StringDType(na_object="")
|
|
27
|
+
|
|
28
|
+
# Use a blank string as missing value sentinel (1) because that's what we used
|
|
29
|
+
# prior to the NumPy 2.0 StringDType and (2) because in many cases, such as CSV
|
|
30
|
+
# input, a distinction between NA and blank cannot usually be made.
|
|
31
|
+
# TODO: Consider changing this to something like ':NA:'.
|
|
32
|
+
# https://numpy.org/doc/stable/user/basics.strings.html#missing-data-support
|
|
@@ -27,7 +27,6 @@ from dataiter import DataFrame
|
|
|
27
27
|
from dataiter import DataFrameColumn
|
|
28
28
|
from dataiter import util
|
|
29
29
|
from dataiter import Vector
|
|
30
|
-
from math import inf
|
|
31
30
|
|
|
32
31
|
class GeoJSON(DataFrame):
|
|
33
32
|
|
|
@@ -35,12 +34,10 @@ class GeoJSON(DataFrame):
|
|
|
35
34
|
A class for GeoJSON data.
|
|
36
35
|
|
|
37
36
|
GeoJSON is a simple wrapper class that reads GeoJSON features into a
|
|
38
|
-
:class:`.DataFrame`. Any operations on the data are thus done with
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
All other data is available in the "metadata" attribute as an
|
|
43
|
-
``attd.AttributeDict``.
|
|
37
|
+
:class:`.DataFrame`. Any operations on the data are thus done with methods
|
|
38
|
+
provided by the data frame class. Geometry is available in the "geometry"
|
|
39
|
+
column, but no special geometric operations are supported. All other data
|
|
40
|
+
is available in the "metadata" attribute as an ``attd.AttributeDict``.
|
|
44
41
|
"""
|
|
45
42
|
|
|
46
43
|
# List of names that are actual attributes, not columns
|
|
@@ -84,27 +81,15 @@ class GeoJSON(DataFrame):
|
|
|
84
81
|
raise TypeError(f"Property type {type(value)} of {key!r} not supported")
|
|
85
82
|
|
|
86
83
|
@classmethod
|
|
87
|
-
def read(cls, path, *, encoding="utf-8", columns=[],
|
|
84
|
+
def read(cls, path, *, encoding="utf-8", columns=[], dtypes={}, **kwargs):
|
|
88
85
|
"""
|
|
89
86
|
Return data from GeoJSON file `path`.
|
|
90
87
|
|
|
91
88
|
Will automatically decompress if `path` ends in ``.bz2|.gz|.xz``.
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
`strings_as_object` is a cutoff point. If any row has more characters
|
|
96
|
-
than that, the whole column will use the object data type. This is
|
|
97
|
-
intended to help limit memory use as NumPy strings are fixed-length and
|
|
98
|
-
can take a huge amount of memory if even a single row is long. If set,
|
|
99
|
-
`dtypes` overrides this.
|
|
100
|
-
|
|
101
|
-
`dtypes` is an optional dict mapping column names to NumPy datatypes.
|
|
102
|
-
|
|
103
|
-
`kwargs` are passed to ``json.load``.
|
|
89
|
+
`columns` is an optional list of columns to limit to. `dtypes` is an
|
|
90
|
+
optional dict mapping column names to NumPy datatypes. `kwargs` are
|
|
91
|
+
passed to ``json.load``.
|
|
104
92
|
"""
|
|
105
|
-
if (not isinstance(strings_as_object, (int, float)) or
|
|
106
|
-
isinstance(strings_as_object, bool)):
|
|
107
|
-
raise TypeError("Expected a number for strings_as_object")
|
|
108
93
|
with util.xopen(path, "rt", encoding=encoding) as f:
|
|
109
94
|
raw = AttributeDict(json.load(f, **kwargs))
|
|
110
95
|
cls._check_raw_data(raw)
|
|
@@ -119,14 +104,6 @@ class GeoJSON(DataFrame):
|
|
|
119
104
|
value = feature.properties.get(key, None)
|
|
120
105
|
data[key].append(value)
|
|
121
106
|
data["geometry"] = [x.geometry for x in raw.features]
|
|
122
|
-
dtypes = dtypes.copy()
|
|
123
|
-
if strings_as_object < inf:
|
|
124
|
-
for name in data:
|
|
125
|
-
if (data[name] and
|
|
126
|
-
name not in dtypes and
|
|
127
|
-
any(isinstance(x, str) for x in data[name]) and
|
|
128
|
-
any(len(x) > strings_as_object for x in data[name] if isinstance(x, str))):
|
|
129
|
-
dtypes[name] = object
|
|
130
107
|
for name, dtype in dtypes.items():
|
|
131
108
|
data[name] = DataFrameColumn(data[name], dtype)
|
|
132
109
|
data = cls(**data)
|
|
@@ -144,19 +121,17 @@ class GeoJSON(DataFrame):
|
|
|
144
121
|
return DataFrame(**data)
|
|
145
122
|
|
|
146
123
|
def to_string(self, *, max_rows=None, max_width=None):
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
return DataFrame.to_string(data, max_rows=max_rows, max_width=max_width)
|
|
124
|
+
if "geometry" in self.colnames:
|
|
125
|
+
geometry = [f"<{x['type']}>" for x in self.geometry]
|
|
126
|
+
self = self.modify(geometry=Vector.fast(geometry, object))
|
|
127
|
+
return DataFrame.to_string(self, max_rows=max_rows, max_width=max_width)
|
|
152
128
|
|
|
153
129
|
def write(self, path, *, encoding="utf-8", **kwargs):
|
|
154
130
|
"""
|
|
155
131
|
Write data to GeoJSON file `path`.
|
|
156
132
|
|
|
157
133
|
Will automatically compress if `path` ends in ``.bz2|.gz|.xz``.
|
|
158
|
-
|
|
159
|
-
`kwargs` are passed to ``json.dump``.
|
|
134
|
+
`kwargs` are passed to ``json.dumps``.
|
|
160
135
|
"""
|
|
161
136
|
kwargs.setdefault("default", str)
|
|
162
137
|
kwargs.setdefault("ensure_ascii", False)
|