dataiter 0.48__tar.gz → 0.50__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataiter-0.50/.gitignore +18 -0
- {dataiter-0.48 → dataiter-0.50}/COPYING +1 -1
- {dataiter-0.48/dataiter.egg-info → dataiter-0.50}/PKG-INFO +8 -8
- {dataiter-0.48 → dataiter-0.50}/dataiter/__init__.py +1 -1
- {dataiter-0.48 → dataiter-0.50}/dataiter/data_frame.py +2 -2
- {dataiter-0.48 → dataiter-0.50}/dataiter/dt.py +5 -1
- {dataiter-0.48 → dataiter-0.50}/dataiter/list_of_dicts.py +55 -4
- dataiter-0.50/dataiter/test/__init__.py +60 -0
- dataiter-0.50/dataiter/test/test_aggregate.py +317 -0
- dataiter-0.50/dataiter/test/test_data_frame.py +742 -0
- dataiter-0.50/dataiter/test/test_dt.py +158 -0
- dataiter-0.50/dataiter/test/test_geojson.py +94 -0
- dataiter-0.50/dataiter/test/test_io.py +50 -0
- dataiter-0.50/dataiter/test/test_list_of_dicts.py +608 -0
- dataiter-0.50/dataiter/test/test_util.py +178 -0
- dataiter-0.50/dataiter/test/test_vector.py +625 -0
- {dataiter-0.48 → dataiter-0.50}/dataiter/util.py +23 -16
- {dataiter-0.48 → dataiter-0.50}/dataiter/vector.py +3 -3
- dataiter-0.50/pyproject.toml +22 -0
- dataiter-0.48/PKG-INFO +0 -72
- dataiter-0.48/dataiter.egg-info/SOURCES.txt +0 -19
- dataiter-0.48/dataiter.egg-info/dependency_links.txt +0 -1
- dataiter-0.48/dataiter.egg-info/requires.txt +0 -3
- dataiter-0.48/dataiter.egg-info/top_level.txt +0 -1
- dataiter-0.48/setup.cfg +0 -4
- dataiter-0.48/setup.py +0 -19
- {dataiter-0.48 → dataiter-0.50}/AUTHORS.md +0 -0
- {dataiter-0.48 → dataiter-0.50}/README.md +0 -0
- {dataiter-0.48 → dataiter-0.50}/dataiter/aggregate.py +0 -0
- {dataiter-0.48 → dataiter-0.50}/dataiter/deco.py +0 -0
- {dataiter-0.48 → dataiter-0.50}/dataiter/geojson.py +0 -0
- {dataiter-0.48 → dataiter-0.50}/dataiter/io.py +0 -0
dataiter-0.50/.gitignore
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
*.lprof
|
|
2
|
+
*.prof
|
|
3
|
+
.env
|
|
4
|
+
.envrc
|
|
5
|
+
.pytest_cache
|
|
6
|
+
__pycache__
|
|
7
|
+
benchmark*.csv
|
|
8
|
+
benchmark-head.py
|
|
9
|
+
build
|
|
10
|
+
data/orig
|
|
11
|
+
dataiter.egg-info
|
|
12
|
+
dist
|
|
13
|
+
doc/_build
|
|
14
|
+
doc/comparison/comparison.html
|
|
15
|
+
test.py
|
|
16
|
+
tmp.csv
|
|
17
|
+
validation/*.csv
|
|
18
|
+
venv
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dataiter
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.50
|
|
4
4
|
Summary: Classes for data manipulation
|
|
5
|
-
|
|
6
|
-
Author: Osmo Salomaa
|
|
7
|
-
|
|
8
|
-
License: MIT
|
|
9
|
-
Requires-Python: >=3.6.0
|
|
10
|
-
Description-Content-Type: text/markdown
|
|
11
|
-
License-File: COPYING
|
|
5
|
+
Project-URL: Homepage, https://github.com/otsaloma/dataiter
|
|
6
|
+
Author-email: Osmo Salomaa <otsaloma@iki.fi>
|
|
7
|
+
License-Expression: MIT
|
|
12
8
|
License-File: AUTHORS.md
|
|
9
|
+
License-File: COPYING
|
|
10
|
+
Requires-Python: >=3.6.0
|
|
13
11
|
Requires-Dist: attd>=0.3
|
|
14
12
|
Requires-Dist: numpy>=1.7
|
|
15
13
|
Requires-Dist: pandas>=1.0
|
|
14
|
+
Requires-Dist: wcwidth>=0.1
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
16
|
|
|
17
17
|
Python Classes for Data Manipulation
|
|
18
18
|
====================================
|
|
@@ -1185,7 +1185,7 @@ class DataFrame(dict):
|
|
|
1185
1185
|
max_width = max_width or util.get_print_width()
|
|
1186
1186
|
truncate_width = truncate_width or dataiter.PRINT_TRUNCATE_WIDTH
|
|
1187
1187
|
n = min(self.nrow, max_rows)
|
|
1188
|
-
columns = {colname: util.
|
|
1188
|
+
columns = {colname: util.upad(
|
|
1189
1189
|
[colname] +
|
|
1190
1190
|
[str(column.dtype)] +
|
|
1191
1191
|
[str(x) for x in column[:n].to_strings(
|
|
@@ -1194,7 +1194,7 @@ class DataFrame(dict):
|
|
|
1194
1194
|
for column in columns.values():
|
|
1195
1195
|
column.insert(2, "─" * util.ulen(column[0]))
|
|
1196
1196
|
row_numbers = [str(i) for i in range(n)]
|
|
1197
|
-
row_numbers = util.
|
|
1197
|
+
row_numbers = util.upad(["", "", ""] + row_numbers)
|
|
1198
1198
|
# If the length of rows exceeds max_width, split to
|
|
1199
1199
|
# batches of columns (like R's print.data.frame).
|
|
1200
1200
|
rows_to_print = []
|
|
@@ -154,6 +154,7 @@ def _pull_datetime(x, function):
|
|
|
154
154
|
out = np.full_like(x, np.nan)
|
|
155
155
|
out = Vector.fast(out, np.datetime64)
|
|
156
156
|
na = np.isnat(x)
|
|
157
|
+
if na.all(): return out
|
|
157
158
|
f = np.vectorize(function)
|
|
158
159
|
out[~na] = f(x[~na].astype(object))
|
|
159
160
|
return out
|
|
@@ -168,6 +169,7 @@ def _pull_int(x, function):
|
|
|
168
169
|
out = np.full_like(x, np.nan, float)
|
|
169
170
|
out = Vector.fast(out, float)
|
|
170
171
|
na = np.isnat(x)
|
|
172
|
+
if na.all(): return out
|
|
171
173
|
f = np.vectorize(function)
|
|
172
174
|
out[~na] = f(x[~na].astype(object))
|
|
173
175
|
return out if na.any() else out.as_integer()
|
|
@@ -182,6 +184,7 @@ def _pull_str(x, function):
|
|
|
182
184
|
out = np.full_like(x, "", object)
|
|
183
185
|
out = Vector.fast(out, object)
|
|
184
186
|
na = np.isnat(x)
|
|
187
|
+
if na.all(): return out
|
|
185
188
|
f = np.vectorize(function)
|
|
186
189
|
out[~na] = f(x[~na].astype(object))
|
|
187
190
|
return out.as_string()
|
|
@@ -193,7 +196,8 @@ def quarter(x):
|
|
|
193
196
|
>>> x = dt.new(["2022-10-15"])
|
|
194
197
|
>>> dt.quarter(x)
|
|
195
198
|
"""
|
|
196
|
-
|
|
199
|
+
y = np.ceil(month(x) / 3)
|
|
200
|
+
return y if np.isnan(y).any() else y.astype(int)
|
|
197
201
|
|
|
198
202
|
def replace(x, year=None, month=None, day=None, hour=None, minute=None, second=None, microsecond=None):
|
|
199
203
|
"""
|
|
@@ -28,6 +28,7 @@ import json
|
|
|
28
28
|
import operator
|
|
29
29
|
import pickle
|
|
30
30
|
import random
|
|
31
|
+
import sys
|
|
31
32
|
|
|
32
33
|
from attd import AttributeDict
|
|
33
34
|
from dataiter import deco
|
|
@@ -219,6 +220,18 @@ class ListOfDicts(list):
|
|
|
219
220
|
"""
|
|
220
221
|
return self.__deepcopy__()
|
|
221
222
|
|
|
223
|
+
@deco.new_from_generator
|
|
224
|
+
def drop_na(self, *keys):
|
|
225
|
+
"""
|
|
226
|
+
Return list without items that have missing values in `keys`.
|
|
227
|
+
|
|
228
|
+
>>> data = di.read_json("data/listings.json")
|
|
229
|
+
>>> data.drop_na("sqft")
|
|
230
|
+
"""
|
|
231
|
+
for item in self:
|
|
232
|
+
if not any(item.get(x, None) is None for x in keys):
|
|
233
|
+
yield item
|
|
234
|
+
|
|
222
235
|
@deco.new_from_generator
|
|
223
236
|
def extend(self, other):
|
|
224
237
|
"""
|
|
@@ -245,8 +258,7 @@ class ListOfDicts(list):
|
|
|
245
258
|
>>> data = data.fill_missing_keys()
|
|
246
259
|
"""
|
|
247
260
|
if not key_value_pairs:
|
|
248
|
-
|
|
249
|
-
key_value_pairs = dict.fromkeys(keys, None)
|
|
261
|
+
key_value_pairs = dict.fromkeys(self.keys(), None)
|
|
250
262
|
key_value_pairs = key_value_pairs.items()
|
|
251
263
|
for item in self:
|
|
252
264
|
for key, value in key_value_pairs:
|
|
@@ -423,6 +435,15 @@ class ListOfDicts(list):
|
|
|
423
435
|
yield item
|
|
424
436
|
yield self[i]
|
|
425
437
|
|
|
438
|
+
def keys(self):
|
|
439
|
+
"""
|
|
440
|
+
Return an iterator over unique keys in all items.
|
|
441
|
+
|
|
442
|
+
>>> data = di.read_json("data/listings.json")
|
|
443
|
+
>>> list(data.keys())
|
|
444
|
+
"""
|
|
445
|
+
yield from dict.fromkeys(itertools.chain(*self))
|
|
446
|
+
|
|
426
447
|
@deco.obsoletes
|
|
427
448
|
@deco.new_from_generator
|
|
428
449
|
def left_join(self, other, *by):
|
|
@@ -539,6 +560,36 @@ class ListOfDicts(list):
|
|
|
539
560
|
"""
|
|
540
561
|
print(self.to_string(max_items=max_items))
|
|
541
562
|
|
|
563
|
+
def print_memory_use(self):
|
|
564
|
+
"""
|
|
565
|
+
Print memory use by key and total.
|
|
566
|
+
|
|
567
|
+
>>> data = di.read_json("data/listings.json")
|
|
568
|
+
>>> data.print_memory_use()
|
|
569
|
+
"""
|
|
570
|
+
from dataiter import DataFrame
|
|
571
|
+
mem = DataFrame()
|
|
572
|
+
for key in self.keys():
|
|
573
|
+
new = DataFrame(key=key)
|
|
574
|
+
values = self.pluck(key)
|
|
575
|
+
values_real = list(filter(None, values))
|
|
576
|
+
first = values_real[0] if values_real else None
|
|
577
|
+
total = sum(sys.getsizeof(x) for x in values)
|
|
578
|
+
new.type = first.__class__.__name__
|
|
579
|
+
new.item_size = int(round(total / len(values)))
|
|
580
|
+
new.total_size = total
|
|
581
|
+
mem = mem.rbind(new)
|
|
582
|
+
new = DataFrame(key="TOTAL")
|
|
583
|
+
new.type = "--"
|
|
584
|
+
new.item_size = mem.item_size.sum()
|
|
585
|
+
new.total_size = mem.total_size.sum()
|
|
586
|
+
mem = mem.rbind(new)
|
|
587
|
+
# Format sizes into sensible values for display.
|
|
588
|
+
mem.item_size = [f"{x:.0f} B" for x in mem.item_size]
|
|
589
|
+
mem.total_size = [f"{x/1024**2:,.0f} MB" for x in mem.total_size]
|
|
590
|
+
mem.colnames = [x.upper() for x in mem.colnames]
|
|
591
|
+
print(mem)
|
|
592
|
+
|
|
542
593
|
def print_na_counts(self):
|
|
543
594
|
"""
|
|
544
595
|
Print counts of missing values by key.
|
|
@@ -550,7 +601,7 @@ class ListOfDicts(list):
|
|
|
550
601
|
>>> data.print_na_counts()
|
|
551
602
|
"""
|
|
552
603
|
print("Missing counts:")
|
|
553
|
-
for key in
|
|
604
|
+
for key in self.keys():
|
|
554
605
|
n = sum(x.get(key, None) is None for x in self)
|
|
555
606
|
if n == 0: continue
|
|
556
607
|
pc = 100 * n / len(self)
|
|
@@ -838,7 +889,7 @@ class ListOfDicts(list):
|
|
|
838
889
|
if not self:
|
|
839
890
|
raise ValueError("Cannot write empty CSV file")
|
|
840
891
|
# Take a superset of all keys.
|
|
841
|
-
keys =
|
|
892
|
+
keys = list(self.keys())
|
|
842
893
|
util.makedirs_for_file(path)
|
|
843
894
|
with util.xopen(path, "wt", encoding=encoding) as f:
|
|
844
895
|
writer = csv.DictWriter(f,
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
|
|
3
|
+
# Copyright (c) 2020 Osmo Salomaa
|
|
4
|
+
#
|
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
# furnished to do so, subject to the following conditions:
|
|
11
|
+
#
|
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
|
13
|
+
# all copies or substantial portions of the Software.
|
|
14
|
+
#
|
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
# THE SOFTWARE.
|
|
22
|
+
|
|
23
|
+
import functools
|
|
24
|
+
|
|
25
|
+
from dataiter import DataFrame
|
|
26
|
+
from dataiter import GeoJSON
|
|
27
|
+
from dataiter import ListOfDicts
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
def cached(function):
|
|
31
|
+
cache = {}
|
|
32
|
+
@functools.wraps(function)
|
|
33
|
+
def wrapper(path):
|
|
34
|
+
if path not in cache:
|
|
35
|
+
cache[path] = function(path)
|
|
36
|
+
return cache[path].deepcopy()
|
|
37
|
+
return wrapper
|
|
38
|
+
|
|
39
|
+
@cached
|
|
40
|
+
def data_frame(path):
|
|
41
|
+
path = get_data_path(path)
|
|
42
|
+
extension = path.suffix.lstrip(".")
|
|
43
|
+
read = getattr(DataFrame, f"read_{extension}")
|
|
44
|
+
return read(path)
|
|
45
|
+
|
|
46
|
+
@cached
|
|
47
|
+
def geojson(path):
|
|
48
|
+
path = get_data_path(path)
|
|
49
|
+
return GeoJSON.read(path)
|
|
50
|
+
|
|
51
|
+
def get_data_path(path):
|
|
52
|
+
root = Path(__file__).parent.parent.parent.resolve()
|
|
53
|
+
return root / "data" / str(path)
|
|
54
|
+
|
|
55
|
+
@cached
|
|
56
|
+
def list_of_dicts(path):
|
|
57
|
+
path = get_data_path(path)
|
|
58
|
+
extension = path.suffix.lstrip(".")
|
|
59
|
+
read = getattr(ListOfDicts, f"read_{extension}")
|
|
60
|
+
return read(path)
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
|
|
3
|
+
# Copyright (c) 2022 Osmo Salomaa
|
|
4
|
+
#
|
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
# furnished to do so, subject to the following conditions:
|
|
11
|
+
#
|
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
|
13
|
+
# all copies or substantial portions of the Software.
|
|
14
|
+
#
|
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
# THE SOFTWARE.
|
|
22
|
+
|
|
23
|
+
import dataiter
|
|
24
|
+
import datetime
|
|
25
|
+
import numpy as np
|
|
26
|
+
import pytest
|
|
27
|
+
|
|
28
|
+
from dataiter import DataFrame
|
|
29
|
+
from dataiter import Vector
|
|
30
|
+
from dataiter.aggregate import all
|
|
31
|
+
from dataiter.aggregate import any
|
|
32
|
+
from dataiter.aggregate import count
|
|
33
|
+
from dataiter.aggregate import count_unique
|
|
34
|
+
from dataiter.aggregate import first
|
|
35
|
+
from dataiter.aggregate import last
|
|
36
|
+
from dataiter.aggregate import max
|
|
37
|
+
from dataiter.aggregate import mean
|
|
38
|
+
from dataiter.aggregate import median
|
|
39
|
+
from dataiter.aggregate import min
|
|
40
|
+
from dataiter.aggregate import mode
|
|
41
|
+
from dataiter.aggregate import nth
|
|
42
|
+
from dataiter.aggregate import quantile
|
|
43
|
+
from dataiter.aggregate import std
|
|
44
|
+
from dataiter.aggregate import sum
|
|
45
|
+
from dataiter.aggregate import var
|
|
46
|
+
from unittest.mock import patch
|
|
47
|
+
|
|
48
|
+
T = True
|
|
49
|
+
F = False
|
|
50
|
+
|
|
51
|
+
D1 = datetime.date.today()
|
|
52
|
+
D2 = D1 + datetime.timedelta(days=1)
|
|
53
|
+
D3 = D1 + datetime.timedelta(days=2)
|
|
54
|
+
D4 = D1 + datetime.timedelta(days=3)
|
|
55
|
+
D5 = D1 + datetime.timedelta(days=4)
|
|
56
|
+
D6 = D1 + datetime.timedelta(days=5)
|
|
57
|
+
D7 = D1 + datetime.timedelta(days=6)
|
|
58
|
+
|
|
59
|
+
NaN = np.nan
|
|
60
|
+
NaT = np.datetime64("NaT")
|
|
61
|
+
|
|
62
|
+
EMPTY_VECTOR = Vector([], float)
|
|
63
|
+
GROUPS = [1, 1, 2, 2, 3, 3, 4, 4, 5, 5]
|
|
64
|
+
|
|
65
|
+
nth0 = lambda x: nth(x, 0)
|
|
66
|
+
quantile05 = lambda x: quantile(x, 0.5)
|
|
67
|
+
|
|
68
|
+
TEST_MATRIX = [
|
|
69
|
+
|
|
70
|
+
# NaNs evaluate to true, because they are not equal to zero.
|
|
71
|
+
# https://numpy.org/doc/stable/reference/generated/numpy.all.html
|
|
72
|
+
(all, [T, T, T, T, T, F, F, F, F, F], [T, T, F, F, F]),
|
|
73
|
+
(all, [1, 2, 3, 4, 5, 0, 0, 0, 0, 0], [T, T, F, F, F]),
|
|
74
|
+
(all, [0.0, 0.0, 0.0, 1.0, 2.0, 3.0, 4.0, NaN, NaN, NaN], [F, F, T, T, T]),
|
|
75
|
+
(all, [D1, D2, D3, D4, D5, NaT, NaT, NaT, NaT, NaT], [T, T, T, T, T]),
|
|
76
|
+
(all, ["a", "b", "c", "d", "e", "", "", "", "", ""], [T, T, F, F, F]),
|
|
77
|
+
|
|
78
|
+
# NaNs evaluate to true, because they are not equal to zero.
|
|
79
|
+
# https://numpy.org/doc/stable/reference/generated/numpy.any.html
|
|
80
|
+
(any, [T, T, T, T, T, F, F, F, F, F], [T, T, T, F, F]),
|
|
81
|
+
(any, [1, 2, 3, 4, 5, 0, 0, 0, 0, 0], [T, T, T, F, F]),
|
|
82
|
+
(any, [0.0, 0.0, 0.0, 1.0, 2.0, 3.0, 4.0, NaN, NaN, NaN], [F, T, T, T, T]),
|
|
83
|
+
(any, [D1, D2, D3, D4, D5, NaT, NaT, NaT, NaT, NaT], [T, T, T, T, T]),
|
|
84
|
+
(any, ["a", "b", "c", "d", "e", "", "", "", "", ""], [T, T, T, F, F]),
|
|
85
|
+
|
|
86
|
+
(count, [T, T, T, T, T, F, F, F, F, F], [2, 2, 2, 2, 2]),
|
|
87
|
+
(count, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [2, 2, 2, 2, 2]),
|
|
88
|
+
(count, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [2, 2, 2, 2, 2]),
|
|
89
|
+
(count, [D1, D2, D3, D4, D5, D6, D7, NaT, NaT, NaT], [2, 2, 2, 2, 2]),
|
|
90
|
+
(count, ["a", "b", "c", "d", "e", "f", "g", "", "", ""], [2, 2, 2, 2, 2]),
|
|
91
|
+
|
|
92
|
+
# NaN is not considered equal to itself and thus all are counted here.
|
|
93
|
+
(count_unique, [T, T, T, T, T, F, F, F, F, F], [1, 1, 2, 1, 1]),
|
|
94
|
+
(count_unique, [1, 1, 3, 3, 5, 6, 7, 8, 9, 10], [1, 1, 2, 2, 2]),
|
|
95
|
+
(count_unique, [1.0, 1.0, 3.0, 3.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [1, 1, 2, 2, 2]),
|
|
96
|
+
(count_unique, [D1, D1, D3, D3, D5, D6, D7, NaT, NaT, NaT], [1, 1, 2, 2, 2]),
|
|
97
|
+
(count_unique, ["a", "a", "c", "c", "e", "f", "g", "", "", ""], [1, 1, 2, 2, 1]),
|
|
98
|
+
|
|
99
|
+
(first, [T, T, T, T, T, F, F, F, F, F], [T, T, T, F, F]),
|
|
100
|
+
(first, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1, 3, 5, 7, 9]),
|
|
101
|
+
(first, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [1.0, 3.0, 5.0, 7.0, NaN]),
|
|
102
|
+
(first, [D1, D2, D3, D4, D5, D6, D7, NaT, NaT, NaT], [D1, D3, D5, D7, NaT]),
|
|
103
|
+
(first, ["a", "b", "c", "d", "e", "f", "g", "", "", ""], ["a", "c", "e", "g", ""]),
|
|
104
|
+
|
|
105
|
+
(last, [T, T, T, T, T, F, F, F, F, F], [T, T, F, F, F]),
|
|
106
|
+
(last, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [2, 4, 6, 8, 10]),
|
|
107
|
+
(last, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [2.0, 4.0, 6.0, NaN, NaN]),
|
|
108
|
+
(last, [D1, D2, D3, D4, D5, D6, D7, NaT, NaT, NaT], [D2, D4, D6, NaT, NaT]),
|
|
109
|
+
(last, ["a", "b", "c", "d", "e", "f", "g", "", "", ""], ["b", "d", "f", "", ""]),
|
|
110
|
+
|
|
111
|
+
(max, [T, T, T, T, T, F, F, F, F, F], [T, T, T, F, F]),
|
|
112
|
+
(max, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [2, 4, 6, 8, 10]),
|
|
113
|
+
(max, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [2.0, 4.0, 6.0, 7.0, NaN]),
|
|
114
|
+
(max, [D1, D2, D3, D4, D5, D6, D7, NaT, NaT, NaT], [D2, D4, D6, D7, NaT]),
|
|
115
|
+
|
|
116
|
+
(mean, [T, T, T, T, T, F, F, F, F, F], [1.0, 1.0, 0.5, 0.0, 0.0]),
|
|
117
|
+
(mean, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1.5, 3.5, 5.5, 7.5, 9.5]),
|
|
118
|
+
(mean, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [1.5, 3.5, 5.5, 7.0, NaN]),
|
|
119
|
+
|
|
120
|
+
(median, [T, T, T, T, T, F, F, F, F, F], [1.0, 1.0, 0.5, 0.0, 0.0]),
|
|
121
|
+
(median, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1.5, 3.5, 5.5, 7.5, 9.5]),
|
|
122
|
+
(median, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [1.5, 3.5, 5.5, 7.0, NaN]),
|
|
123
|
+
|
|
124
|
+
(min, [T, T, T, T, T, F, F, F, F, F], [T, T, F, F, F]),
|
|
125
|
+
(min, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1, 3, 5, 7, 9]),
|
|
126
|
+
(min, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [1.0, 3.0, 5.0, 7.0, NaN]),
|
|
127
|
+
(min, [D1, D2, D3, D4, D5, D6, D7, NaT, NaT, NaT], [D1, D3, D5, D7, NaT]),
|
|
128
|
+
|
|
129
|
+
(mode, [T, T, T, T, T, F, F, F, F, F], [T, T, T, F, F]),
|
|
130
|
+
(mode, [1, 1, 3, 3, 5, 6, 7, 8, 9, 10], [1, 3, 5, 7, 9]),
|
|
131
|
+
(mode, [1.0, 1.0, 3.0, 3.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [1.0, 3.0, 5.0, 7.0, NaN]),
|
|
132
|
+
(mode, [D1, D1, D3, D3, D5, D6, D7, NaT, NaT, NaT], [D1, D3, D5, D7, NaT]),
|
|
133
|
+
(mode, ["a", "a", "c", "c", "e", "f", "g", "", "", ""], ["a", "c", "e", "g", ""]),
|
|
134
|
+
|
|
135
|
+
(nth0, [T, T, T, T, T, F, F, F, F, F], [T, T, T, F, F]),
|
|
136
|
+
(nth0, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1, 3, 5, 7, 9]),
|
|
137
|
+
(nth0, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [1.0, 3.0, 5.0, 7.0, NaN]),
|
|
138
|
+
(nth0, [D1, D2, D3, D4, D5, D6, D7, NaT, NaT, NaT], [D1, D3, D5, D7, NaT]),
|
|
139
|
+
(nth0, ["a", "b", "c", "d", "e", "f", "g", "", "", ""], ["a", "c", "e", "g", ""]),
|
|
140
|
+
|
|
141
|
+
(quantile05, [T, T, T, T, T, F, F, F, F, F], [1.0, 1.0, 0.5, 0.0, 0.0]),
|
|
142
|
+
(quantile05, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1.5, 3.5, 5.5, 7.5, 9.5]),
|
|
143
|
+
(quantile05, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [1.5, 3.5, 5.5, 7.0, NaN]),
|
|
144
|
+
|
|
145
|
+
(std, [T, T, T, T, T, F, F, F, F, F], [0.0, 0.0, 0.5, 0.0, 0.0]),
|
|
146
|
+
(std, [1, 1, 2, 3, 5, 7, 8, 12, 13, 21], [0.0, 0.5, 1.0, 2.0, 4.0]),
|
|
147
|
+
(std, [1.0, 1.0, 2.0, 3.0, 5.0, 7.0, 8.0, NaN, NaN, NaN], [0.0, 0.5, 1.0, NaN, NaN]),
|
|
148
|
+
|
|
149
|
+
(sum, [T, T, T, T, T, F, F, F, F, F], [2, 2, 1, 0, 0]),
|
|
150
|
+
(sum, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [3, 7, 11, 15, 19]),
|
|
151
|
+
(sum, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [3.0, 7.0, 11.0, 7.0, 0.0]),
|
|
152
|
+
|
|
153
|
+
(var, [T, T, T, T, T, F, F, F, F, F], [0.0, 0.0, 0.25, 0.0, 0.0]),
|
|
154
|
+
(var, [1, 1, 2, 3, 5, 7, 8, 12, 13, 21], [0.0, 0.25, 1.0, 4.0, 16.0]),
|
|
155
|
+
(var, [1.0, 1.0, 2.0, 3.0, 5.0, 7.0, 8.0, NaN, NaN, NaN], [0.0, 0.25, 1.0, NaN, NaN]),
|
|
156
|
+
|
|
157
|
+
]
|
|
158
|
+
|
|
159
|
+
class TestAggregate:
|
|
160
|
+
|
|
161
|
+
@pytest.mark.parametrize("use_numba", [False, True])
|
|
162
|
+
@pytest.mark.parametrize("function,input,output", TEST_MATRIX)
|
|
163
|
+
def test_aggregate(self, function, input, output, use_numba):
|
|
164
|
+
if use_numba and not dataiter.USE_NUMBA:
|
|
165
|
+
pytest.skip("No Numba")
|
|
166
|
+
with patch("dataiter.USE_NUMBA", use_numba):
|
|
167
|
+
data = DataFrame(g=GROUPS, a=input)
|
|
168
|
+
stat = data.group_by("g").aggregate(a=function("a"))
|
|
169
|
+
expected = Vector(output)
|
|
170
|
+
try:
|
|
171
|
+
assert stat.a.equal(expected)
|
|
172
|
+
except AssertionError:
|
|
173
|
+
print("")
|
|
174
|
+
print(data)
|
|
175
|
+
print("Expected:")
|
|
176
|
+
print(expected)
|
|
177
|
+
print("Got:")
|
|
178
|
+
print(stat.a)
|
|
179
|
+
raise
|
|
180
|
+
|
|
181
|
+
@pytest.mark.parametrize("use_numba", [False, True])
|
|
182
|
+
def test_aggregate_count(self, use_numba):
|
|
183
|
+
if use_numba and not dataiter.USE_NUMBA:
|
|
184
|
+
pytest.skip("No Numba")
|
|
185
|
+
with patch("dataiter.USE_NUMBA", use_numba):
|
|
186
|
+
data = DataFrame(g=GROUPS)
|
|
187
|
+
stat = data.group_by("g").aggregate(n=count())
|
|
188
|
+
assert (stat.n == 2).all()
|
|
189
|
+
|
|
190
|
+
def test_all(self):
|
|
191
|
+
assert all(EMPTY_VECTOR)
|
|
192
|
+
assert all(Vector([T, T]))
|
|
193
|
+
assert not all(Vector([T, F]))
|
|
194
|
+
assert not all(Vector([F, F]))
|
|
195
|
+
|
|
196
|
+
def test_any(self):
|
|
197
|
+
assert not any(EMPTY_VECTOR)
|
|
198
|
+
assert any(Vector([T, T]))
|
|
199
|
+
assert any(Vector([T, F]))
|
|
200
|
+
assert not any(Vector([F, F]))
|
|
201
|
+
|
|
202
|
+
def test_count(self):
|
|
203
|
+
assert count(EMPTY_VECTOR) == 0
|
|
204
|
+
assert count(Vector([1])) == 1
|
|
205
|
+
assert count(Vector([1, 2])) == 2
|
|
206
|
+
assert count(Vector([1, 2, NaN])) == 3
|
|
207
|
+
assert count(Vector([1, 2, NaN]), drop_na=True) == 2
|
|
208
|
+
|
|
209
|
+
def test_count_unique(self):
|
|
210
|
+
assert count_unique(EMPTY_VECTOR) == 0
|
|
211
|
+
assert count_unique(Vector([1])) == 1
|
|
212
|
+
assert count_unique(Vector([1, 1])) == 1
|
|
213
|
+
assert count_unique(Vector([1, 1, 2])) == 2
|
|
214
|
+
assert count_unique(Vector([1, 1, 2, NaN])) == 3
|
|
215
|
+
assert count_unique(Vector([1, 1, 2, NaN]), drop_na=True) == 2
|
|
216
|
+
|
|
217
|
+
def test_first(self):
|
|
218
|
+
assert first(Vector([1, 2, 3])) == 1
|
|
219
|
+
assert first(Vector([NaN, 1, 2]), drop_na=True) == 1
|
|
220
|
+
|
|
221
|
+
def test_first_nan(self):
|
|
222
|
+
assert np.isnan(first(EMPTY_VECTOR))
|
|
223
|
+
assert np.isnan(first(Vector([NaN, 1, 2])))
|
|
224
|
+
|
|
225
|
+
def test_last(self):
|
|
226
|
+
assert last(Vector([1, 2, 3])) == 3
|
|
227
|
+
assert last(Vector([1, 2, NaN]), drop_na=True) == 2
|
|
228
|
+
|
|
229
|
+
def test_last_nan(self):
|
|
230
|
+
assert np.isnan(last(EMPTY_VECTOR))
|
|
231
|
+
assert np.isnan(last(Vector([1, 2, NaN])))
|
|
232
|
+
|
|
233
|
+
def test_max(self):
|
|
234
|
+
assert max(Vector([3, 2, 1])) == 3
|
|
235
|
+
assert max(Vector([3, 2, NaN])) == 3
|
|
236
|
+
|
|
237
|
+
def test_max_nan(self):
|
|
238
|
+
assert np.isnan(max(EMPTY_VECTOR))
|
|
239
|
+
assert np.isnan(max(Vector([3, 2, NaN]), drop_na=False))
|
|
240
|
+
|
|
241
|
+
def test_mean(self):
|
|
242
|
+
assert np.isclose(mean(Vector([1, 2, 10])), 4.333333)
|
|
243
|
+
assert np.isclose(mean(Vector([1, 2, NaN])), 1.5)
|
|
244
|
+
|
|
245
|
+
def test_mean_nan(self):
|
|
246
|
+
assert np.isnan(mean(EMPTY_VECTOR))
|
|
247
|
+
assert np.isnan(mean(Vector([1, 2, NaN]), drop_na=False))
|
|
248
|
+
|
|
249
|
+
def test_median(self):
|
|
250
|
+
assert median(Vector([1, 4, 6, 8, 5])) == 5
|
|
251
|
+
assert median(Vector([1, 4, 6, NaN, NaN])) == 4
|
|
252
|
+
|
|
253
|
+
def test_median_nan(self):
|
|
254
|
+
assert np.isnan(median(EMPTY_VECTOR))
|
|
255
|
+
assert np.isnan(median(Vector([1, 4, NaN]), drop_na=False))
|
|
256
|
+
|
|
257
|
+
def test_min(self):
|
|
258
|
+
assert min(Vector([3, 2, 1])) == 1
|
|
259
|
+
assert min(Vector([3, 2, NaN])) == 2
|
|
260
|
+
|
|
261
|
+
def test_min_nan(self):
|
|
262
|
+
assert np.isnan(min(EMPTY_VECTOR))
|
|
263
|
+
assert np.isnan(min(Vector([3, 2, NaN]), drop_na=False))
|
|
264
|
+
|
|
265
|
+
def test_mode(self):
|
|
266
|
+
assert mode(Vector([1])) == 1
|
|
267
|
+
assert mode(Vector([1, 2])) == 1
|
|
268
|
+
assert mode(Vector([1, 2, 2])) == 2
|
|
269
|
+
assert mode(Vector([1, 2, 2, NaN])) == 2
|
|
270
|
+
assert mode(Vector([1, 2, 2, NaN]), drop_na=False) == 2
|
|
271
|
+
|
|
272
|
+
def test_mode_nan(self):
|
|
273
|
+
assert np.isnan(mode(EMPTY_VECTOR))
|
|
274
|
+
assert np.isnan(mode(Vector([NaN, NaN], float), drop_na=False))
|
|
275
|
+
|
|
276
|
+
def test_nth(self):
|
|
277
|
+
assert nth(Vector([1, 2, 3]), 0) == 1
|
|
278
|
+
assert nth(Vector([NaN, 1, 2]), 0, drop_na=True) == 1
|
|
279
|
+
|
|
280
|
+
def test_nth_nan(self):
|
|
281
|
+
assert np.isnan(nth(EMPTY_VECTOR, 0))
|
|
282
|
+
assert np.isnan(nth(Vector([NaN, 1, 2]), 0))
|
|
283
|
+
|
|
284
|
+
def test_quantile(self):
|
|
285
|
+
assert quantile(Vector([1, 4, 6, 8, 5]), 0.5) == 5
|
|
286
|
+
assert quantile(Vector([1, 4, 6, NaN, NaN]), 0.5) == 4
|
|
287
|
+
|
|
288
|
+
def test_quantile_nan(self):
|
|
289
|
+
assert np.isnan(quantile(EMPTY_VECTOR, 0.5))
|
|
290
|
+
assert np.isnan(quantile(Vector([1, 4, NaN]), 0.5, drop_na=False))
|
|
291
|
+
|
|
292
|
+
def test_std(self):
|
|
293
|
+
assert np.isclose(std(Vector([3, 6, 7])), 1.699673)
|
|
294
|
+
assert np.isclose(std(Vector([3, 6, NaN])), 1.5)
|
|
295
|
+
|
|
296
|
+
def test_std_nan(self):
|
|
297
|
+
assert np.isnan(std(EMPTY_VECTOR))
|
|
298
|
+
assert np.isnan(std(Vector([1])))
|
|
299
|
+
assert np.isnan(std(Vector([3, 6, NaN]), drop_na=False))
|
|
300
|
+
|
|
301
|
+
def test_sum(self):
|
|
302
|
+
assert sum(EMPTY_VECTOR) == 0
|
|
303
|
+
assert sum(Vector([1])) == 1
|
|
304
|
+
assert sum(Vector([1, 2])) == 3
|
|
305
|
+
assert sum(Vector([1, 2, NaN])) == 3
|
|
306
|
+
|
|
307
|
+
def test_sum_nan(self):
|
|
308
|
+
assert np.isnan(sum(Vector([1, 2, NaN]), drop_na=False))
|
|
309
|
+
|
|
310
|
+
def test_var(self):
|
|
311
|
+
assert np.isclose(var(Vector([3, 6, 7])), 2.888889)
|
|
312
|
+
assert np.isclose(var(Vector([3, 6, NaN])), 2.25)
|
|
313
|
+
|
|
314
|
+
def test_var_nan(self):
|
|
315
|
+
assert np.isnan(var(EMPTY_VECTOR))
|
|
316
|
+
assert np.isnan(var(Vector([1])))
|
|
317
|
+
assert np.isnan(var(Vector([3, 6, NaN]), drop_na=False))
|