dataiter 0.48__tar.gz → 0.50__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. dataiter-0.50/.gitignore +18 -0
  2. {dataiter-0.48 → dataiter-0.50}/COPYING +1 -1
  3. {dataiter-0.48/dataiter.egg-info → dataiter-0.50}/PKG-INFO +8 -8
  4. {dataiter-0.48 → dataiter-0.50}/dataiter/__init__.py +1 -1
  5. {dataiter-0.48 → dataiter-0.50}/dataiter/data_frame.py +2 -2
  6. {dataiter-0.48 → dataiter-0.50}/dataiter/dt.py +5 -1
  7. {dataiter-0.48 → dataiter-0.50}/dataiter/list_of_dicts.py +55 -4
  8. dataiter-0.50/dataiter/test/__init__.py +60 -0
  9. dataiter-0.50/dataiter/test/test_aggregate.py +317 -0
  10. dataiter-0.50/dataiter/test/test_data_frame.py +742 -0
  11. dataiter-0.50/dataiter/test/test_dt.py +158 -0
  12. dataiter-0.50/dataiter/test/test_geojson.py +94 -0
  13. dataiter-0.50/dataiter/test/test_io.py +50 -0
  14. dataiter-0.50/dataiter/test/test_list_of_dicts.py +608 -0
  15. dataiter-0.50/dataiter/test/test_util.py +178 -0
  16. dataiter-0.50/dataiter/test/test_vector.py +625 -0
  17. {dataiter-0.48 → dataiter-0.50}/dataiter/util.py +23 -16
  18. {dataiter-0.48 → dataiter-0.50}/dataiter/vector.py +3 -3
  19. dataiter-0.50/pyproject.toml +22 -0
  20. dataiter-0.48/PKG-INFO +0 -72
  21. dataiter-0.48/dataiter.egg-info/SOURCES.txt +0 -19
  22. dataiter-0.48/dataiter.egg-info/dependency_links.txt +0 -1
  23. dataiter-0.48/dataiter.egg-info/requires.txt +0 -3
  24. dataiter-0.48/dataiter.egg-info/top_level.txt +0 -1
  25. dataiter-0.48/setup.cfg +0 -4
  26. dataiter-0.48/setup.py +0 -19
  27. {dataiter-0.48 → dataiter-0.50}/AUTHORS.md +0 -0
  28. {dataiter-0.48 → dataiter-0.50}/README.md +0 -0
  29. {dataiter-0.48 → dataiter-0.50}/dataiter/aggregate.py +0 -0
  30. {dataiter-0.48 → dataiter-0.50}/dataiter/deco.py +0 -0
  31. {dataiter-0.48 → dataiter-0.50}/dataiter/geojson.py +0 -0
  32. {dataiter-0.48 → dataiter-0.50}/dataiter/io.py +0 -0
@@ -0,0 +1,18 @@
1
+ *.lprof
2
+ *.prof
3
+ .env
4
+ .envrc
5
+ .pytest_cache
6
+ __pycache__
7
+ benchmark*.csv
8
+ benchmark-head.py
9
+ build
10
+ data/orig
11
+ dataiter.egg-info
12
+ dist
13
+ doc/_build
14
+ doc/comparison/comparison.html
15
+ test.py
16
+ tmp.csv
17
+ validation/*.csv
18
+ venv
@@ -1,6 +1,6 @@
1
1
  The MIT License (MIT)
2
2
 
3
- Copyright (c) 2019-2023 Osmo Salomaa
3
+ Copyright (c) 2019-2024 Osmo Salomaa
4
4
 
5
5
  Permission is hereby granted, free of charge, to any person obtaining a copy
6
6
  of this software and associated documentation files (the "Software"), to deal
@@ -1,18 +1,18 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dataiter
3
- Version: 0.48
3
+ Version: 0.50
4
4
  Summary: Classes for data manipulation
5
- Home-page: https://github.com/otsaloma/dataiter
6
- Author: Osmo Salomaa
7
- Author-email: otsaloma@iki.fi
8
- License: MIT
9
- Requires-Python: >=3.6.0
10
- Description-Content-Type: text/markdown
11
- License-File: COPYING
5
+ Project-URL: Homepage, https://github.com/otsaloma/dataiter
6
+ Author-email: Osmo Salomaa <otsaloma@iki.fi>
7
+ License-Expression: MIT
12
8
  License-File: AUTHORS.md
9
+ License-File: COPYING
10
+ Requires-Python: >=3.6.0
13
11
  Requires-Dist: attd>=0.3
14
12
  Requires-Dist: numpy>=1.7
15
13
  Requires-Dist: pandas>=1.0
14
+ Requires-Dist: wcwidth>=0.1
15
+ Description-Content-Type: text/markdown
16
16
 
17
17
  Python Classes for Data Manipulation
18
18
  ====================================
@@ -24,7 +24,7 @@ import contextlib
24
24
 
25
25
  from dataiter import util
26
26
 
27
- __version__ = "0.48"
27
+ __version__ = "0.50"
28
28
 
29
29
  DEFAULT_PEEK_ELEMENTS = 10
30
30
  DEFAULT_PEEK_ITEMS = 3
@@ -1185,7 +1185,7 @@ class DataFrame(dict):
1185
1185
  max_width = max_width or util.get_print_width()
1186
1186
  truncate_width = truncate_width or dataiter.PRINT_TRUNCATE_WIDTH
1187
1187
  n = min(self.nrow, max_rows)
1188
- columns = {colname: util.pad(
1188
+ columns = {colname: util.upad(
1189
1189
  [colname] +
1190
1190
  [str(column.dtype)] +
1191
1191
  [str(x) for x in column[:n].to_strings(
@@ -1194,7 +1194,7 @@ class DataFrame(dict):
1194
1194
  for column in columns.values():
1195
1195
  column.insert(2, "─" * util.ulen(column[0]))
1196
1196
  row_numbers = [str(i) for i in range(n)]
1197
- row_numbers = util.pad(["", "", ""] + row_numbers)
1197
+ row_numbers = util.upad(["", "", ""] + row_numbers)
1198
1198
  # If the length of rows exceeds max_width, split to
1199
1199
  # batches of columns (like R's print.data.frame).
1200
1200
  rows_to_print = []
@@ -154,6 +154,7 @@ def _pull_datetime(x, function):
154
154
  out = np.full_like(x, np.nan)
155
155
  out = Vector.fast(out, np.datetime64)
156
156
  na = np.isnat(x)
157
+ if na.all(): return out
157
158
  f = np.vectorize(function)
158
159
  out[~na] = f(x[~na].astype(object))
159
160
  return out
@@ -168,6 +169,7 @@ def _pull_int(x, function):
168
169
  out = np.full_like(x, np.nan, float)
169
170
  out = Vector.fast(out, float)
170
171
  na = np.isnat(x)
172
+ if na.all(): return out
171
173
  f = np.vectorize(function)
172
174
  out[~na] = f(x[~na].astype(object))
173
175
  return out if na.any() else out.as_integer()
@@ -182,6 +184,7 @@ def _pull_str(x, function):
182
184
  out = np.full_like(x, "", object)
183
185
  out = Vector.fast(out, object)
184
186
  na = np.isnat(x)
187
+ if na.all(): return out
185
188
  f = np.vectorize(function)
186
189
  out[~na] = f(x[~na].astype(object))
187
190
  return out.as_string()
@@ -193,7 +196,8 @@ def quarter(x):
193
196
  >>> x = dt.new(["2022-10-15"])
194
197
  >>> dt.quarter(x)
195
198
  """
196
- return np.ceil(month(x) / 3)
199
+ y = np.ceil(month(x) / 3)
200
+ return y if np.isnan(y).any() else y.astype(int)
197
201
 
198
202
  def replace(x, year=None, month=None, day=None, hour=None, minute=None, second=None, microsecond=None):
199
203
  """
@@ -28,6 +28,7 @@ import json
28
28
  import operator
29
29
  import pickle
30
30
  import random
31
+ import sys
31
32
 
32
33
  from attd import AttributeDict
33
34
  from dataiter import deco
@@ -219,6 +220,18 @@ class ListOfDicts(list):
219
220
  """
220
221
  return self.__deepcopy__()
221
222
 
223
+ @deco.new_from_generator
224
+ def drop_na(self, *keys):
225
+ """
226
+ Return list without items that have missing values in `keys`.
227
+
228
+ >>> data = di.read_json("data/listings.json")
229
+ >>> data.drop_na("sqft")
230
+ """
231
+ for item in self:
232
+ if not any(item.get(x, None) is None for x in keys):
233
+ yield item
234
+
222
235
  @deco.new_from_generator
223
236
  def extend(self, other):
224
237
  """
@@ -245,8 +258,7 @@ class ListOfDicts(list):
245
258
  >>> data = data.fill_missing_keys()
246
259
  """
247
260
  if not key_value_pairs:
248
- keys = util.unique_keys(itertools.chain(*self))
249
- key_value_pairs = dict.fromkeys(keys, None)
261
+ key_value_pairs = dict.fromkeys(self.keys(), None)
250
262
  key_value_pairs = key_value_pairs.items()
251
263
  for item in self:
252
264
  for key, value in key_value_pairs:
@@ -423,6 +435,15 @@ class ListOfDicts(list):
423
435
  yield item
424
436
  yield self[i]
425
437
 
438
+ def keys(self):
439
+ """
440
+ Return an iterator over unique keys in all items.
441
+
442
+ >>> data = di.read_json("data/listings.json")
443
+ >>> list(data.keys())
444
+ """
445
+ yield from dict.fromkeys(itertools.chain(*self))
446
+
426
447
  @deco.obsoletes
427
448
  @deco.new_from_generator
428
449
  def left_join(self, other, *by):
@@ -539,6 +560,36 @@ class ListOfDicts(list):
539
560
  """
540
561
  print(self.to_string(max_items=max_items))
541
562
 
563
+ def print_memory_use(self):
564
+ """
565
+ Print memory use by key and total.
566
+
567
+ >>> data = di.read_json("data/listings.json")
568
+ >>> data.print_memory_use()
569
+ """
570
+ from dataiter import DataFrame
571
+ mem = DataFrame()
572
+ for key in self.keys():
573
+ new = DataFrame(key=key)
574
+ values = self.pluck(key)
575
+ values_real = list(filter(None, values))
576
+ first = values_real[0] if values_real else None
577
+ total = sum(sys.getsizeof(x) for x in values)
578
+ new.type = first.__class__.__name__
579
+ new.item_size = int(round(total / len(values)))
580
+ new.total_size = total
581
+ mem = mem.rbind(new)
582
+ new = DataFrame(key="TOTAL")
583
+ new.type = "--"
584
+ new.item_size = mem.item_size.sum()
585
+ new.total_size = mem.total_size.sum()
586
+ mem = mem.rbind(new)
587
+ # Format sizes into sensible values for display.
588
+ mem.item_size = [f"{x:.0f} B" for x in mem.item_size]
589
+ mem.total_size = [f"{x/1024**2:,.0f} MB" for x in mem.total_size]
590
+ mem.colnames = [x.upper() for x in mem.colnames]
591
+ print(mem)
592
+
542
593
  def print_na_counts(self):
543
594
  """
544
595
  Print counts of missing values by key.
@@ -550,7 +601,7 @@ class ListOfDicts(list):
550
601
  >>> data.print_na_counts()
551
602
  """
552
603
  print("Missing counts:")
553
- for key in util.unique_keys(itertools.chain(*self)):
604
+ for key in self.keys():
554
605
  n = sum(x.get(key, None) is None for x in self)
555
606
  if n == 0: continue
556
607
  pc = 100 * n / len(self)
@@ -838,7 +889,7 @@ class ListOfDicts(list):
838
889
  if not self:
839
890
  raise ValueError("Cannot write empty CSV file")
840
891
  # Take a superset of all keys.
841
- keys = util.unique_keys(itertools.chain(*self))
892
+ keys = list(self.keys())
842
893
  util.makedirs_for_file(path)
843
894
  with util.xopen(path, "wt", encoding=encoding) as f:
844
895
  writer = csv.DictWriter(f,
@@ -0,0 +1,60 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ # Copyright (c) 2020 Osmo Salomaa
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ import functools
24
+
25
+ from dataiter import DataFrame
26
+ from dataiter import GeoJSON
27
+ from dataiter import ListOfDicts
28
+ from pathlib import Path
29
+
30
+ def cached(function):
31
+ cache = {}
32
+ @functools.wraps(function)
33
+ def wrapper(path):
34
+ if path not in cache:
35
+ cache[path] = function(path)
36
+ return cache[path].deepcopy()
37
+ return wrapper
38
+
39
+ @cached
40
+ def data_frame(path):
41
+ path = get_data_path(path)
42
+ extension = path.suffix.lstrip(".")
43
+ read = getattr(DataFrame, f"read_{extension}")
44
+ return read(path)
45
+
46
+ @cached
47
+ def geojson(path):
48
+ path = get_data_path(path)
49
+ return GeoJSON.read(path)
50
+
51
+ def get_data_path(path):
52
+ root = Path(__file__).parent.parent.parent.resolve()
53
+ return root / "data" / str(path)
54
+
55
+ @cached
56
+ def list_of_dicts(path):
57
+ path = get_data_path(path)
58
+ extension = path.suffix.lstrip(".")
59
+ read = getattr(ListOfDicts, f"read_{extension}")
60
+ return read(path)
@@ -0,0 +1,317 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ # Copyright (c) 2022 Osmo Salomaa
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ import dataiter
24
+ import datetime
25
+ import numpy as np
26
+ import pytest
27
+
28
+ from dataiter import DataFrame
29
+ from dataiter import Vector
30
+ from dataiter.aggregate import all
31
+ from dataiter.aggregate import any
32
+ from dataiter.aggregate import count
33
+ from dataiter.aggregate import count_unique
34
+ from dataiter.aggregate import first
35
+ from dataiter.aggregate import last
36
+ from dataiter.aggregate import max
37
+ from dataiter.aggregate import mean
38
+ from dataiter.aggregate import median
39
+ from dataiter.aggregate import min
40
+ from dataiter.aggregate import mode
41
+ from dataiter.aggregate import nth
42
+ from dataiter.aggregate import quantile
43
+ from dataiter.aggregate import std
44
+ from dataiter.aggregate import sum
45
+ from dataiter.aggregate import var
46
+ from unittest.mock import patch
47
+
48
+ T = True
49
+ F = False
50
+
51
+ D1 = datetime.date.today()
52
+ D2 = D1 + datetime.timedelta(days=1)
53
+ D3 = D1 + datetime.timedelta(days=2)
54
+ D4 = D1 + datetime.timedelta(days=3)
55
+ D5 = D1 + datetime.timedelta(days=4)
56
+ D6 = D1 + datetime.timedelta(days=5)
57
+ D7 = D1 + datetime.timedelta(days=6)
58
+
59
+ NaN = np.nan
60
+ NaT = np.datetime64("NaT")
61
+
62
+ EMPTY_VECTOR = Vector([], float)
63
+ GROUPS = [1, 1, 2, 2, 3, 3, 4, 4, 5, 5]
64
+
65
+ nth0 = lambda x: nth(x, 0)
66
+ quantile05 = lambda x: quantile(x, 0.5)
67
+
68
+ TEST_MATRIX = [
69
+
70
+ # NaNs evaluate to true, because they are not equal to zero.
71
+ # https://numpy.org/doc/stable/reference/generated/numpy.all.html
72
+ (all, [T, T, T, T, T, F, F, F, F, F], [T, T, F, F, F]),
73
+ (all, [1, 2, 3, 4, 5, 0, 0, 0, 0, 0], [T, T, F, F, F]),
74
+ (all, [0.0, 0.0, 0.0, 1.0, 2.0, 3.0, 4.0, NaN, NaN, NaN], [F, F, T, T, T]),
75
+ (all, [D1, D2, D3, D4, D5, NaT, NaT, NaT, NaT, NaT], [T, T, T, T, T]),
76
+ (all, ["a", "b", "c", "d", "e", "", "", "", "", ""], [T, T, F, F, F]),
77
+
78
+ # NaNs evaluate to true, because they are not equal to zero.
79
+ # https://numpy.org/doc/stable/reference/generated/numpy.any.html
80
+ (any, [T, T, T, T, T, F, F, F, F, F], [T, T, T, F, F]),
81
+ (any, [1, 2, 3, 4, 5, 0, 0, 0, 0, 0], [T, T, T, F, F]),
82
+ (any, [0.0, 0.0, 0.0, 1.0, 2.0, 3.0, 4.0, NaN, NaN, NaN], [F, T, T, T, T]),
83
+ (any, [D1, D2, D3, D4, D5, NaT, NaT, NaT, NaT, NaT], [T, T, T, T, T]),
84
+ (any, ["a", "b", "c", "d", "e", "", "", "", "", ""], [T, T, T, F, F]),
85
+
86
+ (count, [T, T, T, T, T, F, F, F, F, F], [2, 2, 2, 2, 2]),
87
+ (count, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [2, 2, 2, 2, 2]),
88
+ (count, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [2, 2, 2, 2, 2]),
89
+ (count, [D1, D2, D3, D4, D5, D6, D7, NaT, NaT, NaT], [2, 2, 2, 2, 2]),
90
+ (count, ["a", "b", "c", "d", "e", "f", "g", "", "", ""], [2, 2, 2, 2, 2]),
91
+
92
+ # NaN is not considered equal to itself and thus all are counted here.
93
+ (count_unique, [T, T, T, T, T, F, F, F, F, F], [1, 1, 2, 1, 1]),
94
+ (count_unique, [1, 1, 3, 3, 5, 6, 7, 8, 9, 10], [1, 1, 2, 2, 2]),
95
+ (count_unique, [1.0, 1.0, 3.0, 3.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [1, 1, 2, 2, 2]),
96
+ (count_unique, [D1, D1, D3, D3, D5, D6, D7, NaT, NaT, NaT], [1, 1, 2, 2, 2]),
97
+ (count_unique, ["a", "a", "c", "c", "e", "f", "g", "", "", ""], [1, 1, 2, 2, 1]),
98
+
99
+ (first, [T, T, T, T, T, F, F, F, F, F], [T, T, T, F, F]),
100
+ (first, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1, 3, 5, 7, 9]),
101
+ (first, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [1.0, 3.0, 5.0, 7.0, NaN]),
102
+ (first, [D1, D2, D3, D4, D5, D6, D7, NaT, NaT, NaT], [D1, D3, D5, D7, NaT]),
103
+ (first, ["a", "b", "c", "d", "e", "f", "g", "", "", ""], ["a", "c", "e", "g", ""]),
104
+
105
+ (last, [T, T, T, T, T, F, F, F, F, F], [T, T, F, F, F]),
106
+ (last, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [2, 4, 6, 8, 10]),
107
+ (last, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [2.0, 4.0, 6.0, NaN, NaN]),
108
+ (last, [D1, D2, D3, D4, D5, D6, D7, NaT, NaT, NaT], [D2, D4, D6, NaT, NaT]),
109
+ (last, ["a", "b", "c", "d", "e", "f", "g", "", "", ""], ["b", "d", "f", "", ""]),
110
+
111
+ (max, [T, T, T, T, T, F, F, F, F, F], [T, T, T, F, F]),
112
+ (max, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [2, 4, 6, 8, 10]),
113
+ (max, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [2.0, 4.0, 6.0, 7.0, NaN]),
114
+ (max, [D1, D2, D3, D4, D5, D6, D7, NaT, NaT, NaT], [D2, D4, D6, D7, NaT]),
115
+
116
+ (mean, [T, T, T, T, T, F, F, F, F, F], [1.0, 1.0, 0.5, 0.0, 0.0]),
117
+ (mean, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1.5, 3.5, 5.5, 7.5, 9.5]),
118
+ (mean, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [1.5, 3.5, 5.5, 7.0, NaN]),
119
+
120
+ (median, [T, T, T, T, T, F, F, F, F, F], [1.0, 1.0, 0.5, 0.0, 0.0]),
121
+ (median, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1.5, 3.5, 5.5, 7.5, 9.5]),
122
+ (median, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [1.5, 3.5, 5.5, 7.0, NaN]),
123
+
124
+ (min, [T, T, T, T, T, F, F, F, F, F], [T, T, F, F, F]),
125
+ (min, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1, 3, 5, 7, 9]),
126
+ (min, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [1.0, 3.0, 5.0, 7.0, NaN]),
127
+ (min, [D1, D2, D3, D4, D5, D6, D7, NaT, NaT, NaT], [D1, D3, D5, D7, NaT]),
128
+
129
+ (mode, [T, T, T, T, T, F, F, F, F, F], [T, T, T, F, F]),
130
+ (mode, [1, 1, 3, 3, 5, 6, 7, 8, 9, 10], [1, 3, 5, 7, 9]),
131
+ (mode, [1.0, 1.0, 3.0, 3.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [1.0, 3.0, 5.0, 7.0, NaN]),
132
+ (mode, [D1, D1, D3, D3, D5, D6, D7, NaT, NaT, NaT], [D1, D3, D5, D7, NaT]),
133
+ (mode, ["a", "a", "c", "c", "e", "f", "g", "", "", ""], ["a", "c", "e", "g", ""]),
134
+
135
+ (nth0, [T, T, T, T, T, F, F, F, F, F], [T, T, T, F, F]),
136
+ (nth0, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1, 3, 5, 7, 9]),
137
+ (nth0, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [1.0, 3.0, 5.0, 7.0, NaN]),
138
+ (nth0, [D1, D2, D3, D4, D5, D6, D7, NaT, NaT, NaT], [D1, D3, D5, D7, NaT]),
139
+ (nth0, ["a", "b", "c", "d", "e", "f", "g", "", "", ""], ["a", "c", "e", "g", ""]),
140
+
141
+ (quantile05, [T, T, T, T, T, F, F, F, F, F], [1.0, 1.0, 0.5, 0.0, 0.0]),
142
+ (quantile05, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [1.5, 3.5, 5.5, 7.5, 9.5]),
143
+ (quantile05, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [1.5, 3.5, 5.5, 7.0, NaN]),
144
+
145
+ (std, [T, T, T, T, T, F, F, F, F, F], [0.0, 0.0, 0.5, 0.0, 0.0]),
146
+ (std, [1, 1, 2, 3, 5, 7, 8, 12, 13, 21], [0.0, 0.5, 1.0, 2.0, 4.0]),
147
+ (std, [1.0, 1.0, 2.0, 3.0, 5.0, 7.0, 8.0, NaN, NaN, NaN], [0.0, 0.5, 1.0, NaN, NaN]),
148
+
149
+ (sum, [T, T, T, T, T, F, F, F, F, F], [2, 2, 1, 0, 0]),
150
+ (sum, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [3, 7, 11, 15, 19]),
151
+ (sum, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, NaN, NaN, NaN], [3.0, 7.0, 11.0, 7.0, 0.0]),
152
+
153
+ (var, [T, T, T, T, T, F, F, F, F, F], [0.0, 0.0, 0.25, 0.0, 0.0]),
154
+ (var, [1, 1, 2, 3, 5, 7, 8, 12, 13, 21], [0.0, 0.25, 1.0, 4.0, 16.0]),
155
+ (var, [1.0, 1.0, 2.0, 3.0, 5.0, 7.0, 8.0, NaN, NaN, NaN], [0.0, 0.25, 1.0, NaN, NaN]),
156
+
157
+ ]
158
+
159
+ class TestAggregate:
160
+
161
+ @pytest.mark.parametrize("use_numba", [False, True])
162
+ @pytest.mark.parametrize("function,input,output", TEST_MATRIX)
163
+ def test_aggregate(self, function, input, output, use_numba):
164
+ if use_numba and not dataiter.USE_NUMBA:
165
+ pytest.skip("No Numba")
166
+ with patch("dataiter.USE_NUMBA", use_numba):
167
+ data = DataFrame(g=GROUPS, a=input)
168
+ stat = data.group_by("g").aggregate(a=function("a"))
169
+ expected = Vector(output)
170
+ try:
171
+ assert stat.a.equal(expected)
172
+ except AssertionError:
173
+ print("")
174
+ print(data)
175
+ print("Expected:")
176
+ print(expected)
177
+ print("Got:")
178
+ print(stat.a)
179
+ raise
180
+
181
+ @pytest.mark.parametrize("use_numba", [False, True])
182
+ def test_aggregate_count(self, use_numba):
183
+ if use_numba and not dataiter.USE_NUMBA:
184
+ pytest.skip("No Numba")
185
+ with patch("dataiter.USE_NUMBA", use_numba):
186
+ data = DataFrame(g=GROUPS)
187
+ stat = data.group_by("g").aggregate(n=count())
188
+ assert (stat.n == 2).all()
189
+
190
+ def test_all(self):
191
+ assert all(EMPTY_VECTOR)
192
+ assert all(Vector([T, T]))
193
+ assert not all(Vector([T, F]))
194
+ assert not all(Vector([F, F]))
195
+
196
+ def test_any(self):
197
+ assert not any(EMPTY_VECTOR)
198
+ assert any(Vector([T, T]))
199
+ assert any(Vector([T, F]))
200
+ assert not any(Vector([F, F]))
201
+
202
+ def test_count(self):
203
+ assert count(EMPTY_VECTOR) == 0
204
+ assert count(Vector([1])) == 1
205
+ assert count(Vector([1, 2])) == 2
206
+ assert count(Vector([1, 2, NaN])) == 3
207
+ assert count(Vector([1, 2, NaN]), drop_na=True) == 2
208
+
209
+ def test_count_unique(self):
210
+ assert count_unique(EMPTY_VECTOR) == 0
211
+ assert count_unique(Vector([1])) == 1
212
+ assert count_unique(Vector([1, 1])) == 1
213
+ assert count_unique(Vector([1, 1, 2])) == 2
214
+ assert count_unique(Vector([1, 1, 2, NaN])) == 3
215
+ assert count_unique(Vector([1, 1, 2, NaN]), drop_na=True) == 2
216
+
217
+ def test_first(self):
218
+ assert first(Vector([1, 2, 3])) == 1
219
+ assert first(Vector([NaN, 1, 2]), drop_na=True) == 1
220
+
221
+ def test_first_nan(self):
222
+ assert np.isnan(first(EMPTY_VECTOR))
223
+ assert np.isnan(first(Vector([NaN, 1, 2])))
224
+
225
+ def test_last(self):
226
+ assert last(Vector([1, 2, 3])) == 3
227
+ assert last(Vector([1, 2, NaN]), drop_na=True) == 2
228
+
229
+ def test_last_nan(self):
230
+ assert np.isnan(last(EMPTY_VECTOR))
231
+ assert np.isnan(last(Vector([1, 2, NaN])))
232
+
233
+ def test_max(self):
234
+ assert max(Vector([3, 2, 1])) == 3
235
+ assert max(Vector([3, 2, NaN])) == 3
236
+
237
+ def test_max_nan(self):
238
+ assert np.isnan(max(EMPTY_VECTOR))
239
+ assert np.isnan(max(Vector([3, 2, NaN]), drop_na=False))
240
+
241
+ def test_mean(self):
242
+ assert np.isclose(mean(Vector([1, 2, 10])), 4.333333)
243
+ assert np.isclose(mean(Vector([1, 2, NaN])), 1.5)
244
+
245
+ def test_mean_nan(self):
246
+ assert np.isnan(mean(EMPTY_VECTOR))
247
+ assert np.isnan(mean(Vector([1, 2, NaN]), drop_na=False))
248
+
249
+ def test_median(self):
250
+ assert median(Vector([1, 4, 6, 8, 5])) == 5
251
+ assert median(Vector([1, 4, 6, NaN, NaN])) == 4
252
+
253
+ def test_median_nan(self):
254
+ assert np.isnan(median(EMPTY_VECTOR))
255
+ assert np.isnan(median(Vector([1, 4, NaN]), drop_na=False))
256
+
257
+ def test_min(self):
258
+ assert min(Vector([3, 2, 1])) == 1
259
+ assert min(Vector([3, 2, NaN])) == 2
260
+
261
+ def test_min_nan(self):
262
+ assert np.isnan(min(EMPTY_VECTOR))
263
+ assert np.isnan(min(Vector([3, 2, NaN]), drop_na=False))
264
+
265
+ def test_mode(self):
266
+ assert mode(Vector([1])) == 1
267
+ assert mode(Vector([1, 2])) == 1
268
+ assert mode(Vector([1, 2, 2])) == 2
269
+ assert mode(Vector([1, 2, 2, NaN])) == 2
270
+ assert mode(Vector([1, 2, 2, NaN]), drop_na=False) == 2
271
+
272
+ def test_mode_nan(self):
273
+ assert np.isnan(mode(EMPTY_VECTOR))
274
+ assert np.isnan(mode(Vector([NaN, NaN], float), drop_na=False))
275
+
276
+ def test_nth(self):
277
+ assert nth(Vector([1, 2, 3]), 0) == 1
278
+ assert nth(Vector([NaN, 1, 2]), 0, drop_na=True) == 1
279
+
280
+ def test_nth_nan(self):
281
+ assert np.isnan(nth(EMPTY_VECTOR, 0))
282
+ assert np.isnan(nth(Vector([NaN, 1, 2]), 0))
283
+
284
+ def test_quantile(self):
285
+ assert quantile(Vector([1, 4, 6, 8, 5]), 0.5) == 5
286
+ assert quantile(Vector([1, 4, 6, NaN, NaN]), 0.5) == 4
287
+
288
+ def test_quantile_nan(self):
289
+ assert np.isnan(quantile(EMPTY_VECTOR, 0.5))
290
+ assert np.isnan(quantile(Vector([1, 4, NaN]), 0.5, drop_na=False))
291
+
292
+ def test_std(self):
293
+ assert np.isclose(std(Vector([3, 6, 7])), 1.699673)
294
+ assert np.isclose(std(Vector([3, 6, NaN])), 1.5)
295
+
296
+ def test_std_nan(self):
297
+ assert np.isnan(std(EMPTY_VECTOR))
298
+ assert np.isnan(std(Vector([1])))
299
+ assert np.isnan(std(Vector([3, 6, NaN]), drop_na=False))
300
+
301
+ def test_sum(self):
302
+ assert sum(EMPTY_VECTOR) == 0
303
+ assert sum(Vector([1])) == 1
304
+ assert sum(Vector([1, 2])) == 3
305
+ assert sum(Vector([1, 2, NaN])) == 3
306
+
307
+ def test_sum_nan(self):
308
+ assert np.isnan(sum(Vector([1, 2, NaN]), drop_na=False))
309
+
310
+ def test_var(self):
311
+ assert np.isclose(var(Vector([3, 6, 7])), 2.888889)
312
+ assert np.isclose(var(Vector([3, 6, NaN])), 2.25)
313
+
314
+ def test_var_nan(self):
315
+ assert np.isnan(var(EMPTY_VECTOR))
316
+ assert np.isnan(var(Vector([1])))
317
+ assert np.isnan(var(Vector([3, 6, NaN]), drop_na=False))