perspective-python 4.2.0__cp311-abi3-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- perspective/__init__.py +396 -0
- perspective/extension/finos-perspective-nbextension.json +5 -0
- perspective/handlers/__init__.py +11 -0
- perspective/handlers/aiohttp.py +61 -0
- perspective/handlers/starlette.py +55 -0
- perspective/handlers/tornado.py +184 -0
- perspective/perspective.pyd +0 -0
- perspective/templates/exported_widget.html.template +35 -0
- perspective/tests/__init__.py +11 -0
- perspective/tests/async/test_async_client.py +83 -0
- perspective/tests/async/test_websocket_client.py +124 -0
- perspective/tests/conftest.py +272 -0
- perspective/tests/core/__init__.py +11 -0
- perspective/tests/core/test_async.py +351 -0
- perspective/tests/multi_threaded/__init__.py +11 -0
- perspective/tests/multi_threaded/test_multi_threaded.py +201 -0
- perspective/tests/server/__init__.py +11 -0
- perspective/tests/server/test_server.py +1016 -0
- perspective/tests/server/test_session.py +110 -0
- perspective/tests/table/__init__.py +11 -0
- perspective/tests/table/arrow/date32.arrow +0 -0
- perspective/tests/table/arrow/date64.arrow +0 -0
- perspective/tests/table/arrow/dict.arrow +0 -0
- perspective/tests/table/arrow/dict_update.arrow +0 -0
- perspective/tests/table/arrow/int_float_str.arrow +0 -0
- perspective/tests/table/arrow/int_float_str_file.arrow +0 -0
- perspective/tests/table/arrow/int_float_str_update.arrow +0 -0
- perspective/tests/table/object_sequence.py +402 -0
- perspective/tests/table/test_column_paths.py +89 -0
- perspective/tests/table/test_delete.py +124 -0
- perspective/tests/table/test_exception.py +65 -0
- perspective/tests/table/test_leaks.py +54 -0
- perspective/tests/table/test_ports.py +178 -0
- perspective/tests/table/test_remove.py +102 -0
- perspective/tests/table/test_table.py +641 -0
- perspective/tests/table/test_table_arrow.py +503 -0
- perspective/tests/table/test_table_datetime.py +2409 -0
- perspective/tests/table/test_table_infer.py +201 -0
- perspective/tests/table/test_table_limit.py +45 -0
- perspective/tests/table/test_table_numpy.py +1022 -0
- perspective/tests/table/test_table_pandas.py +1018 -0
- perspective/tests/table/test_table_polars.py +251 -0
- perspective/tests/table/test_table_view_table.py +130 -0
- perspective/tests/table/test_to_arrow.py +417 -0
- perspective/tests/table/test_to_arrow_lz4.py +32 -0
- perspective/tests/table/test_to_format.py +1024 -0
- perspective/tests/table/test_to_polars.py +26 -0
- perspective/tests/table/test_update.py +545 -0
- perspective/tests/table/test_update_arrow.py +980 -0
- perspective/tests/table/test_update_pandas.py +211 -0
- perspective/tests/table/test_view.py +2261 -0
- perspective/tests/table/test_view_expression.py +1940 -0
- perspective/tests/test_dependencies.py +53 -0
- perspective/tests/viewer/__init__.py +11 -0
- perspective/tests/viewer/test_viewer.py +246 -0
- perspective/tests/widget/__init__.py +11 -0
- perspective/tests/widget/test_widget.py +278 -0
- perspective/tests/widget/test_widget_pandas.py +453 -0
- perspective/virtual_servers/__init__.py +134 -0
- perspective/virtual_servers/clickhouse.py +245 -0
- perspective/virtual_servers/duckdb.py +236 -0
- perspective/widget/__init__.py +349 -0
- perspective/widget/viewer/__init__.py +15 -0
- perspective/widget/viewer/validate.py +22 -0
- perspective/widget/viewer/viewer.py +343 -0
- perspective/widget/viewer/viewer_traitlets.py +101 -0
- perspective_python-4.2.0.data/data/share/jupyter/labextensions/@perspective-dev/jupyterlab/install.json +5 -0
- perspective_python-4.2.0.data/data/share/jupyter/labextensions/@perspective-dev/jupyterlab/package.json +71 -0
- perspective_python-4.2.0.data/data/share/jupyter/labextensions/@perspective-dev/jupyterlab/static/253.5f5c9e80605aa4106a28.js +2 -0
- perspective_python-4.2.0.data/data/share/jupyter/labextensions/@perspective-dev/jupyterlab/static/253.5f5c9e80605aa4106a28.js.LICENSE.txt +25 -0
- perspective_python-4.2.0.data/data/share/jupyter/labextensions/@perspective-dev/jupyterlab/static/523.c030af5d3c4f67ff83f6.js +1 -0
- perspective_python-4.2.0.data/data/share/jupyter/labextensions/@perspective-dev/jupyterlab/static/remoteEntry.95a8ea1b44d96032833f.js +1 -0
- perspective_python-4.2.0.data/data/share/jupyter/labextensions/@perspective-dev/jupyterlab/static/style.js +4 -0
- perspective_python-4.2.0.data/data/share/jupyter/labextensions/@perspective-dev/jupyterlab/static/third-party-licenses.json +16 -0
- perspective_python-4.2.0.dist-info/METADATA +27 -0
- perspective_python-4.2.0.dist-info/RECORD +79 -0
- perspective_python-4.2.0.dist-info/WHEEL +4 -0
- perspective_python-4.2.0.dist-info/licenses/LICENSE.md +193 -0
- perspective_python-4.2.0.dist-info/licenses/LICENSE_THIRDPARTY_cargo.yml +17395 -0
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
|
2
|
+
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
|
3
|
+
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
|
4
|
+
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
|
5
|
+
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
|
6
|
+
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
|
7
|
+
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
|
8
|
+
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
|
9
|
+
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
|
10
|
+
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
|
11
|
+
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
|
12
|
+
|
|
13
|
+
from datetime import datetime, date
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
import pandas as pd
|
|
17
|
+
import pyarrow as pa
|
|
18
|
+
from pytest import fixture
|
|
19
|
+
from random import random, randint, choice
|
|
20
|
+
from faker import Faker
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# Our tests construct naive datetimes everywhere
|
|
24
|
+
# so setting it here is an easy way to fix it globally.
|
|
25
|
+
import os
|
|
26
|
+
|
|
27
|
+
# Perspective used to support datetime.date and datetime.datetime
|
|
28
|
+
# as Table() constructor arguments, but now we forward the parameters
|
|
29
|
+
# directly to JSON.loads. So to make sure the tests dont need to be
|
|
30
|
+
# so utterly transmogrified, we have this little hack :)
|
|
31
|
+
import json
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
os.environ["TZ"] = "UTC"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def new_encoder(self, obj):
|
|
38
|
+
if isinstance(obj, datetime):
|
|
39
|
+
return str(obj)
|
|
40
|
+
elif isinstance(obj, date):
|
|
41
|
+
return str(obj)
|
|
42
|
+
else:
|
|
43
|
+
return old(self, obj)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
old = json.JSONEncoder.default
|
|
47
|
+
json.JSONEncoder.default = new_encoder
|
|
48
|
+
|
|
49
|
+
fake = Faker()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _make_date_time_index(size, time_unit):
|
|
53
|
+
return pd.date_range("2000-01-01", periods=size, freq=time_unit)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _make_period_index(size, time_unit):
|
|
57
|
+
return pd.period_range(start="2000", periods=size, freq=time_unit)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _make_dataframe(index, size=10):
|
|
61
|
+
"""Create a new random dataframe of `size` and with a DateTimeIndex of
|
|
62
|
+
frequency `time_unit`.
|
|
63
|
+
"""
|
|
64
|
+
return pd.DataFrame(
|
|
65
|
+
index=index,
|
|
66
|
+
data={
|
|
67
|
+
"a": np.random.rand(size),
|
|
68
|
+
"b": np.random.rand(size),
|
|
69
|
+
"c": np.random.rand(size),
|
|
70
|
+
"d": np.random.rand(size),
|
|
71
|
+
},
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class Util:
|
|
76
|
+
@staticmethod
|
|
77
|
+
def make_arrow(names, data, types=None, legacy=False):
|
|
78
|
+
"""Create an arrow binary that can be loaded and manipulated from memory.
|
|
79
|
+
|
|
80
|
+
Args:
|
|
81
|
+
names (list): a list of str column names
|
|
82
|
+
data (list): a list of lists containing data for each column
|
|
83
|
+
types (list): an optional list of `pyarrow.type` function references.
|
|
84
|
+
Types will be inferred if not provided.
|
|
85
|
+
legacy (bool): if True, use legacy IPC format (pre-pyarrow 0.15). Defaults to False.
|
|
86
|
+
|
|
87
|
+
Returns:
|
|
88
|
+
bytes : a bytes object containing the arrow-serialized output.
|
|
89
|
+
"""
|
|
90
|
+
stream = pa.BufferOutputStream()
|
|
91
|
+
arrays = []
|
|
92
|
+
|
|
93
|
+
for idx, column in enumerate(data):
|
|
94
|
+
# only apply types if array is present
|
|
95
|
+
kwargs = {}
|
|
96
|
+
if types:
|
|
97
|
+
kwargs["type"] = types[idx]
|
|
98
|
+
arrays.append(pa.array(column, **kwargs))
|
|
99
|
+
|
|
100
|
+
batch = pa.RecordBatch.from_arrays(arrays, names)
|
|
101
|
+
table = pa.Table.from_batches([batch])
|
|
102
|
+
writer = pa.RecordBatchStreamWriter(
|
|
103
|
+
stream, table.schema
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
writer.write_table(table)
|
|
107
|
+
writer.close()
|
|
108
|
+
return stream.getvalue().to_pybytes()
|
|
109
|
+
|
|
110
|
+
@staticmethod
|
|
111
|
+
def make_arrow_from_pandas(df, schema=None, legacy=False):
|
|
112
|
+
"""Create an arrow binary from a Pandas dataframe.
|
|
113
|
+
|
|
114
|
+
Args:
|
|
115
|
+
df (:obj:`pandas.DataFrame`)
|
|
116
|
+
schema (:obj:`pyarrow.Schema`)
|
|
117
|
+
legacy (bool): if True, use legacy IPC format (pre-pyarrow 0.15). Defaults to False.
|
|
118
|
+
|
|
119
|
+
Returns:
|
|
120
|
+
bytes : a bytes object containing the arrow-serialized output.
|
|
121
|
+
"""
|
|
122
|
+
stream = pa.BufferOutputStream()
|
|
123
|
+
table = pa.Table.from_pandas(df, schema=schema)
|
|
124
|
+
|
|
125
|
+
writer = pa.RecordBatchStreamWriter(
|
|
126
|
+
stream, table.schema
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
writer.write_table(table)
|
|
130
|
+
writer.close()
|
|
131
|
+
return stream.getvalue().to_pybytes()
|
|
132
|
+
|
|
133
|
+
@staticmethod
|
|
134
|
+
def make_dictionary_arrow(names, data, types=None, legacy=False):
|
|
135
|
+
"""Create an arrow binary that can be loaded and manipulated from memory, with
|
|
136
|
+
each column being a dictionary array of `str` values and `int` indices.
|
|
137
|
+
|
|
138
|
+
Args:
|
|
139
|
+
names (list): a list of str column names
|
|
140
|
+
data (list:tuple): a list of tuples, the first value being a list of indices,
|
|
141
|
+
and the second value being a list of values.
|
|
142
|
+
types (list:list:pyarrow.func): a list of lists, containing the indices type and
|
|
143
|
+
dictionary value type for each array.
|
|
144
|
+
legacy (bool): if True, use legacy IPC format (pre-pyarrow 0.15). Defaults to False.
|
|
145
|
+
|
|
146
|
+
Returns:
|
|
147
|
+
bytes : a bytes object containing the arrow-serialized output.
|
|
148
|
+
"""
|
|
149
|
+
stream = pa.BufferOutputStream()
|
|
150
|
+
|
|
151
|
+
arrays = []
|
|
152
|
+
for idx, column in enumerate(data):
|
|
153
|
+
indice_type = pa.int64()
|
|
154
|
+
value_type = pa.string()
|
|
155
|
+
|
|
156
|
+
if types is not None:
|
|
157
|
+
indice_type = types[idx][0]
|
|
158
|
+
value_type = types[idx][1]
|
|
159
|
+
|
|
160
|
+
indices = pa.array(column[0], type=indice_type)
|
|
161
|
+
values = pa.array(column[1], type=value_type)
|
|
162
|
+
parray = pa.DictionaryArray.from_arrays(indices, values)
|
|
163
|
+
arrays.append(parray)
|
|
164
|
+
|
|
165
|
+
batch = pa.RecordBatch.from_arrays(arrays, names)
|
|
166
|
+
table = pa.Table.from_batches([batch])
|
|
167
|
+
writer = pa.RecordBatchStreamWriter(
|
|
168
|
+
stream, table.schema
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
writer.write_table(table)
|
|
172
|
+
writer.close()
|
|
173
|
+
return stream.getvalue().to_pybytes()
|
|
174
|
+
|
|
175
|
+
@staticmethod
|
|
176
|
+
def to_timestamp(obj):
|
|
177
|
+
"""Return an integer timestamp based on a date/datetime object."""
|
|
178
|
+
classname = obj.__class__.__name__
|
|
179
|
+
if classname == "date":
|
|
180
|
+
return int(datetime(obj.year, obj.month, obj.day).timestamp() * 1000)
|
|
181
|
+
elif classname == "datetime":
|
|
182
|
+
return int(obj.timestamp() * 1000)
|
|
183
|
+
else:
|
|
184
|
+
return -1
|
|
185
|
+
|
|
186
|
+
@staticmethod
|
|
187
|
+
def make_dataframe(size=10, freq="D"):
|
|
188
|
+
index = _make_date_time_index(size, freq)
|
|
189
|
+
return _make_dataframe(index, size)
|
|
190
|
+
|
|
191
|
+
@staticmethod
|
|
192
|
+
def make_period_dataframe(size=10):
|
|
193
|
+
index = _make_period_index(size, "M")
|
|
194
|
+
return _make_dataframe(index, size)
|
|
195
|
+
|
|
196
|
+
@staticmethod
|
|
197
|
+
def make_series(size=10, freq="D"):
|
|
198
|
+
index = _make_date_time_index(size, freq)
|
|
199
|
+
return pd.Series(data=np.random.rand(size), index=index)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
class Sentinel(object):
|
|
203
|
+
"""Generic sentinel class for testing side-effectful code in Python 2 and
|
|
204
|
+
3.
|
|
205
|
+
"""
|
|
206
|
+
|
|
207
|
+
def __init__(self, value):
|
|
208
|
+
self.value = value
|
|
209
|
+
|
|
210
|
+
def get(self):
|
|
211
|
+
return self.value
|
|
212
|
+
|
|
213
|
+
def set(self, new_value):
|
|
214
|
+
self.value = new_value
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
@fixture()
|
|
218
|
+
def sentinel():
|
|
219
|
+
"""Pass `sentinel` into a test and call it with `value` to create a new
|
|
220
|
+
instance of the Sentinel class.
|
|
221
|
+
|
|
222
|
+
Example:
|
|
223
|
+
>>> def test_with_sentinel(self, sentinel):
|
|
224
|
+
>>> s = sentinel(True)
|
|
225
|
+
>>> s.set(False)
|
|
226
|
+
>>> s.get() # returns False
|
|
227
|
+
"""
|
|
228
|
+
|
|
229
|
+
def _sentinel(value):
|
|
230
|
+
return Sentinel(value)
|
|
231
|
+
|
|
232
|
+
return _sentinel
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
@fixture
|
|
236
|
+
def util():
|
|
237
|
+
"""Pass the `Util` class in to a test."""
|
|
238
|
+
return Util
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
@fixture
|
|
242
|
+
def superstore(count=100):
|
|
243
|
+
data = []
|
|
244
|
+
for id in range(count):
|
|
245
|
+
dat = {}
|
|
246
|
+
dat["Row ID"] = id
|
|
247
|
+
dat["Order ID"] = "{}-{}".format(fake.ein(), fake.zipcode())
|
|
248
|
+
dat["Order Date"] = fake.date_this_year()
|
|
249
|
+
dat["Ship Date"] = fake.date_between_dates(dat["Order Date"]).strftime(
|
|
250
|
+
"%Y-%m-%d"
|
|
251
|
+
)
|
|
252
|
+
dat["Order Date"] = dat["Order Date"].strftime("%Y-%m-%d")
|
|
253
|
+
dat["Ship Mode"] = choice(["First Class", "Standard Class", "Second Class"])
|
|
254
|
+
dat["Ship Mode"] = choice(["First Class", "Standard Class", "Second Class"])
|
|
255
|
+
dat["Customer ID"] = fake.zipcode()
|
|
256
|
+
dat["Segment"] = choice(["A", "B", "C", "D"])
|
|
257
|
+
dat["Country"] = "US"
|
|
258
|
+
dat["City"] = fake.city()
|
|
259
|
+
dat["State"] = fake.state()
|
|
260
|
+
dat["Postal Code"] = fake.zipcode()
|
|
261
|
+
dat["Region"] = choice(["Region %d" % i for i in range(5)])
|
|
262
|
+
dat["Product ID"] = fake.bban()
|
|
263
|
+
sector = choice(["Industrials", "Technology", "Financials"])
|
|
264
|
+
industry = choice(["A", "B", "C"])
|
|
265
|
+
dat["Category"] = sector
|
|
266
|
+
dat["Sub-Category"] = industry
|
|
267
|
+
dat["Sales"] = randint(1, 100) * 100
|
|
268
|
+
dat["Quantity"] = randint(1, 100) * 10
|
|
269
|
+
dat["Discount"] = round(random() * 100, 2)
|
|
270
|
+
dat["Profit"] = round(random() * 1000, 2)
|
|
271
|
+
data.append(dat)
|
|
272
|
+
return pd.DataFrame(data)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
|
2
|
+
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
|
3
|
+
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
|
4
|
+
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
|
5
|
+
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
|
6
|
+
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
|
7
|
+
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
|
8
|
+
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
|
9
|
+
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
|
10
|
+
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
|
11
|
+
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
|
2
|
+
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
|
3
|
+
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
|
4
|
+
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
|
5
|
+
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
|
6
|
+
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
|
7
|
+
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
|
8
|
+
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
|
9
|
+
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
|
10
|
+
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
|
11
|
+
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
|
12
|
+
|
|
13
|
+
import queue
|
|
14
|
+
import random
|
|
15
|
+
import threading
|
|
16
|
+
from functools import partial
|
|
17
|
+
import tornado
|
|
18
|
+
|
|
19
|
+
from perspective import (
|
|
20
|
+
Server,
|
|
21
|
+
Client,
|
|
22
|
+
)
|
|
23
|
+
from pytest import mark
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def syncify(f):
|
|
27
|
+
"""Given a function `f` that must be run on `TestAsync.loop`, queue `f` on
|
|
28
|
+
the loop, block until it is evaluated, and return the result.
|
|
29
|
+
"""
|
|
30
|
+
sem = queue.Queue()
|
|
31
|
+
|
|
32
|
+
def _syncify_task():
|
|
33
|
+
assert threading.current_thread().ident == TestAsync.thread.ident
|
|
34
|
+
result = f()
|
|
35
|
+
TestAsync.loop.add_callback(lambda: sem.put(result))
|
|
36
|
+
|
|
37
|
+
def _syncify():
|
|
38
|
+
TestAsync.loop.add_callback(_syncify_task)
|
|
39
|
+
return sem.get()
|
|
40
|
+
|
|
41
|
+
return _syncify
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
data = [{"a": i, "b": i * 0.5, "c": str(i)} for i in range(10)]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class TestAsync(object):
|
|
48
|
+
@classmethod
|
|
49
|
+
def setup_class(cls):
|
|
50
|
+
import asyncio
|
|
51
|
+
|
|
52
|
+
# Storing the current loop, which was set up by the pytest-asyncio
|
|
53
|
+
# tests running in tests/async, delays its cleanup, which foregoes a
|
|
54
|
+
# pytest exception about an unclosed socket. A cleaner way to fix this
|
|
55
|
+
# probably exists (use an event loop fixture? convert these tests to
|
|
56
|
+
# pytest-asyncio?)
|
|
57
|
+
cls.save_old_loop_to_prevent_unclosed_socket_exception = (
|
|
58
|
+
asyncio.get_event_loop()
|
|
59
|
+
)
|
|
60
|
+
asyncio.set_event_loop(asyncio.new_event_loop())
|
|
61
|
+
cls.loop = tornado.ioloop.IOLoop.current()
|
|
62
|
+
cls.thread = threading.Thread(target=cls.loop.start)
|
|
63
|
+
cls.thread.daemon = True
|
|
64
|
+
cls.thread.start()
|
|
65
|
+
|
|
66
|
+
@classmethod
|
|
67
|
+
def teardown_class(cls):
|
|
68
|
+
cls.loop.add_callback(lambda: tornado.ioloop.IOLoop.current().stop())
|
|
69
|
+
cls.thread.join()
|
|
70
|
+
cls.loop.close(all_fds=True)
|
|
71
|
+
|
|
72
|
+
@classmethod
|
|
73
|
+
def loop_is_running(cls):
|
|
74
|
+
return cls.loop.asyncio_loop.is_running()
|
|
75
|
+
|
|
76
|
+
def test_async_queue_process(self):
|
|
77
|
+
server = Server()
|
|
78
|
+
client = Client.from_server(
|
|
79
|
+
server,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
tbl = client.table({"a": "integer", "b": "float", "c": "string"})
|
|
83
|
+
|
|
84
|
+
@syncify
|
|
85
|
+
def _task():
|
|
86
|
+
assert tbl.size() == 0
|
|
87
|
+
for i in range(5):
|
|
88
|
+
tbl.update([data[i]])
|
|
89
|
+
return tbl.size()
|
|
90
|
+
|
|
91
|
+
assert _task() == 5
|
|
92
|
+
tbl.delete()
|
|
93
|
+
|
|
94
|
+
def test_async_queue_process_csv(self):
|
|
95
|
+
"""Make sure GIL release during CSV loading works"""
|
|
96
|
+
server = Server()
|
|
97
|
+
client = Client.from_server(
|
|
98
|
+
server,
|
|
99
|
+
)
|
|
100
|
+
tbl = client.table("x,y,z\n1,a,true\n2,b,false\n3,c,true\n4,d,false")
|
|
101
|
+
|
|
102
|
+
@syncify
|
|
103
|
+
def _task():
|
|
104
|
+
assert tbl.size() == 4
|
|
105
|
+
for i in range(5):
|
|
106
|
+
tbl.update("x,y,z\n1,a,true\n2,b,false\n3,c,true\n4,d,false")
|
|
107
|
+
return tbl.size()
|
|
108
|
+
|
|
109
|
+
assert _task() == 24
|
|
110
|
+
|
|
111
|
+
tbl.delete()
|
|
112
|
+
|
|
113
|
+
def test_async_call_loop(self):
|
|
114
|
+
server = Server()
|
|
115
|
+
client = Client.from_server(
|
|
116
|
+
server,
|
|
117
|
+
)
|
|
118
|
+
tbl = client.table({"a": "integer", "b": "float", "c": "string"})
|
|
119
|
+
tbl.update(data)
|
|
120
|
+
|
|
121
|
+
@syncify
|
|
122
|
+
def _task():
|
|
123
|
+
return tbl.size()
|
|
124
|
+
|
|
125
|
+
assert _task() == 10
|
|
126
|
+
tbl.delete()
|
|
127
|
+
|
|
128
|
+
def test_async_multiple_managers_queue_process(self):
|
|
129
|
+
server = Server()
|
|
130
|
+
client = Client.from_server(server)
|
|
131
|
+
server2 = Server()
|
|
132
|
+
client2 = Client.from_server(server2)
|
|
133
|
+
tbl = client.table({"a": "integer", "b": "float", "c": "string"})
|
|
134
|
+
tbl2 = client2.table({"a": "integer", "b": "float", "c": "string"})
|
|
135
|
+
|
|
136
|
+
@syncify
|
|
137
|
+
def _update_task():
|
|
138
|
+
for i in range(5):
|
|
139
|
+
tbl.update([data[i]])
|
|
140
|
+
tbl2.update([data[i]])
|
|
141
|
+
return tbl.size()
|
|
142
|
+
|
|
143
|
+
assert _update_task() == 5
|
|
144
|
+
|
|
145
|
+
@syncify
|
|
146
|
+
def _flush_to_process():
|
|
147
|
+
view = tbl2.view()
|
|
148
|
+
records = view.to_records()
|
|
149
|
+
for i in range(5):
|
|
150
|
+
tbl2.update([data[i]])
|
|
151
|
+
|
|
152
|
+
view.delete()
|
|
153
|
+
return records
|
|
154
|
+
|
|
155
|
+
assert _flush_to_process() == data[:5]
|
|
156
|
+
|
|
157
|
+
@syncify
|
|
158
|
+
def _delete_task():
|
|
159
|
+
tbl2.delete()
|
|
160
|
+
tbl.delete()
|
|
161
|
+
|
|
162
|
+
_delete_task()
|
|
163
|
+
|
|
164
|
+
@mark.skip(
|
|
165
|
+
reason="This test is failing because we're not calling process after each update like before"
|
|
166
|
+
)
|
|
167
|
+
def test_async_multiple_managers_mixed_queue_process(self):
|
|
168
|
+
sentinel = {"called": 0}
|
|
169
|
+
|
|
170
|
+
def sync_queue_process(f, *args, **kwargs):
|
|
171
|
+
sentinel["called"] += 1
|
|
172
|
+
f(*args, **kwargs)
|
|
173
|
+
|
|
174
|
+
server = Server()
|
|
175
|
+
client = Client.from_server(server)
|
|
176
|
+
|
|
177
|
+
server2 = Server()
|
|
178
|
+
client2 = Client.from_server(server2)
|
|
179
|
+
tbl = client.table({"a": "integer", "b": "float", "c": "string"})
|
|
180
|
+
tbl2 = client2.table({"a": "integer", "b": "float", "c": "string"})
|
|
181
|
+
|
|
182
|
+
@syncify
|
|
183
|
+
def _tbl_task():
|
|
184
|
+
for i in range(5):
|
|
185
|
+
tbl.update([data[i]])
|
|
186
|
+
return tbl.size()
|
|
187
|
+
|
|
188
|
+
assert _tbl_task() == 5
|
|
189
|
+
|
|
190
|
+
for i in range(5):
|
|
191
|
+
tbl2.update([data[i]])
|
|
192
|
+
|
|
193
|
+
assert sentinel["called"] == 5
|
|
194
|
+
|
|
195
|
+
@syncify
|
|
196
|
+
def _tbl_task2():
|
|
197
|
+
view = tbl.view()
|
|
198
|
+
records = view.to_records()
|
|
199
|
+
view.delete()
|
|
200
|
+
tbl.delete()
|
|
201
|
+
return records
|
|
202
|
+
|
|
203
|
+
assert _tbl_task2() == data[:5]
|
|
204
|
+
|
|
205
|
+
view = tbl2.view()
|
|
206
|
+
assert view.to_records() == data[:5]
|
|
207
|
+
|
|
208
|
+
view.delete()
|
|
209
|
+
tbl2.delete()
|
|
210
|
+
|
|
211
|
+
@mark.skip(
|
|
212
|
+
reason="This test is failing because we're not calling process after each update like before"
|
|
213
|
+
)
|
|
214
|
+
def test_async_multiple_managers_delayed_process(self):
|
|
215
|
+
sentinel = {"async": 0, "sync": 0}
|
|
216
|
+
|
|
217
|
+
def _counter(key, f, *args, **kwargs):
|
|
218
|
+
sentinel[key] += 1
|
|
219
|
+
return f(*args, **kwargs)
|
|
220
|
+
|
|
221
|
+
short_delay_queue_process = partial(_counter, "sync")
|
|
222
|
+
long_delay_queue_process = partial(
|
|
223
|
+
TestAsync.loop.add_timeout, 1, _counter, "async"
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
server = Server(on_poll_request=short_delay_queue_process)
|
|
227
|
+
client = Client.from_server(server)
|
|
228
|
+
|
|
229
|
+
server2 = Server(on_poll_request=long_delay_queue_process)
|
|
230
|
+
client2 = Client.from_server(server2)
|
|
231
|
+
tbl = client.table({"a": "integer", "b": "float", "c": "string"})
|
|
232
|
+
tbl2 = client2.table({"a": "integer", "b": "float", "c": "string"})
|
|
233
|
+
|
|
234
|
+
@syncify
|
|
235
|
+
def _tbl_task():
|
|
236
|
+
for i in range(10):
|
|
237
|
+
tbl2.update([data[i]])
|
|
238
|
+
|
|
239
|
+
_tbl_task()
|
|
240
|
+
for i in range(10):
|
|
241
|
+
tbl.update([data[i]])
|
|
242
|
+
|
|
243
|
+
@syncify
|
|
244
|
+
def _tbl_task2():
|
|
245
|
+
size = tbl2.size()
|
|
246
|
+
tbl2.delete()
|
|
247
|
+
return size
|
|
248
|
+
|
|
249
|
+
assert _tbl_task2() == 10
|
|
250
|
+
assert tbl.size() == 10
|
|
251
|
+
assert sentinel["async"] == 1
|
|
252
|
+
assert sentinel["sync"] == 10
|
|
253
|
+
|
|
254
|
+
tbl.delete()
|
|
255
|
+
|
|
256
|
+
def test_async_single_manager_tables_chained(self):
|
|
257
|
+
def call_loop(fn, *args):
|
|
258
|
+
TestAsync.loop.add_callback(fn, *args)
|
|
259
|
+
|
|
260
|
+
server = Server()
|
|
261
|
+
client = Client.from_server(server)
|
|
262
|
+
columns = {"index": "integer", "num1": "integer", "num2": "integer"}
|
|
263
|
+
# tbl = client.table(columns, index="index")
|
|
264
|
+
tbl = client.table(columns)
|
|
265
|
+
view = tbl.view()
|
|
266
|
+
tbl2 = client.table(view.to_arrow())
|
|
267
|
+
|
|
268
|
+
def _update(port_id, delta):
|
|
269
|
+
print("Updating tbl2", delta)
|
|
270
|
+
tbl2.update(delta)
|
|
271
|
+
|
|
272
|
+
view.on_update(_update, mode="row")
|
|
273
|
+
for i in range(1000):
|
|
274
|
+
call_loop(tbl.update, [{"index": i, "num1": i, "num2": 2 * i}])
|
|
275
|
+
i += 1
|
|
276
|
+
|
|
277
|
+
call_loop(tbl.size)
|
|
278
|
+
|
|
279
|
+
q = queue.Queue()
|
|
280
|
+
call_loop(q.put, True)
|
|
281
|
+
q.get()
|
|
282
|
+
|
|
283
|
+
@syncify
|
|
284
|
+
def _tbl_task2():
|
|
285
|
+
size = tbl2.size()
|
|
286
|
+
return size
|
|
287
|
+
|
|
288
|
+
assert _tbl_task2() == 1000
|
|
289
|
+
# assert tbl2.size() == 1000
|
|
290
|
+
view.delete()
|
|
291
|
+
tbl.delete()
|
|
292
|
+
tbl2.delete()
|
|
293
|
+
|
|
294
|
+
def test_async_queue_process_multiple_ports(self):
|
|
295
|
+
server = Server()
|
|
296
|
+
client = Client.from_server(server)
|
|
297
|
+
tbl = client.table({"a": "integer", "b": "float", "c": "string"})
|
|
298
|
+
port_ids = [0]
|
|
299
|
+
port_data = [{"a": 0, "b": 0, "c": "0"}]
|
|
300
|
+
|
|
301
|
+
for i in range(10):
|
|
302
|
+
port_id = tbl.make_port()
|
|
303
|
+
port_ids.append(port_id)
|
|
304
|
+
port_data.append({"a": port_id, "b": port_id * 1.5, "c": str(port_id)})
|
|
305
|
+
|
|
306
|
+
assert port_ids == list(range(0, 11))
|
|
307
|
+
|
|
308
|
+
assert syncify(lambda: tbl.size())() == 0
|
|
309
|
+
|
|
310
|
+
random.shuffle(port_ids)
|
|
311
|
+
|
|
312
|
+
@syncify
|
|
313
|
+
def _tbl_task():
|
|
314
|
+
for port_id in port_ids:
|
|
315
|
+
idx = port_id if port_id < len(port_ids) else len(port_ids) - 1
|
|
316
|
+
tbl.update([port_data[idx]], port_id=port_id)
|
|
317
|
+
size = tbl.size()
|
|
318
|
+
tbl.delete()
|
|
319
|
+
return size
|
|
320
|
+
|
|
321
|
+
assert len(port_ids) == 11
|
|
322
|
+
assert _tbl_task() == 11
|
|
323
|
+
|
|
324
|
+
def test_async_multiple_managers_queue_process_multiple_ports(self):
|
|
325
|
+
server = Server()
|
|
326
|
+
client = Client.from_server(server)
|
|
327
|
+
|
|
328
|
+
server2 = Server()
|
|
329
|
+
client2 = Client.from_server(server2)
|
|
330
|
+
tbl = client.table({"a": "integer", "b": "float", "c": "string"})
|
|
331
|
+
tbl2 = client2.table({"a": "integer", "b": "float", "c": "string"})
|
|
332
|
+
port_ids = [0]
|
|
333
|
+
port_data = [{"a": 0, "b": 0, "c": "0"}]
|
|
334
|
+
|
|
335
|
+
for i in range(10):
|
|
336
|
+
port_id = tbl.make_port()
|
|
337
|
+
port_id2 = tbl2.make_port()
|
|
338
|
+
assert port_id == port_id2
|
|
339
|
+
port_ids.append(port_id)
|
|
340
|
+
port_data.append({"a": port_id, "b": port_id * 1.5, "c": str(port_id)})
|
|
341
|
+
|
|
342
|
+
@syncify
|
|
343
|
+
def _task():
|
|
344
|
+
random.shuffle(port_ids)
|
|
345
|
+
for port_id in port_ids:
|
|
346
|
+
idx = port_id if port_id < len(port_ids) else len(port_ids) - 1
|
|
347
|
+
tbl.update([port_data[idx]], port_id=port_id)
|
|
348
|
+
tbl2.update([port_data[idx]], port_id=port_id)
|
|
349
|
+
return (tbl.size(), tbl2.size())
|
|
350
|
+
|
|
351
|
+
assert _task() == (11, 11)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
|
2
|
+
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
|
3
|
+
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
|
4
|
+
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
|
5
|
+
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
|
6
|
+
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
|
7
|
+
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
|
8
|
+
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
|
9
|
+
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
|
10
|
+
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
|
11
|
+
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|