pyvega 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyvega/__init__.py +919 -0
- pyvega/_client.py +490 -0
- pyvega/_io.py +58 -0
- pyvega/_native.py +1031 -0
- pyvega/_protocol.py +171 -0
- pyvega/_sampler.py +67 -0
- pyvega/_server_cli.py +28 -0
- pyvega/_stats.py +113 -0
- pyvega/broadcast.py +78 -0
- pyvega-0.5.0.dist-info/METADATA +282 -0
- pyvega-0.5.0.dist-info/RECORD +16 -0
- pyvega-0.5.0.dist-info/WHEEL +5 -0
- pyvega-0.5.0.dist-info/entry_points.txt +2 -0
- pyvega-0.5.0.dist-info/licenses/LICENSE +21 -0
- pyvega-0.5.0.dist-info/licenses/NOTICE +42 -0
- pyvega-0.5.0.dist-info/top_level.txt +1 -0
pyvega/__init__.py
ADDED
|
@@ -0,0 +1,919 @@
|
|
|
1
|
+
"""
|
|
2
|
+
PyVega -- Python client for the Vega C++ engine.
|
|
3
|
+
|
|
4
|
+
Connects to a Vega session server over TCP. Computation and the actor
|
|
5
|
+
system run in the C++ server process; this package is a thin RPC client.
|
|
6
|
+
|
|
7
|
+
from pyvega import SparkContext
|
|
8
|
+
|
|
9
|
+
with SparkContext(4) as sc:
|
|
10
|
+
result = sc.range(0, 100).map(lambda x: x * 2).collect()
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import bisect
|
|
16
|
+
import copy
|
|
17
|
+
import math
|
|
18
|
+
import os
|
|
19
|
+
import shlex
|
|
20
|
+
import subprocess
|
|
21
|
+
import threading
|
|
22
|
+
from subprocess import PIPE, Popen
|
|
23
|
+
from typing import Any, Callable, Iterable, Iterator, Optional
|
|
24
|
+
|
|
25
|
+
from pyvega._client import PendingStage, SessionClient, parse_remote, spawn_server
|
|
26
|
+
from pyvega._io import (
|
|
27
|
+
read_pickle_file,
|
|
28
|
+
read_text_file,
|
|
29
|
+
save_as_pickle_file,
|
|
30
|
+
save_as_text_file,
|
|
31
|
+
)
|
|
32
|
+
from pyvega._native import NativeKind, try_compile
|
|
33
|
+
from pyvega._protocol import Action, StageOp
|
|
34
|
+
from pyvega._sampler import RDDSampler
|
|
35
|
+
from pyvega._stats import count_approx_distinct, histogram_partition, histogram_values
|
|
36
|
+
from pyvega.broadcast import Broadcast, BroadcastPickleRegistry
|
|
37
|
+
|
|
38
|
+
__version__ = "0.5.0"
|
|
39
|
+
__all__ = ["SparkContext", "RDD"]
|
|
40
|
+
|
|
41
|
+
_next_rdd_id = 1
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class RDD:
|
|
45
|
+
"""Remote RDD handle backed by the Vega C++ server.
|
|
46
|
+
|
|
47
|
+
Transformations are recorded client-side and sent as one pipeline when an
|
|
48
|
+
action runs, avoiding per-transform RPC round-trips.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
client: SessionClient,
|
|
54
|
+
rdd_id: int,
|
|
55
|
+
num_partitions: int,
|
|
56
|
+
pending: Optional[list[PendingStage]] = None,
|
|
57
|
+
native: bool = False,
|
|
58
|
+
native_kind: NativeKind = "int64",
|
|
59
|
+
line_meta: Optional[tuple[int, int]] = None,
|
|
60
|
+
context: Optional["SparkContext"] = None,
|
|
61
|
+
rdd_id_counter: Optional[int] = None,
|
|
62
|
+
text_file_path: Optional[str] = None,
|
|
63
|
+
text_file_parts: int = 0,
|
|
64
|
+
):
|
|
65
|
+
global _next_rdd_id
|
|
66
|
+
self._client = client
|
|
67
|
+
self._rdd_id = rdd_id
|
|
68
|
+
self._num_partitions = num_partitions
|
|
69
|
+
self._pending = list(pending or [])
|
|
70
|
+
self._native = native
|
|
71
|
+
self._native_kind: NativeKind = native_kind
|
|
72
|
+
self._line_meta = line_meta
|
|
73
|
+
self._context = context
|
|
74
|
+
self._text_file_path = text_file_path
|
|
75
|
+
self._text_file_parts = text_file_parts
|
|
76
|
+
if rdd_id_counter is None:
|
|
77
|
+
rdd_id_counter = _next_rdd_id
|
|
78
|
+
_next_rdd_id += 1
|
|
79
|
+
self._logical_id = rdd_id_counter
|
|
80
|
+
|
|
81
|
+
def _derive(self, stage: PendingStage, native: bool, native_kind: NativeKind) -> RDD:
|
|
82
|
+
new_id = self._logical_id + (0 if self._pending else 1)
|
|
83
|
+
return RDD(
|
|
84
|
+
self._client,
|
|
85
|
+
self._rdd_id,
|
|
86
|
+
self._num_partitions,
|
|
87
|
+
self._pending + [stage],
|
|
88
|
+
native=native,
|
|
89
|
+
native_kind=native_kind,
|
|
90
|
+
line_meta=self._line_meta,
|
|
91
|
+
context=self._context,
|
|
92
|
+
rdd_id_counter=new_id,
|
|
93
|
+
text_file_path=self._text_file_path,
|
|
94
|
+
text_file_parts=self._text_file_parts,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
def _refresh_text_file_source(self) -> None:
|
|
98
|
+
if not self._text_file_path:
|
|
99
|
+
return
|
|
100
|
+
with open(self._text_file_path, encoding="utf-8") as handle:
|
|
101
|
+
lines = [line.rstrip("\n") for line in handle]
|
|
102
|
+
parts = self._text_file_parts or self._num_partitions or 1
|
|
103
|
+
native, native_kind = SparkContext._detect_native_kind(lines)
|
|
104
|
+
self._rdd_id, self._num_partitions = self._client.parallelize(lines, parts)
|
|
105
|
+
self._native = native
|
|
106
|
+
self._native_kind = native_kind
|
|
107
|
+
|
|
108
|
+
def _stage(self, op: StageOp, func, num_partitions: int = 0) -> RDD:
|
|
109
|
+
native = self._native
|
|
110
|
+
native_kind = self._native_kind
|
|
111
|
+
native_op = None
|
|
112
|
+
native_params: tuple[int | float | str, ...] = ()
|
|
113
|
+
if native and op != StageOp.GROUP_BY_KEY:
|
|
114
|
+
compiled = try_compile(op, func, native_kind)
|
|
115
|
+
if compiled is not None:
|
|
116
|
+
native_op = compiled.native_op
|
|
117
|
+
native_params = compiled.params
|
|
118
|
+
else:
|
|
119
|
+
native = False
|
|
120
|
+
return self._derive(
|
|
121
|
+
PendingStage(op, func, num_partitions, native_op, native_params),
|
|
122
|
+
native,
|
|
123
|
+
native_kind,
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
@staticmethod
|
|
127
|
+
def _normalize_func(func: Callable[..., Any]) -> Callable[..., Any]:
|
|
128
|
+
if func is range:
|
|
129
|
+
return lambda x: list(range(x))
|
|
130
|
+
if isinstance(func, type) or func in (str, int, float, bool, list, tuple, set, dict):
|
|
131
|
+
return lambda x, f=func: f(x)
|
|
132
|
+
return func
|
|
133
|
+
|
|
134
|
+
def _materialize(self) -> None:
|
|
135
|
+
self._refresh_text_file_source()
|
|
136
|
+
if not self._pending:
|
|
137
|
+
return
|
|
138
|
+
rdd_id, parts = self._client.materialize(
|
|
139
|
+
self._rdd_id,
|
|
140
|
+
self._num_partitions,
|
|
141
|
+
self._pending,
|
|
142
|
+
self._line_meta,
|
|
143
|
+
)
|
|
144
|
+
self._rdd_id = rdd_id
|
|
145
|
+
self._num_partitions = parts
|
|
146
|
+
self._pending = []
|
|
147
|
+
|
|
148
|
+
def _collect_partitions(self) -> list[list[Any]]:
|
|
149
|
+
self._materialize()
|
|
150
|
+
return self._client.collect_partitions(self._rdd_id)
|
|
151
|
+
|
|
152
|
+
def _collect_partition(self, partition: int) -> list[Any]:
|
|
153
|
+
self._materialize()
|
|
154
|
+
return self._client.collect_partition(self._rdd_id, partition)
|
|
155
|
+
|
|
156
|
+
def _from_partitions(
|
|
157
|
+
self,
|
|
158
|
+
partitions: list[list[Any]],
|
|
159
|
+
native: bool = False,
|
|
160
|
+
native_kind: NativeKind = "int64",
|
|
161
|
+
) -> RDD:
|
|
162
|
+
ctx = self._require_context()
|
|
163
|
+
rdd_id, parts = self._client.parallelize_partitions(partitions)
|
|
164
|
+
return RDD(
|
|
165
|
+
self._client,
|
|
166
|
+
rdd_id,
|
|
167
|
+
parts,
|
|
168
|
+
native=native,
|
|
169
|
+
native_kind=native_kind,
|
|
170
|
+
context=ctx,
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
def map_partitions(self, func, preserves_partitioning: bool = False) -> RDD:
|
|
174
|
+
parts = self._collect_partitions()
|
|
175
|
+
mapped = [list(func(iter(part))) for part in parts]
|
|
176
|
+
if preserves_partitioning:
|
|
177
|
+
return self._from_partitions(mapped)
|
|
178
|
+
flat = [item for part in mapped for item in part]
|
|
179
|
+
return self._require_context().parallelize(flat, self._num_partitions)
|
|
180
|
+
|
|
181
|
+
mapPartitions = map_partitions
|
|
182
|
+
|
|
183
|
+
def map_partitions_with_index(
|
|
184
|
+
self, func, preserves_partitioning: bool = False
|
|
185
|
+
) -> RDD:
|
|
186
|
+
parts = self._collect_partitions()
|
|
187
|
+
mapped = [list(func(index, iter(part))) for index, part in enumerate(parts)]
|
|
188
|
+
if preserves_partitioning:
|
|
189
|
+
return self._from_partitions(mapped)
|
|
190
|
+
flat = [item for part in mapped for item in part]
|
|
191
|
+
return self._require_context().parallelize(flat, self._num_partitions)
|
|
192
|
+
|
|
193
|
+
mapPartitionsWithIndex = map_partitions_with_index
|
|
194
|
+
|
|
195
|
+
def glom(self) -> RDD:
|
|
196
|
+
def func(iterator: Iterable[Any]) -> Iterable[list[Any]]:
|
|
197
|
+
yield list(iterator)
|
|
198
|
+
|
|
199
|
+
return self.map_partitions(func, preserves_partitioning=True)
|
|
200
|
+
|
|
201
|
+
def partition_by(self, num_partitions: int, partition_func) -> RDD:
|
|
202
|
+
buckets: list[list[Any]] = [[] for _ in range(num_partitions)]
|
|
203
|
+
for part in self._collect_partitions():
|
|
204
|
+
for item in part:
|
|
205
|
+
if isinstance(item, tuple) and len(item) >= 1:
|
|
206
|
+
key = item[0]
|
|
207
|
+
else:
|
|
208
|
+
key = item
|
|
209
|
+
bucket = int(partition_func(key)) % num_partitions
|
|
210
|
+
buckets[bucket].append(item)
|
|
211
|
+
return self._from_partitions(buckets)
|
|
212
|
+
|
|
213
|
+
partitionBy = partition_by
|
|
214
|
+
|
|
215
|
+
def map(self, func) -> RDD:
|
|
216
|
+
return self._stage(StageOp.MAP, self._normalize_func(func))
|
|
217
|
+
|
|
218
|
+
def filter(self, func) -> RDD:
|
|
219
|
+
return self._stage(StageOp.FILTER, func)
|
|
220
|
+
|
|
221
|
+
def flat_map(self, func) -> RDD:
|
|
222
|
+
return self._stage(StageOp.FLAT_MAP, self._normalize_func(func))
|
|
223
|
+
|
|
224
|
+
flatMap = flat_map
|
|
225
|
+
|
|
226
|
+
def reduce_by_key(self, func, num_partitions: int = 0) -> RDD:
|
|
227
|
+
if num_partitions == 0:
|
|
228
|
+
num_partitions = self._num_partitions
|
|
229
|
+
return self._stage(StageOp.REDUCE_BY_KEY, func, num_partitions)
|
|
230
|
+
|
|
231
|
+
reduceByKey = reduce_by_key
|
|
232
|
+
|
|
233
|
+
def group_by_key(self, num_partitions: int = 0) -> RDD:
|
|
234
|
+
if num_partitions == 0:
|
|
235
|
+
num_partitions = self._num_partitions
|
|
236
|
+
return self._stage(StageOp.GROUP_BY_KEY, None, num_partitions)
|
|
237
|
+
|
|
238
|
+
groupByKey = group_by_key
|
|
239
|
+
|
|
240
|
+
def _resolve_pending(self, action: Action, action_func=None) -> Any:
|
|
241
|
+
self._refresh_text_file_source()
|
|
242
|
+
fused = self._client._fuse_stages(self._pending, self._line_meta)
|
|
243
|
+
has_shuffle = any(
|
|
244
|
+
stage.op in (StageOp.REDUCE_BY_KEY, StageOp.GROUP_BY_KEY) for stage in fused
|
|
245
|
+
)
|
|
246
|
+
if (
|
|
247
|
+
action == Action.COUNT
|
|
248
|
+
and not has_shuffle
|
|
249
|
+
and self._client.stages_are_native(fused)
|
|
250
|
+
):
|
|
251
|
+
result = self._client.execute(
|
|
252
|
+
self._rdd_id,
|
|
253
|
+
self._num_partitions,
|
|
254
|
+
self._pending,
|
|
255
|
+
action,
|
|
256
|
+
action_func,
|
|
257
|
+
self._line_meta,
|
|
258
|
+
)
|
|
259
|
+
self._pending = []
|
|
260
|
+
return result
|
|
261
|
+
rdd_id, parts = self._client.materialize(
|
|
262
|
+
self._rdd_id,
|
|
263
|
+
self._num_partitions,
|
|
264
|
+
self._pending,
|
|
265
|
+
self._line_meta,
|
|
266
|
+
)
|
|
267
|
+
self._rdd_id = rdd_id
|
|
268
|
+
self._num_partitions = parts
|
|
269
|
+
self._pending = []
|
|
270
|
+
if action == Action.COUNT:
|
|
271
|
+
return self._client.count(rdd_id)
|
|
272
|
+
if action == Action.COLLECT:
|
|
273
|
+
return self._client.collect(rdd_id)
|
|
274
|
+
if action == Action.REDUCE:
|
|
275
|
+
return self._client.reduce(rdd_id, action_func)
|
|
276
|
+
if action == Action.FOREACH:
|
|
277
|
+
self._client.foreach(rdd_id, action_func)
|
|
278
|
+
return None
|
|
279
|
+
raise RuntimeError(f"unsupported action: {action}")
|
|
280
|
+
|
|
281
|
+
def _run(self, action: Action, action_func=None) -> Any:
|
|
282
|
+
if self._pending:
|
|
283
|
+
return self._resolve_pending(action, action_func)
|
|
284
|
+
self._refresh_text_file_source()
|
|
285
|
+
if action == Action.COUNT:
|
|
286
|
+
return self._client.count(self._rdd_id)
|
|
287
|
+
if action == Action.COLLECT:
|
|
288
|
+
return self._client.collect(self._rdd_id)
|
|
289
|
+
if action == Action.REDUCE:
|
|
290
|
+
return self._client.reduce(self._rdd_id, action_func)
|
|
291
|
+
if action == Action.FOREACH:
|
|
292
|
+
self._client.foreach(self._rdd_id, action_func)
|
|
293
|
+
return None
|
|
294
|
+
raise RuntimeError(f"unsupported action: {action}")
|
|
295
|
+
|
|
296
|
+
def collect(self) -> list:
|
|
297
|
+
return self._run(Action.COLLECT)
|
|
298
|
+
|
|
299
|
+
def count(self) -> int:
|
|
300
|
+
return self._run(Action.COUNT)
|
|
301
|
+
|
|
302
|
+
def reduce(self, func):
|
|
303
|
+
return self._run(Action.REDUCE, func)
|
|
304
|
+
|
|
305
|
+
def foreach(self, func) -> None:
|
|
306
|
+
self._run(Action.FOREACH, func)
|
|
307
|
+
|
|
308
|
+
def first(self):
|
|
309
|
+
values = self.collect()
|
|
310
|
+
if not values:
|
|
311
|
+
raise ValueError("Cannot retrieve first element from an empty RDD")
|
|
312
|
+
return values[0]
|
|
313
|
+
|
|
314
|
+
def take(self, n: int) -> list:
|
|
315
|
+
return self.collect()[:n]
|
|
316
|
+
|
|
317
|
+
def sum(self):
|
|
318
|
+
if self.isEmpty():
|
|
319
|
+
return 0
|
|
320
|
+
return self.reduce(lambda a, b: a + b)
|
|
321
|
+
|
|
322
|
+
def isEmpty(self) -> bool:
|
|
323
|
+
return self.count() == 0
|
|
324
|
+
|
|
325
|
+
def distinct(self, num_partitions: int = 0) -> RDD:
|
|
326
|
+
seen: list[Any] = []
|
|
327
|
+
for item in self.collect():
|
|
328
|
+
if item not in seen:
|
|
329
|
+
seen.append(item)
|
|
330
|
+
parts = num_partitions if num_partitions > 0 else self._num_partitions
|
|
331
|
+
return self._require_context().parallelize(seen, parts)
|
|
332
|
+
|
|
333
|
+
def coalesce(self, num_partitions: int, shuffle: bool = False) -> RDD:
|
|
334
|
+
if shuffle:
|
|
335
|
+
data = self.collect()
|
|
336
|
+
return self._require_context().parallelize(data, num_partitions)
|
|
337
|
+
parts = self._collect_partitions()
|
|
338
|
+
if num_partitions >= len(parts):
|
|
339
|
+
while len(parts) < num_partitions:
|
|
340
|
+
parts.append([])
|
|
341
|
+
return self._from_partitions(parts[:num_partitions])
|
|
342
|
+
merged: list[list[Any]] = [[] for _ in range(num_partitions)]
|
|
343
|
+
for index, part in enumerate(parts):
|
|
344
|
+
merged[index % num_partitions].extend(part)
|
|
345
|
+
return self._from_partitions(merged)
|
|
346
|
+
|
|
347
|
+
def repartition(self, num_partitions: int) -> RDD:
|
|
348
|
+
return self.coalesce(num_partitions, shuffle=True)
|
|
349
|
+
|
|
350
|
+
def sort_by(self, keyfunc, ascending: bool = True, num_partitions: int = 0) -> RDD:
|
|
351
|
+
keyed = self.map(lambda x: (keyfunc(x), x))
|
|
352
|
+
sorted_pairs = keyed.sortByKey(ascending, num_partitions or None)
|
|
353
|
+
return sorted_pairs.map(lambda kv: kv[1])
|
|
354
|
+
|
|
355
|
+
def sortByKey(
|
|
356
|
+
self,
|
|
357
|
+
ascending: bool = True,
|
|
358
|
+
numPartitions: int | None = None,
|
|
359
|
+
keyfunc: Callable[[Any], Any] | None = None,
|
|
360
|
+
) -> RDD:
|
|
361
|
+
key_fn = keyfunc or (lambda x: x)
|
|
362
|
+
num_partitions = numPartitions or self._num_partitions
|
|
363
|
+
if num_partitions == 1:
|
|
364
|
+
data = sorted(self.collect(), key=lambda kv: key_fn(kv[0]), reverse=not ascending)
|
|
365
|
+
return self._require_context().parallelize(data, 1)
|
|
366
|
+
|
|
367
|
+
rdd_size = self.count()
|
|
368
|
+
if rdd_size == 0:
|
|
369
|
+
return self
|
|
370
|
+
|
|
371
|
+
max_sample_size = num_partitions * 20.0
|
|
372
|
+
fraction = min(max_sample_size / max(rdd_size, 1), 1.0)
|
|
373
|
+
samples = sorted(
|
|
374
|
+
(kv[0] for kv in self.sample(False, fraction, 1).collect()),
|
|
375
|
+
key=key_fn,
|
|
376
|
+
)
|
|
377
|
+
bounds = [
|
|
378
|
+
samples[int(len(samples) * (i + 1) / num_partitions)]
|
|
379
|
+
for i in range(num_partitions - 1)
|
|
380
|
+
]
|
|
381
|
+
|
|
382
|
+
def range_partitioner(key: Any) -> int:
|
|
383
|
+
pos = bisect.bisect_left(bounds, key_fn(key))
|
|
384
|
+
if ascending:
|
|
385
|
+
return pos
|
|
386
|
+
return num_partitions - 1 - pos
|
|
387
|
+
|
|
388
|
+
def sort_partition(iterator: Iterable[Any]) -> Iterable[Any]:
|
|
389
|
+
return iter(
|
|
390
|
+
sorted(
|
|
391
|
+
iterator,
|
|
392
|
+
key=lambda kv: key_fn(kv[0]),
|
|
393
|
+
reverse=not ascending,
|
|
394
|
+
)
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
return self.partition_by(num_partitions, range_partitioner).map_partitions(
|
|
398
|
+
sort_partition,
|
|
399
|
+
preserves_partitioning=True,
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
def sample(
|
|
403
|
+
self,
|
|
404
|
+
withReplacement: bool,
|
|
405
|
+
fraction: float,
|
|
406
|
+
seed: int | None = None,
|
|
407
|
+
) -> RDD:
|
|
408
|
+
if fraction < 0:
|
|
409
|
+
raise ValueError("Fraction must be nonnegative.")
|
|
410
|
+
sampler = RDDSampler(withReplacement, fraction, seed)
|
|
411
|
+
return self.map_partitions_with_index(sampler.func, preserves_partitioning=True)
|
|
412
|
+
|
|
413
|
+
def takeSample(
|
|
414
|
+
self, withReplacement: bool, num: int, seed: int | None = None
|
|
415
|
+
) -> list[Any]:
|
|
416
|
+
import random
|
|
417
|
+
|
|
418
|
+
fraction = min(1.0, (num * 10) / max(self.count(), 1))
|
|
419
|
+
sampled = self.sample(withReplacement, fraction, seed).collect()
|
|
420
|
+
rng = random.Random(seed)
|
|
421
|
+
if withReplacement:
|
|
422
|
+
return [rng.choice(sampled) for _ in range(num)] if sampled else []
|
|
423
|
+
return rng.sample(sampled, min(num, len(sampled)))
|
|
424
|
+
|
|
425
|
+
take_sample = takeSample
|
|
426
|
+
|
|
427
|
+
def _pipe_impl(self, command: str, env: dict[str, str], checkCode: bool) -> RDD:
|
|
428
|
+
def func(iterator: Iterable[Any]) -> Iterable[str]:
|
|
429
|
+
try:
|
|
430
|
+
proc = Popen(shlex.split(command), env=env, stdin=PIPE, stdout=PIPE, stderr=PIPE)
|
|
431
|
+
except (FileNotFoundError, OSError) as exc:
|
|
432
|
+
if checkCode:
|
|
433
|
+
raise RuntimeError(str(exc)) from exc
|
|
434
|
+
return
|
|
435
|
+
|
|
436
|
+
def write_input() -> None:
|
|
437
|
+
try:
|
|
438
|
+
assert proc.stdin is not None
|
|
439
|
+
for obj in iterator:
|
|
440
|
+
line = str(obj).rstrip("\n") + "\n"
|
|
441
|
+
proc.stdin.write(line.encode("utf-8"))
|
|
442
|
+
finally:
|
|
443
|
+
if proc.stdin is not None:
|
|
444
|
+
proc.stdin.close()
|
|
445
|
+
|
|
446
|
+
threading.Thread(target=write_input, daemon=True).start()
|
|
447
|
+
assert proc.stdout is not None
|
|
448
|
+
for raw in iter(proc.stdout.readline, b""):
|
|
449
|
+
yield raw.rstrip(b"\n").decode("utf-8")
|
|
450
|
+
proc.wait()
|
|
451
|
+
if checkCode and proc.returncode:
|
|
452
|
+
raise RuntimeError(
|
|
453
|
+
f"Pipe command {command!r} exited with code {proc.returncode}"
|
|
454
|
+
)
|
|
455
|
+
|
|
456
|
+
return self.map_partitions(func)
|
|
457
|
+
|
|
458
|
+
def pipe(self, command: str, env: dict[str, str] | None = None, checkCode: bool = False) -> RDD:
|
|
459
|
+
return _PipeRDD(self, command, dict(env or {}), checkCode, self._require_context())
|
|
460
|
+
|
|
461
|
+
def save_as_text_file(self, path: str, compressionCodecClass: str | None = None) -> None:
|
|
462
|
+
del compressionCodecClass
|
|
463
|
+
save_as_text_file(self._collect_partitions(), path)
|
|
464
|
+
|
|
465
|
+
saveAsTextFile = save_as_text_file
|
|
466
|
+
|
|
467
|
+
def save_as_pickle_file(self, path: str) -> None:
|
|
468
|
+
save_as_pickle_file(self._collect_partitions(), path)
|
|
469
|
+
|
|
470
|
+
saveAsPickleFile = save_as_pickle_file
|
|
471
|
+
|
|
472
|
+
def histogram(self, buckets: int | list[Any] | tuple[Any, ...]):
|
|
473
|
+
data = self.collect()
|
|
474
|
+
if not data:
|
|
475
|
+
if isinstance(buckets, int):
|
|
476
|
+
if buckets < 1:
|
|
477
|
+
raise ValueError("number of buckets must be >= 1")
|
|
478
|
+
raise ValueError("can not generate buckets from empty RDD")
|
|
479
|
+
if isinstance(buckets, (list, tuple)):
|
|
480
|
+
if len(buckets) < 2:
|
|
481
|
+
raise ValueError("buckets should have more than one value")
|
|
482
|
+
return list(buckets), [0] * (len(buckets) - 1)
|
|
483
|
+
if isinstance(buckets, int):
|
|
484
|
+
return histogram_values(data, buckets)
|
|
485
|
+
|
|
486
|
+
def partition_hist(iterator: Iterable[Any]) -> Iterable[list[int]]:
|
|
487
|
+
yield histogram_partition(iterator, buckets)
|
|
488
|
+
|
|
489
|
+
def merge(a: list[int], b: list[int]) -> list[int]:
|
|
490
|
+
return [x + y for x, y in zip(a, b)]
|
|
491
|
+
|
|
492
|
+
return buckets, self.map_partitions(partition_hist).reduce(merge)
|
|
493
|
+
|
|
494
|
+
def countApproxDistinct(self, relativeSD: float = 0.05) -> int:
|
|
495
|
+
return count_approx_distinct(self.collect(), relativeSD)
|
|
496
|
+
|
|
497
|
+
count_approx_distinct = countApproxDistinct
|
|
498
|
+
|
|
499
|
+
def treeAggregate(
|
|
500
|
+
self,
|
|
501
|
+
zeroValue,
|
|
502
|
+
seqOp,
|
|
503
|
+
combOp,
|
|
504
|
+
depth: int = 2,
|
|
505
|
+
):
|
|
506
|
+
if depth < 1:
|
|
507
|
+
raise ValueError(f"Depth cannot be smaller than 1 but got {depth}.")
|
|
508
|
+
if self.getNumPartitions() == 0:
|
|
509
|
+
return zeroValue() if callable(zeroValue) else copy.deepcopy(zeroValue)
|
|
510
|
+
|
|
511
|
+
def aggregate_partition(iterator: Iterable[Any]) -> Iterable[Any]:
|
|
512
|
+
acc = zeroValue() if callable(zeroValue) else copy.deepcopy(zeroValue)
|
|
513
|
+
for obj in iterator:
|
|
514
|
+
acc = seqOp(acc, obj)
|
|
515
|
+
yield acc
|
|
516
|
+
|
|
517
|
+
partially = self.map_partitions(aggregate_partition, preserves_partitioning=True)
|
|
518
|
+
num_partitions = partially.getNumPartitions()
|
|
519
|
+
scale = max(int(math.ceil(pow(num_partitions, 1.0 / depth))), 2)
|
|
520
|
+
while num_partitions > scale + num_partitions / scale:
|
|
521
|
+
num_partitions = num_partitions / scale
|
|
522
|
+
cur = int(num_partitions)
|
|
523
|
+
|
|
524
|
+
def map_partition(index: int, iterator: Iterable[Any]) -> Iterable[tuple[int, Any]]:
|
|
525
|
+
for obj in iterator:
|
|
526
|
+
yield (index % cur, obj)
|
|
527
|
+
|
|
528
|
+
partially = (
|
|
529
|
+
partially.map_partitions_with_index(map_partition, preserves_partitioning=False)
|
|
530
|
+
.reduce_by_key(combOp, cur)
|
|
531
|
+
.map(lambda kv: kv[1])
|
|
532
|
+
)
|
|
533
|
+
num_partitions = partially.getNumPartitions()
|
|
534
|
+
return partially.reduce(combOp)
|
|
535
|
+
|
|
536
|
+
tree_aggregate = treeAggregate
|
|
537
|
+
|
|
538
|
+
def repartition_and_sort_within_partitions(
|
|
539
|
+
self,
|
|
540
|
+
num_partitions: int,
|
|
541
|
+
partition_func,
|
|
542
|
+
ascending: bool = True,
|
|
543
|
+
keyfunc: Callable[[Any], Any] | None = None,
|
|
544
|
+
) -> RDD:
|
|
545
|
+
key_fn = keyfunc or (lambda x: x)
|
|
546
|
+
|
|
547
|
+
def sort_partition(iterator: Iterable[Any]) -> Iterable[Any]:
|
|
548
|
+
return iter(
|
|
549
|
+
sorted(
|
|
550
|
+
iterator,
|
|
551
|
+
key=lambda kv: key_fn(kv[0]),
|
|
552
|
+
reverse=not ascending,
|
|
553
|
+
)
|
|
554
|
+
)
|
|
555
|
+
|
|
556
|
+
return self.partition_by(num_partitions, partition_func).map_partitions(
|
|
557
|
+
sort_partition,
|
|
558
|
+
preserves_partitioning=True,
|
|
559
|
+
)
|
|
560
|
+
|
|
561
|
+
repartitionAndSortWithinPartitions = repartition_and_sort_within_partitions
|
|
562
|
+
|
|
563
|
+
def to_local_iterator(self, prefetch_partitions: bool = False) -> Iterator[Any]:
|
|
564
|
+
self._materialize()
|
|
565
|
+
total = self._num_partitions
|
|
566
|
+
cache: dict[int, list[Any]] = {}
|
|
567
|
+
lock = threading.Lock()
|
|
568
|
+
|
|
569
|
+
def prefetch(pid: int) -> None:
|
|
570
|
+
if pid >= total:
|
|
571
|
+
return
|
|
572
|
+
data = self._collect_partition(pid)
|
|
573
|
+
with lock:
|
|
574
|
+
cache[pid] = data
|
|
575
|
+
|
|
576
|
+
for pid in range(total):
|
|
577
|
+
if prefetch_partitions and pid + 1 < total:
|
|
578
|
+
threading.Thread(target=prefetch, args=(pid + 1,), daemon=True).start()
|
|
579
|
+
with lock:
|
|
580
|
+
part = cache.pop(pid, None)
|
|
581
|
+
if part is None:
|
|
582
|
+
part = self._collect_partition(pid)
|
|
583
|
+
yield from part
|
|
584
|
+
|
|
585
|
+
toLocalIterator = to_local_iterator
|
|
586
|
+
|
|
587
|
+
def cartesian(self, other: RDD) -> RDD:
|
|
588
|
+
left = self.collect()
|
|
589
|
+
right = other.collect()
|
|
590
|
+
return self._require_context().parallelize(
|
|
591
|
+
[(a, b) for a in left for b in right],
|
|
592
|
+
self._num_partitions,
|
|
593
|
+
)
|
|
594
|
+
|
|
595
|
+
def zip(self, other: RDD) -> RDD:
|
|
596
|
+
if self._num_partitions != other._num_partitions:
|
|
597
|
+
raise ValueError(
|
|
598
|
+
"Cannot zip RDDs with different number of partitions: "
|
|
599
|
+
f"{self._num_partitions} != {other._num_partitions}"
|
|
600
|
+
)
|
|
601
|
+
return _ZipRDD(self, other, self._require_context())
|
|
602
|
+
|
|
603
|
+
def aggregate(self, zeroValue, seqOp, combOp):
|
|
604
|
+
def apply_seq(acc, value):
|
|
605
|
+
try:
|
|
606
|
+
return seqOp(acc, value)
|
|
607
|
+
except StopIteration as exc:
|
|
608
|
+
raise RuntimeError(
|
|
609
|
+
"Caught StopIteration thrown from user's code; failing the task"
|
|
610
|
+
) from exc
|
|
611
|
+
|
|
612
|
+
def apply_comb(a, b):
|
|
613
|
+
try:
|
|
614
|
+
return combOp(a, b)
|
|
615
|
+
except StopIteration as exc:
|
|
616
|
+
raise RuntimeError(
|
|
617
|
+
"Caught StopIteration thrown from user's code; failing the task"
|
|
618
|
+
) from exc
|
|
619
|
+
|
|
620
|
+
partials: list[Any] = []
|
|
621
|
+
for part in self._collect_partitions():
|
|
622
|
+
acc = zeroValue() if callable(zeroValue) else copy.deepcopy(zeroValue)
|
|
623
|
+
for value in part:
|
|
624
|
+
acc = apply_seq(acc, value)
|
|
625
|
+
partials.append(acc)
|
|
626
|
+
if not partials:
|
|
627
|
+
return zeroValue() if callable(zeroValue) else copy.deepcopy(zeroValue)
|
|
628
|
+
acc = partials[0]
|
|
629
|
+
for partial in partials[1:]:
|
|
630
|
+
acc = apply_comb(acc, partial)
|
|
631
|
+
return acc
|
|
632
|
+
|
|
633
|
+
def aggregate_by_key(self, zeroValue, seqOp, combOp, num_partitions: int = 0):
|
|
634
|
+
parts = num_partitions or self._num_partitions
|
|
635
|
+
|
|
636
|
+
def merge_values(kv):
|
|
637
|
+
key, values = kv
|
|
638
|
+
acc = zeroValue() if callable(zeroValue) else copy.deepcopy(zeroValue)
|
|
639
|
+
for value in values:
|
|
640
|
+
acc = seqOp(acc, value)
|
|
641
|
+
return (key, acc)
|
|
642
|
+
|
|
643
|
+
return self.group_by_key(parts).map(merge_values)
|
|
644
|
+
|
|
645
|
+
aggregateByKey = aggregate_by_key
|
|
646
|
+
|
|
647
|
+
def fold(self, zeroValue, op):
|
|
648
|
+
return self.aggregate(zeroValue, op, op)
|
|
649
|
+
|
|
650
|
+
def fold_by_key(self, zeroValue, op, num_partitions: int = 0):
|
|
651
|
+
return self.aggregate_by_key(zeroValue, op, op, num_partitions)
|
|
652
|
+
|
|
653
|
+
foldByKey = fold_by_key
|
|
654
|
+
|
|
655
|
+
def cache(self) -> RDD:
|
|
656
|
+
return self
|
|
657
|
+
|
|
658
|
+
def id(self) -> int:
|
|
659
|
+
return self._logical_id
|
|
660
|
+
|
|
661
|
+
def _require_context(self) -> "SparkContext":
|
|
662
|
+
if self._context is None:
|
|
663
|
+
raise RuntimeError("RDD has no SparkContext for client-side operation")
|
|
664
|
+
return self._context
|
|
665
|
+
|
|
666
|
+
@property
|
|
667
|
+
def num_partitions(self) -> int:
|
|
668
|
+
return self._num_partitions
|
|
669
|
+
|
|
670
|
+
def getNumPartitions(self) -> int:
|
|
671
|
+
return self._num_partitions
|
|
672
|
+
|
|
673
|
+
def __repr__(self) -> str:
|
|
674
|
+
pending = len(self._pending)
|
|
675
|
+
suffix = f", pending_ops={pending}" if pending else ""
|
|
676
|
+
return f"RDD(num_partitions={self.num_partitions}{suffix})"
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
class _PipeRDD(RDD):
|
|
680
|
+
def __init__(
|
|
681
|
+
self,
|
|
682
|
+
parent: RDD,
|
|
683
|
+
command: str,
|
|
684
|
+
env: dict[str, str],
|
|
685
|
+
checkCode: bool,
|
|
686
|
+
context: SparkContext,
|
|
687
|
+
):
|
|
688
|
+
self._pipe_parent = parent
|
|
689
|
+
self._pipe_command = command
|
|
690
|
+
self._pipe_env = env
|
|
691
|
+
self._pipe_check_code = checkCode
|
|
692
|
+
super().__init__(
|
|
693
|
+
parent._client,
|
|
694
|
+
parent._rdd_id,
|
|
695
|
+
parent._num_partitions,
|
|
696
|
+
pending=list(parent._pending),
|
|
697
|
+
native=parent._native,
|
|
698
|
+
native_kind=parent._native_kind,
|
|
699
|
+
line_meta=parent._line_meta,
|
|
700
|
+
context=context,
|
|
701
|
+
text_file_path=parent._text_file_path,
|
|
702
|
+
text_file_parts=parent._text_file_parts,
|
|
703
|
+
)
|
|
704
|
+
|
|
705
|
+
def _materialize_pipe(self) -> None:
|
|
706
|
+
if self._pipe_parent is None:
|
|
707
|
+
return
|
|
708
|
+
piped = self._pipe_parent._pipe_impl(
|
|
709
|
+
self._pipe_command, self._pipe_env, self._pipe_check_code
|
|
710
|
+
)
|
|
711
|
+
self._rdd_id = piped._rdd_id
|
|
712
|
+
self._num_partitions = piped._num_partitions
|
|
713
|
+
self._pending = list(piped._pending)
|
|
714
|
+
self._native = piped._native
|
|
715
|
+
self._native_kind = piped._native_kind
|
|
716
|
+
self._pipe_parent = None
|
|
717
|
+
|
|
718
|
+
def _resolve_pending(self, action: Action, action_func=None) -> Any:
|
|
719
|
+
self._materialize_pipe()
|
|
720
|
+
return super()._resolve_pending(action, action_func)
|
|
721
|
+
|
|
722
|
+
def _run(self, action: Action, action_func=None) -> Any:
|
|
723
|
+
if self._pipe_parent is not None:
|
|
724
|
+
self._materialize_pipe()
|
|
725
|
+
return super()._run(action, action_func)
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
class _ZipRDD(RDD):
|
|
729
|
+
def __init__(self, left: RDD, right: RDD, context: SparkContext):
|
|
730
|
+
self._zip_left = left
|
|
731
|
+
self._zip_right = right
|
|
732
|
+
super().__init__(
|
|
733
|
+
left._client,
|
|
734
|
+
left._rdd_id,
|
|
735
|
+
left._num_partitions,
|
|
736
|
+
pending=list(left._pending),
|
|
737
|
+
native=left._native,
|
|
738
|
+
native_kind=left._native_kind,
|
|
739
|
+
line_meta=left._line_meta,
|
|
740
|
+
context=context,
|
|
741
|
+
text_file_path=left._text_file_path,
|
|
742
|
+
text_file_parts=left._text_file_parts,
|
|
743
|
+
)
|
|
744
|
+
|
|
745
|
+
def _materialize_zipped(self) -> None:
|
|
746
|
+
left_parts = self._zip_left._collect_partitions()
|
|
747
|
+
right_parts = self._zip_right._collect_partitions()
|
|
748
|
+
if len(left_parts) != len(right_parts):
|
|
749
|
+
raise RuntimeError("Cannot zip RDDs with different number of partitions")
|
|
750
|
+
out: list[list[Any]] = []
|
|
751
|
+
for left, right in zip(left_parts, right_parts):
|
|
752
|
+
if len(left) != len(right):
|
|
753
|
+
raise ValueError(
|
|
754
|
+
"Cannot zip RDDs with different number of elements in partition"
|
|
755
|
+
)
|
|
756
|
+
out.append(list(zip(left, right)))
|
|
757
|
+
rdd_id, parts = self._client.parallelize_partitions(out)
|
|
758
|
+
self._rdd_id = rdd_id
|
|
759
|
+
self._num_partitions = parts
|
|
760
|
+
self._pending = []
|
|
761
|
+
self._native = False
|
|
762
|
+
self._zip_left = None
|
|
763
|
+
self._zip_right = None
|
|
764
|
+
|
|
765
|
+
def _collect_partitions(self) -> list[list[Any]]:
|
|
766
|
+
if self._zip_left is not None:
|
|
767
|
+
self._materialize_zipped()
|
|
768
|
+
return super()._collect_partitions()
|
|
769
|
+
|
|
770
|
+
def _resolve_pending(self, action: Action, action_func=None) -> Any:
|
|
771
|
+
self._materialize_zipped()
|
|
772
|
+
return super()._resolve_pending(action, action_func)
|
|
773
|
+
|
|
774
|
+
def _run(self, action: Action, action_func=None) -> Any:
|
|
775
|
+
if self._zip_left is not None:
|
|
776
|
+
self._materialize_zipped()
|
|
777
|
+
return super()._run(action, action_func)
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
class SparkContext:
|
|
781
|
+
"""Entry point for PyVega. Talks to a Vega C++ session server."""
|
|
782
|
+
|
|
783
|
+
def __init__(self, num_workers: int = 0, remote: Optional[str] = None):
|
|
784
|
+
self._server_proc = None
|
|
785
|
+
self._num_workers = num_workers
|
|
786
|
+
self._pickled_broadcast_vars = BroadcastPickleRegistry()
|
|
787
|
+
|
|
788
|
+
if remote:
|
|
789
|
+
host, port = parse_remote(remote)
|
|
790
|
+
else:
|
|
791
|
+
self._server_proc, host, port = spawn_server(num_workers)
|
|
792
|
+
|
|
793
|
+
self._client = SessionClient(host, port, num_workers)
|
|
794
|
+
self._num_workers = self._client.num_workers
|
|
795
|
+
|
|
796
|
+
def range(self, start: int, end: int, num_partitions: int = 0) -> RDD:
|
|
797
|
+
"""Create an RDD of integers [start, end) generated on the server."""
|
|
798
|
+
if num_partitions == 0:
|
|
799
|
+
num_partitions = self._num_workers
|
|
800
|
+
rdd_id, parts = self._client.parallelize_range(start, end, num_partitions)
|
|
801
|
+
return RDD(
|
|
802
|
+
self._client, rdd_id, parts, native=True, native_kind="int64", context=self
|
|
803
|
+
)
|
|
804
|
+
|
|
805
|
+
def float_range(self, start: int, end: int, num_partitions: int = 0) -> RDD:
|
|
806
|
+
"""Create a native float64 RDD [start, end) generated on the server."""
|
|
807
|
+
if num_partitions == 0:
|
|
808
|
+
num_partitions = self._num_workers
|
|
809
|
+
rdd_id, parts = self._client.parallelize_float_range(start, end, num_partitions)
|
|
810
|
+
return RDD(
|
|
811
|
+
self._client, rdd_id, parts, native=True, native_kind="float64", context=self
|
|
812
|
+
)
|
|
813
|
+
|
|
814
|
+
def string_lines(
|
|
815
|
+
self,
|
|
816
|
+
start: int,
|
|
817
|
+
end: int,
|
|
818
|
+
num_keys: int,
|
|
819
|
+
num_tokens: int = 97,
|
|
820
|
+
num_partitions: int = 0,
|
|
821
|
+
) -> RDD:
|
|
822
|
+
"""Create a native string RDD of synthetic lines generated on the server."""
|
|
823
|
+
if num_partitions == 0:
|
|
824
|
+
num_partitions = self._num_workers
|
|
825
|
+
rdd_id, parts = self._client.parallelize_string_lines(
|
|
826
|
+
start, end, num_keys, num_tokens, num_partitions
|
|
827
|
+
)
|
|
828
|
+
return RDD(
|
|
829
|
+
self._client,
|
|
830
|
+
rdd_id,
|
|
831
|
+
parts,
|
|
832
|
+
native=True,
|
|
833
|
+
native_kind="str",
|
|
834
|
+
line_meta=(num_keys, num_tokens),
|
|
835
|
+
context=self,
|
|
836
|
+
)
|
|
837
|
+
|
|
838
|
+
@staticmethod
|
|
839
|
+
def _detect_native_kind(data_list: list) -> tuple[bool, NativeKind]:
|
|
840
|
+
if not data_list:
|
|
841
|
+
return False, "int64"
|
|
842
|
+
if all(isinstance(x, int) and not isinstance(x, bool) for x in data_list):
|
|
843
|
+
return True, "int64"
|
|
844
|
+
if all(
|
|
845
|
+
isinstance(x, (int, float)) and not isinstance(x, bool) for x in data_list
|
|
846
|
+
):
|
|
847
|
+
return True, "float64"
|
|
848
|
+
if all(isinstance(x, str) for x in data_list):
|
|
849
|
+
return True, "str"
|
|
850
|
+
return False, "int64"
|
|
851
|
+
|
|
852
|
+
def parallelize(self, data, num_partitions: int = 0) -> RDD:
|
|
853
|
+
if num_partitions == 0:
|
|
854
|
+
num_partitions = self._num_workers
|
|
855
|
+
data_list = list(data)
|
|
856
|
+
native, native_kind = self._detect_native_kind(data_list)
|
|
857
|
+
rdd_id, parts = self._client.parallelize(data_list, num_partitions)
|
|
858
|
+
return RDD(
|
|
859
|
+
self._client, rdd_id, parts, native=native, native_kind=native_kind, context=self
|
|
860
|
+
)
|
|
861
|
+
|
|
862
|
+
def empty_rdd(self) -> RDD:
|
|
863
|
+
return self.parallelize([], 1)
|
|
864
|
+
|
|
865
|
+
emptyRDD = empty_rdd
|
|
866
|
+
|
|
867
|
+
def text_file(self, path: str, min_partitions: int = 0) -> RDD:
|
|
868
|
+
parts = min_partitions or self._num_workers
|
|
869
|
+
if os.path.isdir(path):
|
|
870
|
+
return self.parallelize(read_text_file(path), parts)
|
|
871
|
+
return RDD(
|
|
872
|
+
self._client,
|
|
873
|
+
0,
|
|
874
|
+
parts,
|
|
875
|
+
context=self,
|
|
876
|
+
text_file_path=path,
|
|
877
|
+
text_file_parts=parts,
|
|
878
|
+
)
|
|
879
|
+
|
|
880
|
+
textFile = text_file
|
|
881
|
+
|
|
882
|
+
def pickle_file(self, path: str, min_partitions: int = 0) -> RDD:
|
|
883
|
+
data = read_pickle_file(path)
|
|
884
|
+
parts = min_partitions or self._num_workers
|
|
885
|
+
return self.parallelize(data, parts)
|
|
886
|
+
|
|
887
|
+
pickleFile = pickle_file
|
|
888
|
+
|
|
889
|
+
def broadcast(self, value: Any) -> Broadcast[Any]:
|
|
890
|
+
return Broadcast(self, value, self._pickled_broadcast_vars)
|
|
891
|
+
|
|
892
|
+
def union(self, rdds: list[RDD]) -> RDD:
|
|
893
|
+
data: list[Any] = []
|
|
894
|
+
for rdd in rdds:
|
|
895
|
+
data.extend(rdd.collect())
|
|
896
|
+
return self.parallelize(data, max(1, len(rdds)))
|
|
897
|
+
|
|
898
|
+
@property
|
|
899
|
+
def num_workers(self) -> int:
|
|
900
|
+
return self._num_workers
|
|
901
|
+
|
|
902
|
+
def stop(self) -> None:
|
|
903
|
+
self._client.close()
|
|
904
|
+
if self._server_proc is not None:
|
|
905
|
+
self._server_proc.terminate()
|
|
906
|
+
try:
|
|
907
|
+
self._server_proc.wait(timeout=5)
|
|
908
|
+
except Exception:
|
|
909
|
+
self._server_proc.kill()
|
|
910
|
+
self._server_proc = None
|
|
911
|
+
|
|
912
|
+
def __enter__(self):
|
|
913
|
+
return self
|
|
914
|
+
|
|
915
|
+
def __exit__(self, *exc):
|
|
916
|
+
self.stop()
|
|
917
|
+
|
|
918
|
+
def __repr__(self) -> str:
|
|
919
|
+
return f"SparkContext(num_workers={self.num_workers})"
|