pyvega 0.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pyvega/__init__.py ADDED
@@ -0,0 +1,919 @@
1
+ """
2
+ PyVega -- Python client for the Vega C++ engine.
3
+
4
+ Connects to a Vega session server over TCP. Computation and the actor
5
+ system run in the C++ server process; this package is a thin RPC client.
6
+
7
+ from pyvega import SparkContext
8
+
9
+ with SparkContext(4) as sc:
10
+ result = sc.range(0, 100).map(lambda x: x * 2).collect()
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import bisect
16
+ import copy
17
+ import math
18
+ import os
19
+ import shlex
20
+ import subprocess
21
+ import threading
22
+ from subprocess import PIPE, Popen
23
+ from typing import Any, Callable, Iterable, Iterator, Optional
24
+
25
+ from pyvega._client import PendingStage, SessionClient, parse_remote, spawn_server
26
+ from pyvega._io import (
27
+ read_pickle_file,
28
+ read_text_file,
29
+ save_as_pickle_file,
30
+ save_as_text_file,
31
+ )
32
+ from pyvega._native import NativeKind, try_compile
33
+ from pyvega._protocol import Action, StageOp
34
+ from pyvega._sampler import RDDSampler
35
+ from pyvega._stats import count_approx_distinct, histogram_partition, histogram_values
36
+ from pyvega.broadcast import Broadcast, BroadcastPickleRegistry
37
+
38
+ __version__ = "0.5.0"
39
+ __all__ = ["SparkContext", "RDD"]
40
+
41
+ _next_rdd_id = 1
42
+
43
+
44
+ class RDD:
45
+ """Remote RDD handle backed by the Vega C++ server.
46
+
47
+ Transformations are recorded client-side and sent as one pipeline when an
48
+ action runs, avoiding per-transform RPC round-trips.
49
+ """
50
+
51
+ def __init__(
52
+ self,
53
+ client: SessionClient,
54
+ rdd_id: int,
55
+ num_partitions: int,
56
+ pending: Optional[list[PendingStage]] = None,
57
+ native: bool = False,
58
+ native_kind: NativeKind = "int64",
59
+ line_meta: Optional[tuple[int, int]] = None,
60
+ context: Optional["SparkContext"] = None,
61
+ rdd_id_counter: Optional[int] = None,
62
+ text_file_path: Optional[str] = None,
63
+ text_file_parts: int = 0,
64
+ ):
65
+ global _next_rdd_id
66
+ self._client = client
67
+ self._rdd_id = rdd_id
68
+ self._num_partitions = num_partitions
69
+ self._pending = list(pending or [])
70
+ self._native = native
71
+ self._native_kind: NativeKind = native_kind
72
+ self._line_meta = line_meta
73
+ self._context = context
74
+ self._text_file_path = text_file_path
75
+ self._text_file_parts = text_file_parts
76
+ if rdd_id_counter is None:
77
+ rdd_id_counter = _next_rdd_id
78
+ _next_rdd_id += 1
79
+ self._logical_id = rdd_id_counter
80
+
81
+ def _derive(self, stage: PendingStage, native: bool, native_kind: NativeKind) -> RDD:
82
+ new_id = self._logical_id + (0 if self._pending else 1)
83
+ return RDD(
84
+ self._client,
85
+ self._rdd_id,
86
+ self._num_partitions,
87
+ self._pending + [stage],
88
+ native=native,
89
+ native_kind=native_kind,
90
+ line_meta=self._line_meta,
91
+ context=self._context,
92
+ rdd_id_counter=new_id,
93
+ text_file_path=self._text_file_path,
94
+ text_file_parts=self._text_file_parts,
95
+ )
96
+
97
+ def _refresh_text_file_source(self) -> None:
98
+ if not self._text_file_path:
99
+ return
100
+ with open(self._text_file_path, encoding="utf-8") as handle:
101
+ lines = [line.rstrip("\n") for line in handle]
102
+ parts = self._text_file_parts or self._num_partitions or 1
103
+ native, native_kind = SparkContext._detect_native_kind(lines)
104
+ self._rdd_id, self._num_partitions = self._client.parallelize(lines, parts)
105
+ self._native = native
106
+ self._native_kind = native_kind
107
+
108
+ def _stage(self, op: StageOp, func, num_partitions: int = 0) -> RDD:
109
+ native = self._native
110
+ native_kind = self._native_kind
111
+ native_op = None
112
+ native_params: tuple[int | float | str, ...] = ()
113
+ if native and op != StageOp.GROUP_BY_KEY:
114
+ compiled = try_compile(op, func, native_kind)
115
+ if compiled is not None:
116
+ native_op = compiled.native_op
117
+ native_params = compiled.params
118
+ else:
119
+ native = False
120
+ return self._derive(
121
+ PendingStage(op, func, num_partitions, native_op, native_params),
122
+ native,
123
+ native_kind,
124
+ )
125
+
126
+ @staticmethod
127
+ def _normalize_func(func: Callable[..., Any]) -> Callable[..., Any]:
128
+ if func is range:
129
+ return lambda x: list(range(x))
130
+ if isinstance(func, type) or func in (str, int, float, bool, list, tuple, set, dict):
131
+ return lambda x, f=func: f(x)
132
+ return func
133
+
134
+ def _materialize(self) -> None:
135
+ self._refresh_text_file_source()
136
+ if not self._pending:
137
+ return
138
+ rdd_id, parts = self._client.materialize(
139
+ self._rdd_id,
140
+ self._num_partitions,
141
+ self._pending,
142
+ self._line_meta,
143
+ )
144
+ self._rdd_id = rdd_id
145
+ self._num_partitions = parts
146
+ self._pending = []
147
+
148
+ def _collect_partitions(self) -> list[list[Any]]:
149
+ self._materialize()
150
+ return self._client.collect_partitions(self._rdd_id)
151
+
152
+ def _collect_partition(self, partition: int) -> list[Any]:
153
+ self._materialize()
154
+ return self._client.collect_partition(self._rdd_id, partition)
155
+
156
+ def _from_partitions(
157
+ self,
158
+ partitions: list[list[Any]],
159
+ native: bool = False,
160
+ native_kind: NativeKind = "int64",
161
+ ) -> RDD:
162
+ ctx = self._require_context()
163
+ rdd_id, parts = self._client.parallelize_partitions(partitions)
164
+ return RDD(
165
+ self._client,
166
+ rdd_id,
167
+ parts,
168
+ native=native,
169
+ native_kind=native_kind,
170
+ context=ctx,
171
+ )
172
+
173
+ def map_partitions(self, func, preserves_partitioning: bool = False) -> RDD:
174
+ parts = self._collect_partitions()
175
+ mapped = [list(func(iter(part))) for part in parts]
176
+ if preserves_partitioning:
177
+ return self._from_partitions(mapped)
178
+ flat = [item for part in mapped for item in part]
179
+ return self._require_context().parallelize(flat, self._num_partitions)
180
+
181
+ mapPartitions = map_partitions
182
+
183
+ def map_partitions_with_index(
184
+ self, func, preserves_partitioning: bool = False
185
+ ) -> RDD:
186
+ parts = self._collect_partitions()
187
+ mapped = [list(func(index, iter(part))) for index, part in enumerate(parts)]
188
+ if preserves_partitioning:
189
+ return self._from_partitions(mapped)
190
+ flat = [item for part in mapped for item in part]
191
+ return self._require_context().parallelize(flat, self._num_partitions)
192
+
193
+ mapPartitionsWithIndex = map_partitions_with_index
194
+
195
+ def glom(self) -> RDD:
196
+ def func(iterator: Iterable[Any]) -> Iterable[list[Any]]:
197
+ yield list(iterator)
198
+
199
+ return self.map_partitions(func, preserves_partitioning=True)
200
+
201
+ def partition_by(self, num_partitions: int, partition_func) -> RDD:
202
+ buckets: list[list[Any]] = [[] for _ in range(num_partitions)]
203
+ for part in self._collect_partitions():
204
+ for item in part:
205
+ if isinstance(item, tuple) and len(item) >= 1:
206
+ key = item[0]
207
+ else:
208
+ key = item
209
+ bucket = int(partition_func(key)) % num_partitions
210
+ buckets[bucket].append(item)
211
+ return self._from_partitions(buckets)
212
+
213
+ partitionBy = partition_by
214
+
215
+ def map(self, func) -> RDD:
216
+ return self._stage(StageOp.MAP, self._normalize_func(func))
217
+
218
+ def filter(self, func) -> RDD:
219
+ return self._stage(StageOp.FILTER, func)
220
+
221
+ def flat_map(self, func) -> RDD:
222
+ return self._stage(StageOp.FLAT_MAP, self._normalize_func(func))
223
+
224
+ flatMap = flat_map
225
+
226
+ def reduce_by_key(self, func, num_partitions: int = 0) -> RDD:
227
+ if num_partitions == 0:
228
+ num_partitions = self._num_partitions
229
+ return self._stage(StageOp.REDUCE_BY_KEY, func, num_partitions)
230
+
231
+ reduceByKey = reduce_by_key
232
+
233
+ def group_by_key(self, num_partitions: int = 0) -> RDD:
234
+ if num_partitions == 0:
235
+ num_partitions = self._num_partitions
236
+ return self._stage(StageOp.GROUP_BY_KEY, None, num_partitions)
237
+
238
+ groupByKey = group_by_key
239
+
240
+ def _resolve_pending(self, action: Action, action_func=None) -> Any:
241
+ self._refresh_text_file_source()
242
+ fused = self._client._fuse_stages(self._pending, self._line_meta)
243
+ has_shuffle = any(
244
+ stage.op in (StageOp.REDUCE_BY_KEY, StageOp.GROUP_BY_KEY) for stage in fused
245
+ )
246
+ if (
247
+ action == Action.COUNT
248
+ and not has_shuffle
249
+ and self._client.stages_are_native(fused)
250
+ ):
251
+ result = self._client.execute(
252
+ self._rdd_id,
253
+ self._num_partitions,
254
+ self._pending,
255
+ action,
256
+ action_func,
257
+ self._line_meta,
258
+ )
259
+ self._pending = []
260
+ return result
261
+ rdd_id, parts = self._client.materialize(
262
+ self._rdd_id,
263
+ self._num_partitions,
264
+ self._pending,
265
+ self._line_meta,
266
+ )
267
+ self._rdd_id = rdd_id
268
+ self._num_partitions = parts
269
+ self._pending = []
270
+ if action == Action.COUNT:
271
+ return self._client.count(rdd_id)
272
+ if action == Action.COLLECT:
273
+ return self._client.collect(rdd_id)
274
+ if action == Action.REDUCE:
275
+ return self._client.reduce(rdd_id, action_func)
276
+ if action == Action.FOREACH:
277
+ self._client.foreach(rdd_id, action_func)
278
+ return None
279
+ raise RuntimeError(f"unsupported action: {action}")
280
+
281
+ def _run(self, action: Action, action_func=None) -> Any:
282
+ if self._pending:
283
+ return self._resolve_pending(action, action_func)
284
+ self._refresh_text_file_source()
285
+ if action == Action.COUNT:
286
+ return self._client.count(self._rdd_id)
287
+ if action == Action.COLLECT:
288
+ return self._client.collect(self._rdd_id)
289
+ if action == Action.REDUCE:
290
+ return self._client.reduce(self._rdd_id, action_func)
291
+ if action == Action.FOREACH:
292
+ self._client.foreach(self._rdd_id, action_func)
293
+ return None
294
+ raise RuntimeError(f"unsupported action: {action}")
295
+
296
+ def collect(self) -> list:
297
+ return self._run(Action.COLLECT)
298
+
299
+ def count(self) -> int:
300
+ return self._run(Action.COUNT)
301
+
302
+ def reduce(self, func):
303
+ return self._run(Action.REDUCE, func)
304
+
305
+ def foreach(self, func) -> None:
306
+ self._run(Action.FOREACH, func)
307
+
308
+ def first(self):
309
+ values = self.collect()
310
+ if not values:
311
+ raise ValueError("Cannot retrieve first element from an empty RDD")
312
+ return values[0]
313
+
314
+ def take(self, n: int) -> list:
315
+ return self.collect()[:n]
316
+
317
+ def sum(self):
318
+ if self.isEmpty():
319
+ return 0
320
+ return self.reduce(lambda a, b: a + b)
321
+
322
+ def isEmpty(self) -> bool:
323
+ return self.count() == 0
324
+
325
+ def distinct(self, num_partitions: int = 0) -> RDD:
326
+ seen: list[Any] = []
327
+ for item in self.collect():
328
+ if item not in seen:
329
+ seen.append(item)
330
+ parts = num_partitions if num_partitions > 0 else self._num_partitions
331
+ return self._require_context().parallelize(seen, parts)
332
+
333
+ def coalesce(self, num_partitions: int, shuffle: bool = False) -> RDD:
334
+ if shuffle:
335
+ data = self.collect()
336
+ return self._require_context().parallelize(data, num_partitions)
337
+ parts = self._collect_partitions()
338
+ if num_partitions >= len(parts):
339
+ while len(parts) < num_partitions:
340
+ parts.append([])
341
+ return self._from_partitions(parts[:num_partitions])
342
+ merged: list[list[Any]] = [[] for _ in range(num_partitions)]
343
+ for index, part in enumerate(parts):
344
+ merged[index % num_partitions].extend(part)
345
+ return self._from_partitions(merged)
346
+
347
+ def repartition(self, num_partitions: int) -> RDD:
348
+ return self.coalesce(num_partitions, shuffle=True)
349
+
350
+ def sort_by(self, keyfunc, ascending: bool = True, num_partitions: int = 0) -> RDD:
351
+ keyed = self.map(lambda x: (keyfunc(x), x))
352
+ sorted_pairs = keyed.sortByKey(ascending, num_partitions or None)
353
+ return sorted_pairs.map(lambda kv: kv[1])
354
+
355
+ def sortByKey(
356
+ self,
357
+ ascending: bool = True,
358
+ numPartitions: int | None = None,
359
+ keyfunc: Callable[[Any], Any] | None = None,
360
+ ) -> RDD:
361
+ key_fn = keyfunc or (lambda x: x)
362
+ num_partitions = numPartitions or self._num_partitions
363
+ if num_partitions == 1:
364
+ data = sorted(self.collect(), key=lambda kv: key_fn(kv[0]), reverse=not ascending)
365
+ return self._require_context().parallelize(data, 1)
366
+
367
+ rdd_size = self.count()
368
+ if rdd_size == 0:
369
+ return self
370
+
371
+ max_sample_size = num_partitions * 20.0
372
+ fraction = min(max_sample_size / max(rdd_size, 1), 1.0)
373
+ samples = sorted(
374
+ (kv[0] for kv in self.sample(False, fraction, 1).collect()),
375
+ key=key_fn,
376
+ )
377
+ bounds = [
378
+ samples[int(len(samples) * (i + 1) / num_partitions)]
379
+ for i in range(num_partitions - 1)
380
+ ]
381
+
382
+ def range_partitioner(key: Any) -> int:
383
+ pos = bisect.bisect_left(bounds, key_fn(key))
384
+ if ascending:
385
+ return pos
386
+ return num_partitions - 1 - pos
387
+
388
+ def sort_partition(iterator: Iterable[Any]) -> Iterable[Any]:
389
+ return iter(
390
+ sorted(
391
+ iterator,
392
+ key=lambda kv: key_fn(kv[0]),
393
+ reverse=not ascending,
394
+ )
395
+ )
396
+
397
+ return self.partition_by(num_partitions, range_partitioner).map_partitions(
398
+ sort_partition,
399
+ preserves_partitioning=True,
400
+ )
401
+
402
+ def sample(
403
+ self,
404
+ withReplacement: bool,
405
+ fraction: float,
406
+ seed: int | None = None,
407
+ ) -> RDD:
408
+ if fraction < 0:
409
+ raise ValueError("Fraction must be nonnegative.")
410
+ sampler = RDDSampler(withReplacement, fraction, seed)
411
+ return self.map_partitions_with_index(sampler.func, preserves_partitioning=True)
412
+
413
+ def takeSample(
414
+ self, withReplacement: bool, num: int, seed: int | None = None
415
+ ) -> list[Any]:
416
+ import random
417
+
418
+ fraction = min(1.0, (num * 10) / max(self.count(), 1))
419
+ sampled = self.sample(withReplacement, fraction, seed).collect()
420
+ rng = random.Random(seed)
421
+ if withReplacement:
422
+ return [rng.choice(sampled) for _ in range(num)] if sampled else []
423
+ return rng.sample(sampled, min(num, len(sampled)))
424
+
425
+ take_sample = takeSample
426
+
427
+ def _pipe_impl(self, command: str, env: dict[str, str], checkCode: bool) -> RDD:
428
+ def func(iterator: Iterable[Any]) -> Iterable[str]:
429
+ try:
430
+ proc = Popen(shlex.split(command), env=env, stdin=PIPE, stdout=PIPE, stderr=PIPE)
431
+ except (FileNotFoundError, OSError) as exc:
432
+ if checkCode:
433
+ raise RuntimeError(str(exc)) from exc
434
+ return
435
+
436
+ def write_input() -> None:
437
+ try:
438
+ assert proc.stdin is not None
439
+ for obj in iterator:
440
+ line = str(obj).rstrip("\n") + "\n"
441
+ proc.stdin.write(line.encode("utf-8"))
442
+ finally:
443
+ if proc.stdin is not None:
444
+ proc.stdin.close()
445
+
446
+ threading.Thread(target=write_input, daemon=True).start()
447
+ assert proc.stdout is not None
448
+ for raw in iter(proc.stdout.readline, b""):
449
+ yield raw.rstrip(b"\n").decode("utf-8")
450
+ proc.wait()
451
+ if checkCode and proc.returncode:
452
+ raise RuntimeError(
453
+ f"Pipe command {command!r} exited with code {proc.returncode}"
454
+ )
455
+
456
+ return self.map_partitions(func)
457
+
458
+ def pipe(self, command: str, env: dict[str, str] | None = None, checkCode: bool = False) -> RDD:
459
+ return _PipeRDD(self, command, dict(env or {}), checkCode, self._require_context())
460
+
461
+ def save_as_text_file(self, path: str, compressionCodecClass: str | None = None) -> None:
462
+ del compressionCodecClass
463
+ save_as_text_file(self._collect_partitions(), path)
464
+
465
+ saveAsTextFile = save_as_text_file
466
+
467
+ def save_as_pickle_file(self, path: str) -> None:
468
+ save_as_pickle_file(self._collect_partitions(), path)
469
+
470
+ saveAsPickleFile = save_as_pickle_file
471
+
472
+ def histogram(self, buckets: int | list[Any] | tuple[Any, ...]):
473
+ data = self.collect()
474
+ if not data:
475
+ if isinstance(buckets, int):
476
+ if buckets < 1:
477
+ raise ValueError("number of buckets must be >= 1")
478
+ raise ValueError("can not generate buckets from empty RDD")
479
+ if isinstance(buckets, (list, tuple)):
480
+ if len(buckets) < 2:
481
+ raise ValueError("buckets should have more than one value")
482
+ return list(buckets), [0] * (len(buckets) - 1)
483
+ if isinstance(buckets, int):
484
+ return histogram_values(data, buckets)
485
+
486
+ def partition_hist(iterator: Iterable[Any]) -> Iterable[list[int]]:
487
+ yield histogram_partition(iterator, buckets)
488
+
489
+ def merge(a: list[int], b: list[int]) -> list[int]:
490
+ return [x + y for x, y in zip(a, b)]
491
+
492
+ return buckets, self.map_partitions(partition_hist).reduce(merge)
493
+
494
+ def countApproxDistinct(self, relativeSD: float = 0.05) -> int:
495
+ return count_approx_distinct(self.collect(), relativeSD)
496
+
497
+ count_approx_distinct = countApproxDistinct
498
+
499
+ def treeAggregate(
500
+ self,
501
+ zeroValue,
502
+ seqOp,
503
+ combOp,
504
+ depth: int = 2,
505
+ ):
506
+ if depth < 1:
507
+ raise ValueError(f"Depth cannot be smaller than 1 but got {depth}.")
508
+ if self.getNumPartitions() == 0:
509
+ return zeroValue() if callable(zeroValue) else copy.deepcopy(zeroValue)
510
+
511
+ def aggregate_partition(iterator: Iterable[Any]) -> Iterable[Any]:
512
+ acc = zeroValue() if callable(zeroValue) else copy.deepcopy(zeroValue)
513
+ for obj in iterator:
514
+ acc = seqOp(acc, obj)
515
+ yield acc
516
+
517
+ partially = self.map_partitions(aggregate_partition, preserves_partitioning=True)
518
+ num_partitions = partially.getNumPartitions()
519
+ scale = max(int(math.ceil(pow(num_partitions, 1.0 / depth))), 2)
520
+ while num_partitions > scale + num_partitions / scale:
521
+ num_partitions = num_partitions / scale
522
+ cur = int(num_partitions)
523
+
524
+ def map_partition(index: int, iterator: Iterable[Any]) -> Iterable[tuple[int, Any]]:
525
+ for obj in iterator:
526
+ yield (index % cur, obj)
527
+
528
+ partially = (
529
+ partially.map_partitions_with_index(map_partition, preserves_partitioning=False)
530
+ .reduce_by_key(combOp, cur)
531
+ .map(lambda kv: kv[1])
532
+ )
533
+ num_partitions = partially.getNumPartitions()
534
+ return partially.reduce(combOp)
535
+
536
+ tree_aggregate = treeAggregate
537
+
538
+ def repartition_and_sort_within_partitions(
539
+ self,
540
+ num_partitions: int,
541
+ partition_func,
542
+ ascending: bool = True,
543
+ keyfunc: Callable[[Any], Any] | None = None,
544
+ ) -> RDD:
545
+ key_fn = keyfunc or (lambda x: x)
546
+
547
+ def sort_partition(iterator: Iterable[Any]) -> Iterable[Any]:
548
+ return iter(
549
+ sorted(
550
+ iterator,
551
+ key=lambda kv: key_fn(kv[0]),
552
+ reverse=not ascending,
553
+ )
554
+ )
555
+
556
+ return self.partition_by(num_partitions, partition_func).map_partitions(
557
+ sort_partition,
558
+ preserves_partitioning=True,
559
+ )
560
+
561
+ repartitionAndSortWithinPartitions = repartition_and_sort_within_partitions
562
+
563
+ def to_local_iterator(self, prefetch_partitions: bool = False) -> Iterator[Any]:
564
+ self._materialize()
565
+ total = self._num_partitions
566
+ cache: dict[int, list[Any]] = {}
567
+ lock = threading.Lock()
568
+
569
+ def prefetch(pid: int) -> None:
570
+ if pid >= total:
571
+ return
572
+ data = self._collect_partition(pid)
573
+ with lock:
574
+ cache[pid] = data
575
+
576
+ for pid in range(total):
577
+ if prefetch_partitions and pid + 1 < total:
578
+ threading.Thread(target=prefetch, args=(pid + 1,), daemon=True).start()
579
+ with lock:
580
+ part = cache.pop(pid, None)
581
+ if part is None:
582
+ part = self._collect_partition(pid)
583
+ yield from part
584
+
585
+ toLocalIterator = to_local_iterator
586
+
587
+ def cartesian(self, other: RDD) -> RDD:
588
+ left = self.collect()
589
+ right = other.collect()
590
+ return self._require_context().parallelize(
591
+ [(a, b) for a in left for b in right],
592
+ self._num_partitions,
593
+ )
594
+
595
+ def zip(self, other: RDD) -> RDD:
596
+ if self._num_partitions != other._num_partitions:
597
+ raise ValueError(
598
+ "Cannot zip RDDs with different number of partitions: "
599
+ f"{self._num_partitions} != {other._num_partitions}"
600
+ )
601
+ return _ZipRDD(self, other, self._require_context())
602
+
603
+ def aggregate(self, zeroValue, seqOp, combOp):
604
+ def apply_seq(acc, value):
605
+ try:
606
+ return seqOp(acc, value)
607
+ except StopIteration as exc:
608
+ raise RuntimeError(
609
+ "Caught StopIteration thrown from user's code; failing the task"
610
+ ) from exc
611
+
612
+ def apply_comb(a, b):
613
+ try:
614
+ return combOp(a, b)
615
+ except StopIteration as exc:
616
+ raise RuntimeError(
617
+ "Caught StopIteration thrown from user's code; failing the task"
618
+ ) from exc
619
+
620
+ partials: list[Any] = []
621
+ for part in self._collect_partitions():
622
+ acc = zeroValue() if callable(zeroValue) else copy.deepcopy(zeroValue)
623
+ for value in part:
624
+ acc = apply_seq(acc, value)
625
+ partials.append(acc)
626
+ if not partials:
627
+ return zeroValue() if callable(zeroValue) else copy.deepcopy(zeroValue)
628
+ acc = partials[0]
629
+ for partial in partials[1:]:
630
+ acc = apply_comb(acc, partial)
631
+ return acc
632
+
633
+ def aggregate_by_key(self, zeroValue, seqOp, combOp, num_partitions: int = 0):
634
+ parts = num_partitions or self._num_partitions
635
+
636
+ def merge_values(kv):
637
+ key, values = kv
638
+ acc = zeroValue() if callable(zeroValue) else copy.deepcopy(zeroValue)
639
+ for value in values:
640
+ acc = seqOp(acc, value)
641
+ return (key, acc)
642
+
643
+ return self.group_by_key(parts).map(merge_values)
644
+
645
+ aggregateByKey = aggregate_by_key
646
+
647
+ def fold(self, zeroValue, op):
648
+ return self.aggregate(zeroValue, op, op)
649
+
650
+ def fold_by_key(self, zeroValue, op, num_partitions: int = 0):
651
+ return self.aggregate_by_key(zeroValue, op, op, num_partitions)
652
+
653
+ foldByKey = fold_by_key
654
+
655
+ def cache(self) -> RDD:
656
+ return self
657
+
658
+ def id(self) -> int:
659
+ return self._logical_id
660
+
661
+ def _require_context(self) -> "SparkContext":
662
+ if self._context is None:
663
+ raise RuntimeError("RDD has no SparkContext for client-side operation")
664
+ return self._context
665
+
666
+ @property
667
+ def num_partitions(self) -> int:
668
+ return self._num_partitions
669
+
670
+ def getNumPartitions(self) -> int:
671
+ return self._num_partitions
672
+
673
+ def __repr__(self) -> str:
674
+ pending = len(self._pending)
675
+ suffix = f", pending_ops={pending}" if pending else ""
676
+ return f"RDD(num_partitions={self.num_partitions}{suffix})"
677
+
678
+
679
+ class _PipeRDD(RDD):
680
+ def __init__(
681
+ self,
682
+ parent: RDD,
683
+ command: str,
684
+ env: dict[str, str],
685
+ checkCode: bool,
686
+ context: SparkContext,
687
+ ):
688
+ self._pipe_parent = parent
689
+ self._pipe_command = command
690
+ self._pipe_env = env
691
+ self._pipe_check_code = checkCode
692
+ super().__init__(
693
+ parent._client,
694
+ parent._rdd_id,
695
+ parent._num_partitions,
696
+ pending=list(parent._pending),
697
+ native=parent._native,
698
+ native_kind=parent._native_kind,
699
+ line_meta=parent._line_meta,
700
+ context=context,
701
+ text_file_path=parent._text_file_path,
702
+ text_file_parts=parent._text_file_parts,
703
+ )
704
+
705
+ def _materialize_pipe(self) -> None:
706
+ if self._pipe_parent is None:
707
+ return
708
+ piped = self._pipe_parent._pipe_impl(
709
+ self._pipe_command, self._pipe_env, self._pipe_check_code
710
+ )
711
+ self._rdd_id = piped._rdd_id
712
+ self._num_partitions = piped._num_partitions
713
+ self._pending = list(piped._pending)
714
+ self._native = piped._native
715
+ self._native_kind = piped._native_kind
716
+ self._pipe_parent = None
717
+
718
+ def _resolve_pending(self, action: Action, action_func=None) -> Any:
719
+ self._materialize_pipe()
720
+ return super()._resolve_pending(action, action_func)
721
+
722
+ def _run(self, action: Action, action_func=None) -> Any:
723
+ if self._pipe_parent is not None:
724
+ self._materialize_pipe()
725
+ return super()._run(action, action_func)
726
+
727
+
728
+ class _ZipRDD(RDD):
729
+ def __init__(self, left: RDD, right: RDD, context: SparkContext):
730
+ self._zip_left = left
731
+ self._zip_right = right
732
+ super().__init__(
733
+ left._client,
734
+ left._rdd_id,
735
+ left._num_partitions,
736
+ pending=list(left._pending),
737
+ native=left._native,
738
+ native_kind=left._native_kind,
739
+ line_meta=left._line_meta,
740
+ context=context,
741
+ text_file_path=left._text_file_path,
742
+ text_file_parts=left._text_file_parts,
743
+ )
744
+
745
+ def _materialize_zipped(self) -> None:
746
+ left_parts = self._zip_left._collect_partitions()
747
+ right_parts = self._zip_right._collect_partitions()
748
+ if len(left_parts) != len(right_parts):
749
+ raise RuntimeError("Cannot zip RDDs with different number of partitions")
750
+ out: list[list[Any]] = []
751
+ for left, right in zip(left_parts, right_parts):
752
+ if len(left) != len(right):
753
+ raise ValueError(
754
+ "Cannot zip RDDs with different number of elements in partition"
755
+ )
756
+ out.append(list(zip(left, right)))
757
+ rdd_id, parts = self._client.parallelize_partitions(out)
758
+ self._rdd_id = rdd_id
759
+ self._num_partitions = parts
760
+ self._pending = []
761
+ self._native = False
762
+ self._zip_left = None
763
+ self._zip_right = None
764
+
765
+ def _collect_partitions(self) -> list[list[Any]]:
766
+ if self._zip_left is not None:
767
+ self._materialize_zipped()
768
+ return super()._collect_partitions()
769
+
770
+ def _resolve_pending(self, action: Action, action_func=None) -> Any:
771
+ self._materialize_zipped()
772
+ return super()._resolve_pending(action, action_func)
773
+
774
+ def _run(self, action: Action, action_func=None) -> Any:
775
+ if self._zip_left is not None:
776
+ self._materialize_zipped()
777
+ return super()._run(action, action_func)
778
+
779
+
780
+ class SparkContext:
781
+ """Entry point for PyVega. Talks to a Vega C++ session server."""
782
+
783
+ def __init__(self, num_workers: int = 0, remote: Optional[str] = None):
784
+ self._server_proc = None
785
+ self._num_workers = num_workers
786
+ self._pickled_broadcast_vars = BroadcastPickleRegistry()
787
+
788
+ if remote:
789
+ host, port = parse_remote(remote)
790
+ else:
791
+ self._server_proc, host, port = spawn_server(num_workers)
792
+
793
+ self._client = SessionClient(host, port, num_workers)
794
+ self._num_workers = self._client.num_workers
795
+
796
+ def range(self, start: int, end: int, num_partitions: int = 0) -> RDD:
797
+ """Create an RDD of integers [start, end) generated on the server."""
798
+ if num_partitions == 0:
799
+ num_partitions = self._num_workers
800
+ rdd_id, parts = self._client.parallelize_range(start, end, num_partitions)
801
+ return RDD(
802
+ self._client, rdd_id, parts, native=True, native_kind="int64", context=self
803
+ )
804
+
805
+ def float_range(self, start: int, end: int, num_partitions: int = 0) -> RDD:
806
+ """Create a native float64 RDD [start, end) generated on the server."""
807
+ if num_partitions == 0:
808
+ num_partitions = self._num_workers
809
+ rdd_id, parts = self._client.parallelize_float_range(start, end, num_partitions)
810
+ return RDD(
811
+ self._client, rdd_id, parts, native=True, native_kind="float64", context=self
812
+ )
813
+
814
+ def string_lines(
815
+ self,
816
+ start: int,
817
+ end: int,
818
+ num_keys: int,
819
+ num_tokens: int = 97,
820
+ num_partitions: int = 0,
821
+ ) -> RDD:
822
+ """Create a native string RDD of synthetic lines generated on the server."""
823
+ if num_partitions == 0:
824
+ num_partitions = self._num_workers
825
+ rdd_id, parts = self._client.parallelize_string_lines(
826
+ start, end, num_keys, num_tokens, num_partitions
827
+ )
828
+ return RDD(
829
+ self._client,
830
+ rdd_id,
831
+ parts,
832
+ native=True,
833
+ native_kind="str",
834
+ line_meta=(num_keys, num_tokens),
835
+ context=self,
836
+ )
837
+
838
+ @staticmethod
839
+ def _detect_native_kind(data_list: list) -> tuple[bool, NativeKind]:
840
+ if not data_list:
841
+ return False, "int64"
842
+ if all(isinstance(x, int) and not isinstance(x, bool) for x in data_list):
843
+ return True, "int64"
844
+ if all(
845
+ isinstance(x, (int, float)) and not isinstance(x, bool) for x in data_list
846
+ ):
847
+ return True, "float64"
848
+ if all(isinstance(x, str) for x in data_list):
849
+ return True, "str"
850
+ return False, "int64"
851
+
852
+ def parallelize(self, data, num_partitions: int = 0) -> RDD:
853
+ if num_partitions == 0:
854
+ num_partitions = self._num_workers
855
+ data_list = list(data)
856
+ native, native_kind = self._detect_native_kind(data_list)
857
+ rdd_id, parts = self._client.parallelize(data_list, num_partitions)
858
+ return RDD(
859
+ self._client, rdd_id, parts, native=native, native_kind=native_kind, context=self
860
+ )
861
+
862
+ def empty_rdd(self) -> RDD:
863
+ return self.parallelize([], 1)
864
+
865
+ emptyRDD = empty_rdd
866
+
867
+ def text_file(self, path: str, min_partitions: int = 0) -> RDD:
868
+ parts = min_partitions or self._num_workers
869
+ if os.path.isdir(path):
870
+ return self.parallelize(read_text_file(path), parts)
871
+ return RDD(
872
+ self._client,
873
+ 0,
874
+ parts,
875
+ context=self,
876
+ text_file_path=path,
877
+ text_file_parts=parts,
878
+ )
879
+
880
+ textFile = text_file
881
+
882
+ def pickle_file(self, path: str, min_partitions: int = 0) -> RDD:
883
+ data = read_pickle_file(path)
884
+ parts = min_partitions or self._num_workers
885
+ return self.parallelize(data, parts)
886
+
887
+ pickleFile = pickle_file
888
+
889
+ def broadcast(self, value: Any) -> Broadcast[Any]:
890
+ return Broadcast(self, value, self._pickled_broadcast_vars)
891
+
892
+ def union(self, rdds: list[RDD]) -> RDD:
893
+ data: list[Any] = []
894
+ for rdd in rdds:
895
+ data.extend(rdd.collect())
896
+ return self.parallelize(data, max(1, len(rdds)))
897
+
898
+ @property
899
+ def num_workers(self) -> int:
900
+ return self._num_workers
901
+
902
+ def stop(self) -> None:
903
+ self._client.close()
904
+ if self._server_proc is not None:
905
+ self._server_proc.terminate()
906
+ try:
907
+ self._server_proc.wait(timeout=5)
908
+ except Exception:
909
+ self._server_proc.kill()
910
+ self._server_proc = None
911
+
912
+ def __enter__(self):
913
+ return self
914
+
915
+ def __exit__(self, *exc):
916
+ self.stop()
917
+
918
+ def __repr__(self) -> str:
919
+ return f"SparkContext(num_workers={self.num_workers})"