pyvega 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyvega-0.5.0/LICENSE +21 -0
- pyvega-0.5.0/NOTICE +42 -0
- pyvega-0.5.0/PKG-INFO +282 -0
- pyvega-0.5.0/README.md +256 -0
- pyvega-0.5.0/pyproject.toml +35 -0
- pyvega-0.5.0/python/pyvega/__init__.py +919 -0
- pyvega-0.5.0/python/pyvega/_client.py +490 -0
- pyvega-0.5.0/python/pyvega/_io.py +58 -0
- pyvega-0.5.0/python/pyvega/_native.py +1031 -0
- pyvega-0.5.0/python/pyvega/_protocol.py +171 -0
- pyvega-0.5.0/python/pyvega/_sampler.py +67 -0
- pyvega-0.5.0/python/pyvega/_server_cli.py +28 -0
- pyvega-0.5.0/python/pyvega/_stats.py +113 -0
- pyvega-0.5.0/python/pyvega/broadcast.py +78 -0
- pyvega-0.5.0/python/pyvega.egg-info/PKG-INFO +282 -0
- pyvega-0.5.0/python/pyvega.egg-info/SOURCES.txt +20 -0
- pyvega-0.5.0/python/pyvega.egg-info/dependency_links.txt +1 -0
- pyvega-0.5.0/python/pyvega.egg-info/entry_points.txt +2 -0
- pyvega-0.5.0/python/pyvega.egg-info/requires.txt +4 -0
- pyvega-0.5.0/python/pyvega.egg-info/top_level.txt +1 -0
- pyvega-0.5.0/setup.cfg +4 -0
- pyvega-0.5.0/setup.py +8 -0
pyvega-0.5.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 The Vega Authors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
pyvega-0.5.0/NOTICE
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
Vega
|
|
2
|
+
Copyright (c) 2026 The Vega Authors
|
|
3
|
+
|
|
4
|
+
This product is licensed under the MIT License (see LICENSE).
|
|
5
|
+
|
|
6
|
+
Vega is not affiliated with, endorsed by, or sponsored by the Apache Software
|
|
7
|
+
Foundation or the Apache Spark project. "Apache Spark", "Spark", and "PySpark"
|
|
8
|
+
are trademarks of the Apache Software Foundation. Vega provides a Spark-like
|
|
9
|
+
API for familiarity and compatibility testing only.
|
|
10
|
+
|
|
11
|
+
--------------------------------------------------------------------------------
|
|
12
|
+
Third-party components
|
|
13
|
+
--------------------------------------------------------------------------------
|
|
14
|
+
|
|
15
|
+
This repository's tooling fetches and/or builds against the following
|
|
16
|
+
third-party software. These components are NOT part of Vega and retain their
|
|
17
|
+
own licenses.
|
|
18
|
+
|
|
19
|
+
1. Apache Spark
|
|
20
|
+
- Used by: PySpark compatibility tests under scripts/pyspark_compat and the
|
|
21
|
+
vendored/fetched sources under third_party/spark (fetched via
|
|
22
|
+
scripts/fetch_spark_tests.sh; not required to build or run Vega).
|
|
23
|
+
- License: Apache License 2.0
|
|
24
|
+
- https://github.com/apache/spark
|
|
25
|
+
- When distributing any portion of Apache Spark sources, you must comply
|
|
26
|
+
with the Apache License 2.0, including its NOTICE file
|
|
27
|
+
(third_party/spark/LICENSE and third_party/spark/NOTICE).
|
|
28
|
+
|
|
29
|
+
2. pybind11
|
|
30
|
+
- Used by: vega_server (embedded Python).
|
|
31
|
+
- License: BSD-3-Clause
|
|
32
|
+
- https://github.com/pybind/pybind11
|
|
33
|
+
|
|
34
|
+
3. GoogleTest
|
|
35
|
+
- Used by: C++ test suite.
|
|
36
|
+
- License: BSD-3-Clause
|
|
37
|
+
- https://github.com/google/googletest
|
|
38
|
+
|
|
39
|
+
4. cloudpickle
|
|
40
|
+
- Used by: PyVega client (runtime dependency).
|
|
41
|
+
- License: BSD-3-Clause
|
|
42
|
+
- https://github.com/cloudpipe/cloudpickle
|
pyvega-0.5.0/PKG-INFO
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pyvega
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: Python client for the Vega C++ distributed data processing engine
|
|
5
|
+
Author: The Vega Authors
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/sphulinga/vega
|
|
8
|
+
Project-URL: Documentation, https://github.com/sphulinga/vega/tree/main/docs
|
|
9
|
+
Project-URL: Repository, https://github.com/sphulinga/vega
|
|
10
|
+
Project-URL: Issues, https://github.com/sphulinga/vega/issues
|
|
11
|
+
Keywords: distributed,rdd,spark,data-processing,actors
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: C++
|
|
17
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
18
|
+
Requires-Python: >=3.8
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
License-File: NOTICE
|
|
22
|
+
Requires-Dist: cloudpickle>=3.0
|
|
23
|
+
Provides-Extra: test
|
|
24
|
+
Requires-Dist: pytest>=7.0; extra == "test"
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
# Vega
|
|
28
|
+
|
|
29
|
+
An in-memory distributed data processing engine written in C++17. Vega
|
|
30
|
+
provides a Spark-like RDD API with lazy evaluation, but replaces Spark's
|
|
31
|
+
centralized driver/scheduler with a **decentralized actor model** where
|
|
32
|
+
workers coordinate through message passing.
|
|
33
|
+
|
|
34
|
+
**PyVega** is the Python client: it connects to a C++ **session server** over TCP,
|
|
35
|
+
records transformations lazily, and executes them on an embedded actor runtime with
|
|
36
|
+
Python UDF support.
|
|
37
|
+
|
|
38
|
+
## Documentation
|
|
39
|
+
|
|
40
|
+
| Guide | Description |
|
|
41
|
+
|-------|-------------|
|
|
42
|
+
| [**Docs index**](docs/README.md) | Start here |
|
|
43
|
+
| [Getting started](docs/getting-started.md) | Build, install, first program |
|
|
44
|
+
| [Server, workers, and clients](docs/server-and-workers.md) | `vega_server`, worker threads, cluster peers |
|
|
45
|
+
| [Python RDD guide](docs/python-guide.md) | PyVega API reference with examples |
|
|
46
|
+
| [DataFrame-style operations](docs/dataframe-operations.md) | SQL-like patterns using RDDs |
|
|
47
|
+
| [PySpark compatibility](docs/pyspark-compat.md) | Run PySpark RDD code and upstream tests |
|
|
48
|
+
|
|
49
|
+
## Quick start (Python)
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
# Build session server
|
|
53
|
+
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DVEGA_BUILD_SERVER=ON
|
|
54
|
+
cmake --build build --target vega_server -j$(nproc)
|
|
55
|
+
|
|
56
|
+
# Install client
|
|
57
|
+
python -m venv .venv && source .venv/bin/activate
|
|
58
|
+
pip install -e ".[test]"
|
|
59
|
+
export VEGA_SERVER="$PWD/build/vega_server"
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from pyvega import SparkContext
|
|
64
|
+
|
|
65
|
+
with SparkContext(4) as sc:
|
|
66
|
+
print(sc.range(0, 10).map(lambda x: x * 2).collect())
|
|
67
|
+
# [0, 2, 4, 6, 8, 10, 12, 14, 16, 18]
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
**Manual server:** `vega-server --bind 127.0.0.1:9500` then
|
|
71
|
+
`SparkContext(remote="127.0.0.1:9500")`. See [server and clients](docs/server-and-workers.md).
|
|
72
|
+
|
|
73
|
+
> **Note:** PyVega provides **RDD** APIs, not Spark SQL DataFrames. See
|
|
74
|
+
> [DataFrame-style operations](docs/dataframe-operations.md) for RDD equivalents.
|
|
75
|
+
|
|
76
|
+
## Architecture
|
|
77
|
+
|
|
78
|
+
```
|
|
79
|
+
User Code (Python or C++)
|
|
80
|
+
|
|
|
81
|
+
SparkContext / Cluster
|
|
82
|
+
|
|
|
83
|
+
+-----------+
|
|
84
|
+
| JobActor | (builds DAG, distributes tasks)
|
|
85
|
+
+-----------+
|
|
86
|
+
/ | \
|
|
87
|
+
+------+ +------+ +------+
|
|
88
|
+
|Worker| |Worker| |Worker| (execute partitions in parallel)
|
|
89
|
+
+------+ +------+ +------+
|
|
90
|
+
\ | /
|
|
91
|
+
+-------------------------+
|
|
92
|
+
| Actor System | (thread pool, message routing)
|
|
93
|
+
+-------------------------+
|
|
94
|
+
| Transport Layer | (Local / TCP + Gossip)
|
|
95
|
+
+-------------------------+
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
For PyVega, the actor system runs **inside `vega_server`**. The Python process is a thin
|
|
99
|
+
RPC client; worker threads are not separate OS processes.
|
|
100
|
+
|
|
101
|
+
**Key difference from Spark:** there is no central scheduler in cluster mode. Peers
|
|
102
|
+
discover each other via gossip and coordinate execution directly. In single-node /
|
|
103
|
+
session-server mode, the JobActor still orchestrates stages but all workers are local
|
|
104
|
+
threads.
|
|
105
|
+
|
|
106
|
+
### Actor framework
|
|
107
|
+
|
|
108
|
+
Vega includes a lightweight actor framework inspired by
|
|
109
|
+
[rotor](https://github.com/basiliscos/cpp-rotor):
|
|
110
|
+
|
|
111
|
+
- **Message-based communication** — typed messages dispatched to registered handlers
|
|
112
|
+
- **Thread pool execution** — actors process messages in parallel across worker threads
|
|
113
|
+
- **Sequential per-actor** — each actor processes one message at a time
|
|
114
|
+
- **Python GIL integration** — worker threads acquire the GIL when executing Python UDFs
|
|
115
|
+
|
|
116
|
+
## Build (C++)
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
cmake -B build -DCMAKE_BUILD_TYPE=Release
|
|
120
|
+
cmake --build build -j$(nproc)
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Targets include `vega_server`, `vega_worker`, examples, and C++ tests.
|
|
124
|
+
|
|
125
|
+
## Python API (PyVega)
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
from pyvega import SparkContext
|
|
129
|
+
|
|
130
|
+
with SparkContext(4) as sc:
|
|
131
|
+
rdd = sc.parallelize([1, 2, 3, 4, 5], num_partitions=2)
|
|
132
|
+
|
|
133
|
+
result = (
|
|
134
|
+
rdd.filter(lambda x: x > 2)
|
|
135
|
+
.map(lambda x: x * 10)
|
|
136
|
+
.collect()
|
|
137
|
+
)
|
|
138
|
+
# [30, 40, 50]
|
|
139
|
+
|
|
140
|
+
# Word count
|
|
141
|
+
lines = ["hello world hello", "hello vega"]
|
|
142
|
+
counts = (
|
|
143
|
+
sc.parallelize(lines)
|
|
144
|
+
.flat_map(lambda line: line.split())
|
|
145
|
+
.map(lambda word: (word, 1))
|
|
146
|
+
.reduce_by_key(lambda a, b: a + b)
|
|
147
|
+
.collect()
|
|
148
|
+
)
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Full API: [Python RDD guide](docs/python-guide.md).
|
|
152
|
+
|
|
153
|
+
## Running on a cluster (C++)
|
|
154
|
+
|
|
155
|
+
`vega_worker` runs **equal peers** (no special driver role). PyVega does not connect
|
|
156
|
+
to cluster peers yet; use this for C++ multi-node examples.
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
# Terminal 1
|
|
160
|
+
./build/vega_worker --bind 127.0.0.1:9001 --id peer-1
|
|
161
|
+
|
|
162
|
+
# Terminal 2
|
|
163
|
+
./build/vega_worker --bind 127.0.0.1:9002 --id peer-2 --seeds 127.0.0.1:9001
|
|
164
|
+
|
|
165
|
+
# Terminal 3 — submitter (same binary, not a "driver")
|
|
166
|
+
./build/examples/cluster_word_count --bind 127.0.0.1:9003 --id peer-3 \
|
|
167
|
+
--seeds 127.0.0.1:9001 --expect-peers 3
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
Single-node demo: `./build/examples/cluster_word_count --local`
|
|
171
|
+
|
|
172
|
+
| Flag | Description |
|
|
173
|
+
|------|-------------|
|
|
174
|
+
| `--bind HOST:PORT` | Listen address |
|
|
175
|
+
| `--seeds H1:P1,H2:P2` | Gossip seed peers |
|
|
176
|
+
| `--id NAME` | Node identifier |
|
|
177
|
+
| `--expect-peers N` | Wait for N peers before running (examples) |
|
|
178
|
+
|
|
179
|
+
Details: [Server, workers, and clients](docs/server-and-workers.md).
|
|
180
|
+
|
|
181
|
+
## C++ API
|
|
182
|
+
|
|
183
|
+
```cpp
|
|
184
|
+
#include <vega/vega.hpp>
|
|
185
|
+
|
|
186
|
+
int main() {
|
|
187
|
+
vega::SparkContext ctx(4);
|
|
188
|
+
auto rdd = ctx.parallelize(std::vector<int>{1, 2, 3, 4, 5}, 2);
|
|
189
|
+
auto result = rdd
|
|
190
|
+
.filter([](const int& x) { return x > 2; })
|
|
191
|
+
.map([](const int& x) { return x * 10; })
|
|
192
|
+
.collect();
|
|
193
|
+
}
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
### Available operations
|
|
197
|
+
|
|
198
|
+
| Transformation | Python | C++ |
|
|
199
|
+
|----------------|--------|-----|
|
|
200
|
+
| `map` | `rdd.map(func)` | `rdd.map(func)` |
|
|
201
|
+
| `filter` | `rdd.filter(func)` | `rdd.filter(pred)` |
|
|
202
|
+
| `flat_map` | `rdd.flat_map(func)` | `rdd.flatMap(func)` |
|
|
203
|
+
| `reduceByKey` | `rdd.reduce_by_key(func)` | `rdd.reduceByKey(func)` |
|
|
204
|
+
| `groupByKey` | `rdd.group_by_key()` | `rdd.groupByKey()` |
|
|
205
|
+
|
|
206
|
+
| Action | Python | C++ |
|
|
207
|
+
|--------|--------|-----|
|
|
208
|
+
| `collect` | `rdd.collect()` | `rdd.collect()` |
|
|
209
|
+
| `count` | `rdd.count()` | `rdd.count()` |
|
|
210
|
+
| `reduce` | `rdd.reduce(func)` | `rdd.reduce(func)` |
|
|
211
|
+
| `foreach` | `rdd.foreach(func)` | `rdd.foreach(func)` |
|
|
212
|
+
|
|
213
|
+
## Project structure
|
|
214
|
+
|
|
215
|
+
```
|
|
216
|
+
vega/
|
|
217
|
+
docs/ User guides (start at docs/README.md)
|
|
218
|
+
include/vega/ C++ header-only library
|
|
219
|
+
core/ RDD, SparkContext, shuffle
|
|
220
|
+
actors/ JobActor, WorkerActor, messages
|
|
221
|
+
execution/ DAG builder, stages
|
|
222
|
+
cluster/ P2P cluster, gossip
|
|
223
|
+
network/ TCP framing, routing
|
|
224
|
+
src/
|
|
225
|
+
vega_server.cpp TCP session server (PyVega backend)
|
|
226
|
+
server_session.cpp Session RPC + embedded Python UDFs
|
|
227
|
+
vega_worker.cpp Cluster peer daemon
|
|
228
|
+
python/pyvega/ PyVega client package
|
|
229
|
+
scripts/
|
|
230
|
+
pyspark_compat/ PySpark RDD API shim
|
|
231
|
+
run_pyspark_tests.py Upstream Spark RDD tests vs PyVega
|
|
232
|
+
third_party/spark/ Vendored Spark test sources
|
|
233
|
+
examples/ word_count, cluster_word_count, benchmarks
|
|
234
|
+
tests/ C++ unit tests + Spark RDD suite ports
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
## Spark compatibility testing
|
|
238
|
+
|
|
239
|
+
```bash
|
|
240
|
+
# C++ Spark RDD logic ports
|
|
241
|
+
cmake --build build --target test_spark_rdd_suite
|
|
242
|
+
cd build && ctest -R SparkRDDSuite --output-on-failure
|
|
243
|
+
|
|
244
|
+
# Upstream PySpark RDD tests (50/54 pass; 4 require JVM)
|
|
245
|
+
VEGA_SERVER=build/vega_server \
|
|
246
|
+
python scripts/run_pyspark_tests.py --filter pyspark.tests.test_rdd
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
See [PySpark compatibility](docs/pyspark-compat.md).
|
|
250
|
+
|
|
251
|
+
## Execution flow
|
|
252
|
+
|
|
253
|
+
### PyVega (session server)
|
|
254
|
+
|
|
255
|
+
1. Client builds an RDD chain lazily (`map`, `filter`, …)
|
|
256
|
+
2. First action sends fused stages over TCP
|
|
257
|
+
3. Server materializes lineage and submits a job to JobActor
|
|
258
|
+
4. WorkerActors compute partitions (native C++ or embedded Python)
|
|
259
|
+
5. Shuffle stages (`reduceByKey`, `groupByKey`) run as multi-stage DAGs
|
|
260
|
+
6. Results return to the Python client
|
|
261
|
+
|
|
262
|
+
### Multi-node C++ cluster
|
|
263
|
+
|
|
264
|
+
1. Peers start and discover each other via gossip
|
|
265
|
+
2. Submitter distributes partition data and task assignments over TCP
|
|
266
|
+
3. Peers execute tasks and return serialized results
|
|
267
|
+
|
|
268
|
+
## Contributing
|
|
269
|
+
|
|
270
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for build/test instructions and PR
|
|
271
|
+
guidelines, and our [Code of Conduct](CODE_OF_CONDUCT.md). Report security
|
|
272
|
+
issues privately per [SECURITY.md](SECURITY.md).
|
|
273
|
+
|
|
274
|
+
## License
|
|
275
|
+
|
|
276
|
+
Licensed under the [MIT License](LICENSE).
|
|
277
|
+
|
|
278
|
+
Vega is **not affiliated with the Apache Software Foundation**. "Apache Spark"
|
|
279
|
+
and "PySpark" are trademarks of the ASF; Vega offers a Spark-like API for
|
|
280
|
+
familiarity and compatibility testing only. See [NOTICE](NOTICE) for
|
|
281
|
+
third-party attributions (Apache Spark test sources, pybind11, GoogleTest,
|
|
282
|
+
cloudpickle).
|
pyvega-0.5.0/README.md
ADDED
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
# Vega
|
|
2
|
+
|
|
3
|
+
An in-memory distributed data processing engine written in C++17. Vega
|
|
4
|
+
provides a Spark-like RDD API with lazy evaluation, but replaces Spark's
|
|
5
|
+
centralized driver/scheduler with a **decentralized actor model** where
|
|
6
|
+
workers coordinate through message passing.
|
|
7
|
+
|
|
8
|
+
**PyVega** is the Python client: it connects to a C++ **session server** over TCP,
|
|
9
|
+
records transformations lazily, and executes them on an embedded actor runtime with
|
|
10
|
+
Python UDF support.
|
|
11
|
+
|
|
12
|
+
## Documentation
|
|
13
|
+
|
|
14
|
+
| Guide | Description |
|
|
15
|
+
|-------|-------------|
|
|
16
|
+
| [**Docs index**](docs/README.md) | Start here |
|
|
17
|
+
| [Getting started](docs/getting-started.md) | Build, install, first program |
|
|
18
|
+
| [Server, workers, and clients](docs/server-and-workers.md) | `vega_server`, worker threads, cluster peers |
|
|
19
|
+
| [Python RDD guide](docs/python-guide.md) | PyVega API reference with examples |
|
|
20
|
+
| [DataFrame-style operations](docs/dataframe-operations.md) | SQL-like patterns using RDDs |
|
|
21
|
+
| [PySpark compatibility](docs/pyspark-compat.md) | Run PySpark RDD code and upstream tests |
|
|
22
|
+
|
|
23
|
+
## Quick start (Python)
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
# Build session server
|
|
27
|
+
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DVEGA_BUILD_SERVER=ON
|
|
28
|
+
cmake --build build --target vega_server -j$(nproc)
|
|
29
|
+
|
|
30
|
+
# Install client
|
|
31
|
+
python -m venv .venv && source .venv/bin/activate
|
|
32
|
+
pip install -e ".[test]"
|
|
33
|
+
export VEGA_SERVER="$PWD/build/vega_server"
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
from pyvega import SparkContext
|
|
38
|
+
|
|
39
|
+
with SparkContext(4) as sc:
|
|
40
|
+
print(sc.range(0, 10).map(lambda x: x * 2).collect())
|
|
41
|
+
# [0, 2, 4, 6, 8, 10, 12, 14, 16, 18]
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
**Manual server:** `vega-server --bind 127.0.0.1:9500` then
|
|
45
|
+
`SparkContext(remote="127.0.0.1:9500")`. See [server and clients](docs/server-and-workers.md).
|
|
46
|
+
|
|
47
|
+
> **Note:** PyVega provides **RDD** APIs, not Spark SQL DataFrames. See
|
|
48
|
+
> [DataFrame-style operations](docs/dataframe-operations.md) for RDD equivalents.
|
|
49
|
+
|
|
50
|
+
## Architecture
|
|
51
|
+
|
|
52
|
+
```
|
|
53
|
+
User Code (Python or C++)
|
|
54
|
+
|
|
|
55
|
+
SparkContext / Cluster
|
|
56
|
+
|
|
|
57
|
+
+-----------+
|
|
58
|
+
| JobActor | (builds DAG, distributes tasks)
|
|
59
|
+
+-----------+
|
|
60
|
+
/ | \
|
|
61
|
+
+------+ +------+ +------+
|
|
62
|
+
|Worker| |Worker| |Worker| (execute partitions in parallel)
|
|
63
|
+
+------+ +------+ +------+
|
|
64
|
+
\ | /
|
|
65
|
+
+-------------------------+
|
|
66
|
+
| Actor System | (thread pool, message routing)
|
|
67
|
+
+-------------------------+
|
|
68
|
+
| Transport Layer | (Local / TCP + Gossip)
|
|
69
|
+
+-------------------------+
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
For PyVega, the actor system runs **inside `vega_server`**. The Python process is a thin
|
|
73
|
+
RPC client; worker threads are not separate OS processes.
|
|
74
|
+
|
|
75
|
+
**Key difference from Spark:** there is no central scheduler in cluster mode. Peers
|
|
76
|
+
discover each other via gossip and coordinate execution directly. In single-node /
|
|
77
|
+
session-server mode, the JobActor still orchestrates stages but all workers are local
|
|
78
|
+
threads.
|
|
79
|
+
|
|
80
|
+
### Actor framework
|
|
81
|
+
|
|
82
|
+
Vega includes a lightweight actor framework inspired by
|
|
83
|
+
[rotor](https://github.com/basiliscos/cpp-rotor):
|
|
84
|
+
|
|
85
|
+
- **Message-based communication** — typed messages dispatched to registered handlers
|
|
86
|
+
- **Thread pool execution** — actors process messages in parallel across worker threads
|
|
87
|
+
- **Sequential per-actor** — each actor processes one message at a time
|
|
88
|
+
- **Python GIL integration** — worker threads acquire the GIL when executing Python UDFs
|
|
89
|
+
|
|
90
|
+
## Build (C++)
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
cmake -B build -DCMAKE_BUILD_TYPE=Release
|
|
94
|
+
cmake --build build -j$(nproc)
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Targets include `vega_server`, `vega_worker`, examples, and C++ tests.
|
|
98
|
+
|
|
99
|
+
## Python API (PyVega)
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
from pyvega import SparkContext
|
|
103
|
+
|
|
104
|
+
with SparkContext(4) as sc:
|
|
105
|
+
rdd = sc.parallelize([1, 2, 3, 4, 5], num_partitions=2)
|
|
106
|
+
|
|
107
|
+
result = (
|
|
108
|
+
rdd.filter(lambda x: x > 2)
|
|
109
|
+
.map(lambda x: x * 10)
|
|
110
|
+
.collect()
|
|
111
|
+
)
|
|
112
|
+
# [30, 40, 50]
|
|
113
|
+
|
|
114
|
+
# Word count
|
|
115
|
+
lines = ["hello world hello", "hello vega"]
|
|
116
|
+
counts = (
|
|
117
|
+
sc.parallelize(lines)
|
|
118
|
+
.flat_map(lambda line: line.split())
|
|
119
|
+
.map(lambda word: (word, 1))
|
|
120
|
+
.reduce_by_key(lambda a, b: a + b)
|
|
121
|
+
.collect()
|
|
122
|
+
)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Full API: [Python RDD guide](docs/python-guide.md).
|
|
126
|
+
|
|
127
|
+
## Running on a cluster (C++)
|
|
128
|
+
|
|
129
|
+
`vega_worker` runs **equal peers** (no special driver role). PyVega does not connect
|
|
130
|
+
to cluster peers yet; use this for C++ multi-node examples.
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
# Terminal 1
|
|
134
|
+
./build/vega_worker --bind 127.0.0.1:9001 --id peer-1
|
|
135
|
+
|
|
136
|
+
# Terminal 2
|
|
137
|
+
./build/vega_worker --bind 127.0.0.1:9002 --id peer-2 --seeds 127.0.0.1:9001
|
|
138
|
+
|
|
139
|
+
# Terminal 3 — submitter (same binary, not a "driver")
|
|
140
|
+
./build/examples/cluster_word_count --bind 127.0.0.1:9003 --id peer-3 \
|
|
141
|
+
--seeds 127.0.0.1:9001 --expect-peers 3
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Single-node demo: `./build/examples/cluster_word_count --local`
|
|
145
|
+
|
|
146
|
+
| Flag | Description |
|
|
147
|
+
|------|-------------|
|
|
148
|
+
| `--bind HOST:PORT` | Listen address |
|
|
149
|
+
| `--seeds H1:P1,H2:P2` | Gossip seed peers |
|
|
150
|
+
| `--id NAME` | Node identifier |
|
|
151
|
+
| `--expect-peers N` | Wait for N peers before running (examples) |
|
|
152
|
+
|
|
153
|
+
Details: [Server, workers, and clients](docs/server-and-workers.md).
|
|
154
|
+
|
|
155
|
+
## C++ API
|
|
156
|
+
|
|
157
|
+
```cpp
|
|
158
|
+
#include <vega/vega.hpp>
|
|
159
|
+
|
|
160
|
+
int main() {
|
|
161
|
+
vega::SparkContext ctx(4);
|
|
162
|
+
auto rdd = ctx.parallelize(std::vector<int>{1, 2, 3, 4, 5}, 2);
|
|
163
|
+
auto result = rdd
|
|
164
|
+
.filter([](const int& x) { return x > 2; })
|
|
165
|
+
.map([](const int& x) { return x * 10; })
|
|
166
|
+
.collect();
|
|
167
|
+
}
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
### Available operations
|
|
171
|
+
|
|
172
|
+
| Transformation | Python | C++ |
|
|
173
|
+
|----------------|--------|-----|
|
|
174
|
+
| `map` | `rdd.map(func)` | `rdd.map(func)` |
|
|
175
|
+
| `filter` | `rdd.filter(func)` | `rdd.filter(pred)` |
|
|
176
|
+
| `flat_map` | `rdd.flat_map(func)` | `rdd.flatMap(func)` |
|
|
177
|
+
| `reduceByKey` | `rdd.reduce_by_key(func)` | `rdd.reduceByKey(func)` |
|
|
178
|
+
| `groupByKey` | `rdd.group_by_key()` | `rdd.groupByKey()` |
|
|
179
|
+
|
|
180
|
+
| Action | Python | C++ |
|
|
181
|
+
|--------|--------|-----|
|
|
182
|
+
| `collect` | `rdd.collect()` | `rdd.collect()` |
|
|
183
|
+
| `count` | `rdd.count()` | `rdd.count()` |
|
|
184
|
+
| `reduce` | `rdd.reduce(func)` | `rdd.reduce(func)` |
|
|
185
|
+
| `foreach` | `rdd.foreach(func)` | `rdd.foreach(func)` |
|
|
186
|
+
|
|
187
|
+
## Project structure
|
|
188
|
+
|
|
189
|
+
```
|
|
190
|
+
vega/
|
|
191
|
+
docs/ User guides (start at docs/README.md)
|
|
192
|
+
include/vega/ C++ header-only library
|
|
193
|
+
core/ RDD, SparkContext, shuffle
|
|
194
|
+
actors/ JobActor, WorkerActor, messages
|
|
195
|
+
execution/ DAG builder, stages
|
|
196
|
+
cluster/ P2P cluster, gossip
|
|
197
|
+
network/ TCP framing, routing
|
|
198
|
+
src/
|
|
199
|
+
vega_server.cpp TCP session server (PyVega backend)
|
|
200
|
+
server_session.cpp Session RPC + embedded Python UDFs
|
|
201
|
+
vega_worker.cpp Cluster peer daemon
|
|
202
|
+
python/pyvega/ PyVega client package
|
|
203
|
+
scripts/
|
|
204
|
+
pyspark_compat/ PySpark RDD API shim
|
|
205
|
+
run_pyspark_tests.py Upstream Spark RDD tests vs PyVega
|
|
206
|
+
third_party/spark/ Vendored Spark test sources
|
|
207
|
+
examples/ word_count, cluster_word_count, benchmarks
|
|
208
|
+
tests/ C++ unit tests + Spark RDD suite ports
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
## Spark compatibility testing
|
|
212
|
+
|
|
213
|
+
```bash
|
|
214
|
+
# C++ Spark RDD logic ports
|
|
215
|
+
cmake --build build --target test_spark_rdd_suite
|
|
216
|
+
cd build && ctest -R SparkRDDSuite --output-on-failure
|
|
217
|
+
|
|
218
|
+
# Upstream PySpark RDD tests (50/54 pass; 4 require JVM)
|
|
219
|
+
VEGA_SERVER=build/vega_server \
|
|
220
|
+
python scripts/run_pyspark_tests.py --filter pyspark.tests.test_rdd
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
See [PySpark compatibility](docs/pyspark-compat.md).
|
|
224
|
+
|
|
225
|
+
## Execution flow
|
|
226
|
+
|
|
227
|
+
### PyVega (session server)
|
|
228
|
+
|
|
229
|
+
1. Client builds an RDD chain lazily (`map`, `filter`, …)
|
|
230
|
+
2. First action sends fused stages over TCP
|
|
231
|
+
3. Server materializes lineage and submits a job to JobActor
|
|
232
|
+
4. WorkerActors compute partitions (native C++ or embedded Python)
|
|
233
|
+
5. Shuffle stages (`reduceByKey`, `groupByKey`) run as multi-stage DAGs
|
|
234
|
+
6. Results return to the Python client
|
|
235
|
+
|
|
236
|
+
### Multi-node C++ cluster
|
|
237
|
+
|
|
238
|
+
1. Peers start and discover each other via gossip
|
|
239
|
+
2. Submitter distributes partition data and task assignments over TCP
|
|
240
|
+
3. Peers execute tasks and return serialized results
|
|
241
|
+
|
|
242
|
+
## Contributing
|
|
243
|
+
|
|
244
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for build/test instructions and PR
|
|
245
|
+
guidelines, and our [Code of Conduct](CODE_OF_CONDUCT.md). Report security
|
|
246
|
+
issues privately per [SECURITY.md](SECURITY.md).
|
|
247
|
+
|
|
248
|
+
## License
|
|
249
|
+
|
|
250
|
+
Licensed under the [MIT License](LICENSE).
|
|
251
|
+
|
|
252
|
+
Vega is **not affiliated with the Apache Software Foundation**. "Apache Spark"
|
|
253
|
+
and "PySpark" are trademarks of the ASF; Vega offers a Spark-like API for
|
|
254
|
+
familiarity and compatibility testing only. See [NOTICE](NOTICE) for
|
|
255
|
+
third-party attributions (Apache Spark test sources, pybind11, GoogleTest,
|
|
256
|
+
cloudpickle).
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=45"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "pyvega"
|
|
7
|
+
version = "0.5.0"
|
|
8
|
+
description = "Python client for the Vega C++ distributed data processing engine"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = {text = "MIT"}
|
|
12
|
+
authors = [{name = "The Vega Authors"}]
|
|
13
|
+
keywords = ["distributed", "rdd", "spark", "data-processing", "actors"]
|
|
14
|
+
dependencies = ["cloudpickle>=3.0"]
|
|
15
|
+
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 3 - Alpha",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: C++",
|
|
22
|
+
"Topic :: System :: Distributed Computing",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[project.optional-dependencies]
|
|
26
|
+
test = ["pytest>=7.0"]
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://github.com/sphulinga/vega"
|
|
30
|
+
Documentation = "https://github.com/sphulinga/vega/tree/main/docs"
|
|
31
|
+
Repository = "https://github.com/sphulinga/vega"
|
|
32
|
+
Issues = "https://github.com/sphulinga/vega/issues"
|
|
33
|
+
|
|
34
|
+
[project.scripts]
|
|
35
|
+
vega-server = "pyvega._server_cli:main"
|