pyvega 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pyvega-0.5.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 The Vega Authors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
pyvega-0.5.0/NOTICE ADDED
@@ -0,0 +1,42 @@
1
+ Vega
2
+ Copyright (c) 2026 The Vega Authors
3
+
4
+ This product is licensed under the MIT License (see LICENSE).
5
+
6
+ Vega is not affiliated with, endorsed by, or sponsored by the Apache Software
7
+ Foundation or the Apache Spark project. "Apache Spark", "Spark", and "PySpark"
8
+ are trademarks of the Apache Software Foundation. Vega provides a Spark-like
9
+ API for familiarity and compatibility testing only.
10
+
11
+ --------------------------------------------------------------------------------
12
+ Third-party components
13
+ --------------------------------------------------------------------------------
14
+
15
+ This repository's tooling fetches and/or builds against the following
16
+ third-party software. These components are NOT part of Vega and retain their
17
+ own licenses.
18
+
19
+ 1. Apache Spark
20
+ - Used by: PySpark compatibility tests under scripts/pyspark_compat and the
21
+ vendored/fetched sources under third_party/spark (fetched via
22
+ scripts/fetch_spark_tests.sh; not required to build or run Vega).
23
+ - License: Apache License 2.0
24
+ - https://github.com/apache/spark
25
+ - When distributing any portion of Apache Spark sources, you must comply
26
+ with the Apache License 2.0, including its NOTICE file
27
+ (third_party/spark/LICENSE and third_party/spark/NOTICE).
28
+
29
+ 2. pybind11
30
+ - Used by: vega_server (embedded Python).
31
+ - License: BSD-3-Clause
32
+ - https://github.com/pybind/pybind11
33
+
34
+ 3. GoogleTest
35
+ - Used by: C++ test suite.
36
+ - License: BSD-3-Clause
37
+ - https://github.com/google/googletest
38
+
39
+ 4. cloudpickle
40
+ - Used by: PyVega client (runtime dependency).
41
+ - License: BSD-3-Clause
42
+ - https://github.com/cloudpipe/cloudpickle
pyvega-0.5.0/PKG-INFO ADDED
@@ -0,0 +1,282 @@
1
+ Metadata-Version: 2.4
2
+ Name: pyvega
3
+ Version: 0.5.0
4
+ Summary: Python client for the Vega C++ distributed data processing engine
5
+ Author: The Vega Authors
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/sphulinga/vega
8
+ Project-URL: Documentation, https://github.com/sphulinga/vega/tree/main/docs
9
+ Project-URL: Repository, https://github.com/sphulinga/vega
10
+ Project-URL: Issues, https://github.com/sphulinga/vega/issues
11
+ Keywords: distributed,rdd,spark,data-processing,actors
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: C++
17
+ Classifier: Topic :: System :: Distributed Computing
18
+ Requires-Python: >=3.8
19
+ Description-Content-Type: text/markdown
20
+ License-File: LICENSE
21
+ License-File: NOTICE
22
+ Requires-Dist: cloudpickle>=3.0
23
+ Provides-Extra: test
24
+ Requires-Dist: pytest>=7.0; extra == "test"
25
+ Dynamic: license-file
26
+
27
+ # Vega
28
+
29
+ An in-memory distributed data processing engine written in C++17. Vega
30
+ provides a Spark-like RDD API with lazy evaluation, but replaces Spark's
31
+ centralized driver/scheduler with a **decentralized actor model** where
32
+ workers coordinate through message passing.
33
+
34
+ **PyVega** is the Python client: it connects to a C++ **session server** over TCP,
35
+ records transformations lazily, and executes them on an embedded actor runtime with
36
+ Python UDF support.
37
+
38
+ ## Documentation
39
+
40
+ | Guide | Description |
41
+ |-------|-------------|
42
+ | [**Docs index**](docs/README.md) | Start here |
43
+ | [Getting started](docs/getting-started.md) | Build, install, first program |
44
+ | [Server, workers, and clients](docs/server-and-workers.md) | `vega_server`, worker threads, cluster peers |
45
+ | [Python RDD guide](docs/python-guide.md) | PyVega API reference with examples |
46
+ | [DataFrame-style operations](docs/dataframe-operations.md) | SQL-like patterns using RDDs |
47
+ | [PySpark compatibility](docs/pyspark-compat.md) | Run PySpark RDD code and upstream tests |
48
+
49
+ ## Quick start (Python)
50
+
51
+ ```bash
52
+ # Build session server
53
+ cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DVEGA_BUILD_SERVER=ON
54
+ cmake --build build --target vega_server -j$(nproc)
55
+
56
+ # Install client
57
+ python -m venv .venv && source .venv/bin/activate
58
+ pip install -e ".[test]"
59
+ export VEGA_SERVER="$PWD/build/vega_server"
60
+ ```
61
+
62
+ ```python
63
+ from pyvega import SparkContext
64
+
65
+ with SparkContext(4) as sc:
66
+ print(sc.range(0, 10).map(lambda x: x * 2).collect())
67
+ # [0, 2, 4, 6, 8, 10, 12, 14, 16, 18]
68
+ ```
69
+
70
+ **Manual server:** `vega-server --bind 127.0.0.1:9500` then
71
+ `SparkContext(remote="127.0.0.1:9500")`. See [server and clients](docs/server-and-workers.md).
72
+
73
+ > **Note:** PyVega provides **RDD** APIs, not Spark SQL DataFrames. See
74
+ > [DataFrame-style operations](docs/dataframe-operations.md) for RDD equivalents.
75
+
76
+ ## Architecture
77
+
78
+ ```
79
+ User Code (Python or C++)
80
+ |
81
+ SparkContext / Cluster
82
+ |
83
+ +-----------+
84
+ | JobActor | (builds DAG, distributes tasks)
85
+ +-----------+
86
+ / | \
87
+ +------+ +------+ +------+
88
+ |Worker| |Worker| |Worker| (execute partitions in parallel)
89
+ +------+ +------+ +------+
90
+ \ | /
91
+ +-------------------------+
92
+ | Actor System | (thread pool, message routing)
93
+ +-------------------------+
94
+ | Transport Layer | (Local / TCP + Gossip)
95
+ +-------------------------+
96
+ ```
97
+
98
+ For PyVega, the actor system runs **inside `vega_server`**. The Python process is a thin
99
+ RPC client; worker threads are not separate OS processes.
100
+
101
+ **Key difference from Spark:** there is no central scheduler in cluster mode. Peers
102
+ discover each other via gossip and coordinate execution directly. In single-node /
103
+ session-server mode, the JobActor still orchestrates stages but all workers are local
104
+ threads.
105
+
106
+ ### Actor framework
107
+
108
+ Vega includes a lightweight actor framework inspired by
109
+ [rotor](https://github.com/basiliscos/cpp-rotor):
110
+
111
+ - **Message-based communication** — typed messages dispatched to registered handlers
112
+ - **Thread pool execution** — actors process messages in parallel across worker threads
113
+ - **Sequential per-actor** — each actor processes one message at a time
114
+ - **Python GIL integration** — worker threads acquire the GIL when executing Python UDFs
115
+
116
+ ## Build (C++)
117
+
118
+ ```bash
119
+ cmake -B build -DCMAKE_BUILD_TYPE=Release
120
+ cmake --build build -j$(nproc)
121
+ ```
122
+
123
+ Targets include `vega_server`, `vega_worker`, examples, and C++ tests.
124
+
125
+ ## Python API (PyVega)
126
+
127
+ ```python
128
+ from pyvega import SparkContext
129
+
130
+ with SparkContext(4) as sc:
131
+ rdd = sc.parallelize([1, 2, 3, 4, 5], num_partitions=2)
132
+
133
+ result = (
134
+ rdd.filter(lambda x: x > 2)
135
+ .map(lambda x: x * 10)
136
+ .collect()
137
+ )
138
+ # [30, 40, 50]
139
+
140
+ # Word count
141
+ lines = ["hello world hello", "hello vega"]
142
+ counts = (
143
+ sc.parallelize(lines)
144
+ .flat_map(lambda line: line.split())
145
+ .map(lambda word: (word, 1))
146
+ .reduce_by_key(lambda a, b: a + b)
147
+ .collect()
148
+ )
149
+ ```
150
+
151
+ Full API: [Python RDD guide](docs/python-guide.md).
152
+
153
+ ## Running on a cluster (C++)
154
+
155
+ `vega_worker` runs **equal peers** (no special driver role). PyVega does not connect
156
+ to cluster peers yet; use this for C++ multi-node examples.
157
+
158
+ ```bash
159
+ # Terminal 1
160
+ ./build/vega_worker --bind 127.0.0.1:9001 --id peer-1
161
+
162
+ # Terminal 2
163
+ ./build/vega_worker --bind 127.0.0.1:9002 --id peer-2 --seeds 127.0.0.1:9001
164
+
165
+ # Terminal 3 — submitter (same binary, not a "driver")
166
+ ./build/examples/cluster_word_count --bind 127.0.0.1:9003 --id peer-3 \
167
+ --seeds 127.0.0.1:9001 --expect-peers 3
168
+ ```
169
+
170
+ Single-node demo: `./build/examples/cluster_word_count --local`
171
+
172
+ | Flag | Description |
173
+ |------|-------------|
174
+ | `--bind HOST:PORT` | Listen address |
175
+ | `--seeds H1:P1,H2:P2` | Gossip seed peers |
176
+ | `--id NAME` | Node identifier |
177
+ | `--expect-peers N` | Wait for N peers before running (examples) |
178
+
179
+ Details: [Server, workers, and clients](docs/server-and-workers.md).
180
+
181
+ ## C++ API
182
+
183
+ ```cpp
184
+ #include <vega/vega.hpp>
185
+
186
+ int main() {
187
+ vega::SparkContext ctx(4);
188
+ auto rdd = ctx.parallelize(std::vector<int>{1, 2, 3, 4, 5}, 2);
189
+ auto result = rdd
190
+ .filter([](const int& x) { return x > 2; })
191
+ .map([](const int& x) { return x * 10; })
192
+ .collect();
193
+ }
194
+ ```
195
+
196
+ ### Available operations
197
+
198
+ | Transformation | Python | C++ |
199
+ |----------------|--------|-----|
200
+ | `map` | `rdd.map(func)` | `rdd.map(func)` |
201
+ | `filter` | `rdd.filter(func)` | `rdd.filter(pred)` |
202
+ | `flat_map` | `rdd.flat_map(func)` | `rdd.flatMap(func)` |
203
+ | `reduceByKey` | `rdd.reduce_by_key(func)` | `rdd.reduceByKey(func)` |
204
+ | `groupByKey` | `rdd.group_by_key()` | `rdd.groupByKey()` |
205
+
206
+ | Action | Python | C++ |
207
+ |--------|--------|-----|
208
+ | `collect` | `rdd.collect()` | `rdd.collect()` |
209
+ | `count` | `rdd.count()` | `rdd.count()` |
210
+ | `reduce` | `rdd.reduce(func)` | `rdd.reduce(func)` |
211
+ | `foreach` | `rdd.foreach(func)` | `rdd.foreach(func)` |
212
+
213
+ ## Project structure
214
+
215
+ ```
216
+ vega/
217
+ docs/ User guides (start at docs/README.md)
218
+ include/vega/ C++ header-only library
219
+ core/ RDD, SparkContext, shuffle
220
+ actors/ JobActor, WorkerActor, messages
221
+ execution/ DAG builder, stages
222
+ cluster/ P2P cluster, gossip
223
+ network/ TCP framing, routing
224
+ src/
225
+ vega_server.cpp TCP session server (PyVega backend)
226
+ server_session.cpp Session RPC + embedded Python UDFs
227
+ vega_worker.cpp Cluster peer daemon
228
+ python/pyvega/ PyVega client package
229
+ scripts/
230
+ pyspark_compat/ PySpark RDD API shim
231
+ run_pyspark_tests.py Upstream Spark RDD tests vs PyVega
232
+ third_party/spark/ Vendored Spark test sources
233
+ examples/ word_count, cluster_word_count, benchmarks
234
+ tests/ C++ unit tests + Spark RDD suite ports
235
+ ```
236
+
237
+ ## Spark compatibility testing
238
+
239
+ ```bash
240
+ # C++ Spark RDD logic ports
241
+ cmake --build build --target test_spark_rdd_suite
242
+ cd build && ctest -R SparkRDDSuite --output-on-failure
243
+
244
+ # Upstream PySpark RDD tests (50/54 pass; 4 require JVM)
245
+ VEGA_SERVER=build/vega_server \
246
+ python scripts/run_pyspark_tests.py --filter pyspark.tests.test_rdd
247
+ ```
248
+
249
+ See [PySpark compatibility](docs/pyspark-compat.md).
250
+
251
+ ## Execution flow
252
+
253
+ ### PyVega (session server)
254
+
255
+ 1. Client builds an RDD chain lazily (`map`, `filter`, …)
256
+ 2. First action sends fused stages over TCP
257
+ 3. Server materializes lineage and submits a job to JobActor
258
+ 4. WorkerActors compute partitions (native C++ or embedded Python)
259
+ 5. Shuffle stages (`reduceByKey`, `groupByKey`) run as multi-stage DAGs
260
+ 6. Results return to the Python client
261
+
262
+ ### Multi-node C++ cluster
263
+
264
+ 1. Peers start and discover each other via gossip
265
+ 2. Submitter distributes partition data and task assignments over TCP
266
+ 3. Peers execute tasks and return serialized results
267
+
268
+ ## Contributing
269
+
270
+ See [CONTRIBUTING.md](CONTRIBUTING.md) for build/test instructions and PR
271
+ guidelines, and our [Code of Conduct](CODE_OF_CONDUCT.md). Report security
272
+ issues privately per [SECURITY.md](SECURITY.md).
273
+
274
+ ## License
275
+
276
+ Licensed under the [MIT License](LICENSE).
277
+
278
+ Vega is **not affiliated with the Apache Software Foundation**. "Apache Spark"
279
+ and "PySpark" are trademarks of the ASF; Vega offers a Spark-like API for
280
+ familiarity and compatibility testing only. See [NOTICE](NOTICE) for
281
+ third-party attributions (Apache Spark test sources, pybind11, GoogleTest,
282
+ cloudpickle).
pyvega-0.5.0/README.md ADDED
@@ -0,0 +1,256 @@
1
+ # Vega
2
+
3
+ An in-memory distributed data processing engine written in C++17. Vega
4
+ provides a Spark-like RDD API with lazy evaluation, but replaces Spark's
5
+ centralized driver/scheduler with a **decentralized actor model** where
6
+ workers coordinate through message passing.
7
+
8
+ **PyVega** is the Python client: it connects to a C++ **session server** over TCP,
9
+ records transformations lazily, and executes them on an embedded actor runtime with
10
+ Python UDF support.
11
+
12
+ ## Documentation
13
+
14
+ | Guide | Description |
15
+ |-------|-------------|
16
+ | [**Docs index**](docs/README.md) | Start here |
17
+ | [Getting started](docs/getting-started.md) | Build, install, first program |
18
+ | [Server, workers, and clients](docs/server-and-workers.md) | `vega_server`, worker threads, cluster peers |
19
+ | [Python RDD guide](docs/python-guide.md) | PyVega API reference with examples |
20
+ | [DataFrame-style operations](docs/dataframe-operations.md) | SQL-like patterns using RDDs |
21
+ | [PySpark compatibility](docs/pyspark-compat.md) | Run PySpark RDD code and upstream tests |
22
+
23
+ ## Quick start (Python)
24
+
25
+ ```bash
26
+ # Build session server
27
+ cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DVEGA_BUILD_SERVER=ON
28
+ cmake --build build --target vega_server -j$(nproc)
29
+
30
+ # Install client
31
+ python -m venv .venv && source .venv/bin/activate
32
+ pip install -e ".[test]"
33
+ export VEGA_SERVER="$PWD/build/vega_server"
34
+ ```
35
+
36
+ ```python
37
+ from pyvega import SparkContext
38
+
39
+ with SparkContext(4) as sc:
40
+ print(sc.range(0, 10).map(lambda x: x * 2).collect())
41
+ # [0, 2, 4, 6, 8, 10, 12, 14, 16, 18]
42
+ ```
43
+
44
+ **Manual server:** `vega-server --bind 127.0.0.1:9500` then
45
+ `SparkContext(remote="127.0.0.1:9500")`. See [server and clients](docs/server-and-workers.md).
46
+
47
+ > **Note:** PyVega provides **RDD** APIs, not Spark SQL DataFrames. See
48
+ > [DataFrame-style operations](docs/dataframe-operations.md) for RDD equivalents.
49
+
50
+ ## Architecture
51
+
52
+ ```
53
+ User Code (Python or C++)
54
+ |
55
+ SparkContext / Cluster
56
+ |
57
+ +-----------+
58
+ | JobActor | (builds DAG, distributes tasks)
59
+ +-----------+
60
+ / | \
61
+ +------+ +------+ +------+
62
+ |Worker| |Worker| |Worker| (execute partitions in parallel)
63
+ +------+ +------+ +------+
64
+ \ | /
65
+ +-------------------------+
66
+ | Actor System | (thread pool, message routing)
67
+ +-------------------------+
68
+ | Transport Layer | (Local / TCP + Gossip)
69
+ +-------------------------+
70
+ ```
71
+
72
+ For PyVega, the actor system runs **inside `vega_server`**. The Python process is a thin
73
+ RPC client; worker threads are not separate OS processes.
74
+
75
+ **Key difference from Spark:** there is no central scheduler in cluster mode. Peers
76
+ discover each other via gossip and coordinate execution directly. In single-node /
77
+ session-server mode, the JobActor still orchestrates stages but all workers are local
78
+ threads.
79
+
80
+ ### Actor framework
81
+
82
+ Vega includes a lightweight actor framework inspired by
83
+ [rotor](https://github.com/basiliscos/cpp-rotor):
84
+
85
+ - **Message-based communication** — typed messages dispatched to registered handlers
86
+ - **Thread pool execution** — actors process messages in parallel across worker threads
87
+ - **Sequential per-actor** — each actor processes one message at a time
88
+ - **Python GIL integration** — worker threads acquire the GIL when executing Python UDFs
89
+
90
+ ## Build (C++)
91
+
92
+ ```bash
93
+ cmake -B build -DCMAKE_BUILD_TYPE=Release
94
+ cmake --build build -j$(nproc)
95
+ ```
96
+
97
+ Targets include `vega_server`, `vega_worker`, examples, and C++ tests.
98
+
99
+ ## Python API (PyVega)
100
+
101
+ ```python
102
+ from pyvega import SparkContext
103
+
104
+ with SparkContext(4) as sc:
105
+ rdd = sc.parallelize([1, 2, 3, 4, 5], num_partitions=2)
106
+
107
+ result = (
108
+ rdd.filter(lambda x: x > 2)
109
+ .map(lambda x: x * 10)
110
+ .collect()
111
+ )
112
+ # [30, 40, 50]
113
+
114
+ # Word count
115
+ lines = ["hello world hello", "hello vega"]
116
+ counts = (
117
+ sc.parallelize(lines)
118
+ .flat_map(lambda line: line.split())
119
+ .map(lambda word: (word, 1))
120
+ .reduce_by_key(lambda a, b: a + b)
121
+ .collect()
122
+ )
123
+ ```
124
+
125
+ Full API: [Python RDD guide](docs/python-guide.md).
126
+
127
+ ## Running on a cluster (C++)
128
+
129
+ `vega_worker` runs **equal peers** (no special driver role). PyVega does not connect
130
+ to cluster peers yet; use this for C++ multi-node examples.
131
+
132
+ ```bash
133
+ # Terminal 1
134
+ ./build/vega_worker --bind 127.0.0.1:9001 --id peer-1
135
+
136
+ # Terminal 2
137
+ ./build/vega_worker --bind 127.0.0.1:9002 --id peer-2 --seeds 127.0.0.1:9001
138
+
139
+ # Terminal 3 — submitter (same binary, not a "driver")
140
+ ./build/examples/cluster_word_count --bind 127.0.0.1:9003 --id peer-3 \
141
+ --seeds 127.0.0.1:9001 --expect-peers 3
142
+ ```
143
+
144
+ Single-node demo: `./build/examples/cluster_word_count --local`
145
+
146
+ | Flag | Description |
147
+ |------|-------------|
148
+ | `--bind HOST:PORT` | Listen address |
149
+ | `--seeds H1:P1,H2:P2` | Gossip seed peers |
150
+ | `--id NAME` | Node identifier |
151
+ | `--expect-peers N` | Wait for N peers before running (examples) |
152
+
153
+ Details: [Server, workers, and clients](docs/server-and-workers.md).
154
+
155
+ ## C++ API
156
+
157
+ ```cpp
158
+ #include <vega/vega.hpp>
159
+
160
+ int main() {
161
+ vega::SparkContext ctx(4);
162
+ auto rdd = ctx.parallelize(std::vector<int>{1, 2, 3, 4, 5}, 2);
163
+ auto result = rdd
164
+ .filter([](const int& x) { return x > 2; })
165
+ .map([](const int& x) { return x * 10; })
166
+ .collect();
167
+ }
168
+ ```
169
+
170
+ ### Available operations
171
+
172
+ | Transformation | Python | C++ |
173
+ |----------------|--------|-----|
174
+ | `map` | `rdd.map(func)` | `rdd.map(func)` |
175
+ | `filter` | `rdd.filter(func)` | `rdd.filter(pred)` |
176
+ | `flat_map` | `rdd.flat_map(func)` | `rdd.flatMap(func)` |
177
+ | `reduceByKey` | `rdd.reduce_by_key(func)` | `rdd.reduceByKey(func)` |
178
+ | `groupByKey` | `rdd.group_by_key()` | `rdd.groupByKey()` |
179
+
180
+ | Action | Python | C++ |
181
+ |--------|--------|-----|
182
+ | `collect` | `rdd.collect()` | `rdd.collect()` |
183
+ | `count` | `rdd.count()` | `rdd.count()` |
184
+ | `reduce` | `rdd.reduce(func)` | `rdd.reduce(func)` |
185
+ | `foreach` | `rdd.foreach(func)` | `rdd.foreach(func)` |
186
+
187
+ ## Project structure
188
+
189
+ ```
190
+ vega/
191
+ docs/ User guides (start at docs/README.md)
192
+ include/vega/ C++ header-only library
193
+ core/ RDD, SparkContext, shuffle
194
+ actors/ JobActor, WorkerActor, messages
195
+ execution/ DAG builder, stages
196
+ cluster/ P2P cluster, gossip
197
+ network/ TCP framing, routing
198
+ src/
199
+ vega_server.cpp TCP session server (PyVega backend)
200
+ server_session.cpp Session RPC + embedded Python UDFs
201
+ vega_worker.cpp Cluster peer daemon
202
+ python/pyvega/ PyVega client package
203
+ scripts/
204
+ pyspark_compat/ PySpark RDD API shim
205
+ run_pyspark_tests.py Upstream Spark RDD tests vs PyVega
206
+ third_party/spark/ Vendored Spark test sources
207
+ examples/ word_count, cluster_word_count, benchmarks
208
+ tests/ C++ unit tests + Spark RDD suite ports
209
+ ```
210
+
211
+ ## Spark compatibility testing
212
+
213
+ ```bash
214
+ # C++ Spark RDD logic ports
215
+ cmake --build build --target test_spark_rdd_suite
216
+ cd build && ctest -R SparkRDDSuite --output-on-failure
217
+
218
+ # Upstream PySpark RDD tests (50/54 pass; 4 require JVM)
219
+ VEGA_SERVER=build/vega_server \
220
+ python scripts/run_pyspark_tests.py --filter pyspark.tests.test_rdd
221
+ ```
222
+
223
+ See [PySpark compatibility](docs/pyspark-compat.md).
224
+
225
+ ## Execution flow
226
+
227
+ ### PyVega (session server)
228
+
229
+ 1. Client builds an RDD chain lazily (`map`, `filter`, …)
230
+ 2. First action sends fused stages over TCP
231
+ 3. Server materializes lineage and submits a job to JobActor
232
+ 4. WorkerActors compute partitions (native C++ or embedded Python)
233
+ 5. Shuffle stages (`reduceByKey`, `groupByKey`) run as multi-stage DAGs
234
+ 6. Results return to the Python client
235
+
236
+ ### Multi-node C++ cluster
237
+
238
+ 1. Peers start and discover each other via gossip
239
+ 2. Submitter distributes partition data and task assignments over TCP
240
+ 3. Peers execute tasks and return serialized results
241
+
242
+ ## Contributing
243
+
244
+ See [CONTRIBUTING.md](CONTRIBUTING.md) for build/test instructions and PR
245
+ guidelines, and our [Code of Conduct](CODE_OF_CONDUCT.md). Report security
246
+ issues privately per [SECURITY.md](SECURITY.md).
247
+
248
+ ## License
249
+
250
+ Licensed under the [MIT License](LICENSE).
251
+
252
+ Vega is **not affiliated with the Apache Software Foundation**. "Apache Spark"
253
+ and "PySpark" are trademarks of the ASF; Vega offers a Spark-like API for
254
+ familiarity and compatibility testing only. See [NOTICE](NOTICE) for
255
+ third-party attributions (Apache Spark test sources, pybind11, GoogleTest,
256
+ cloudpickle).
@@ -0,0 +1,35 @@
1
+ [build-system]
2
+ requires = ["setuptools>=45"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "pyvega"
7
+ version = "0.5.0"
8
+ description = "Python client for the Vega C++ distributed data processing engine"
9
+ readme = "README.md"
10
+ requires-python = ">=3.8"
11
+ license = {text = "MIT"}
12
+ authors = [{name = "The Vega Authors"}]
13
+ keywords = ["distributed", "rdd", "spark", "data-processing", "actors"]
14
+ dependencies = ["cloudpickle>=3.0"]
15
+
16
+ classifiers = [
17
+ "Development Status :: 3 - Alpha",
18
+ "Intended Audience :: Developers",
19
+ "License :: OSI Approved :: MIT License",
20
+ "Programming Language :: Python :: 3",
21
+ "Programming Language :: C++",
22
+ "Topic :: System :: Distributed Computing",
23
+ ]
24
+
25
+ [project.optional-dependencies]
26
+ test = ["pytest>=7.0"]
27
+
28
+ [project.urls]
29
+ Homepage = "https://github.com/sphulinga/vega"
30
+ Documentation = "https://github.com/sphulinga/vega/tree/main/docs"
31
+ Repository = "https://github.com/sphulinga/vega"
32
+ Issues = "https://github.com/sphulinga/vega/issues"
33
+
34
+ [project.scripts]
35
+ vega-server = "pyvega._server_cli:main"