mock-pyspark 0.0.1.dev20260804__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mock_pyspark-0.0.1.dev20260804/PKG-INFO +142 -0
- mock_pyspark-0.0.1.dev20260804/README.md +113 -0
- mock_pyspark-0.0.1.dev20260804/mock_pyspark.egg-info/PKG-INFO +142 -0
- mock_pyspark-0.0.1.dev20260804/mock_pyspark.egg-info/SOURCES.txt +143 -0
- mock_pyspark-0.0.1.dev20260804/mock_pyspark.egg-info/dependency_links.txt +1 -0
- mock_pyspark-0.0.1.dev20260804/mock_pyspark.egg-info/requires.txt +13 -0
- mock_pyspark-0.0.1.dev20260804/mock_pyspark.egg-info/top_level.txt +1 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/__init__.py +33 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/_aliasing.py +105 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/_internal/__init__.py +0 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/_internal/config.py +27 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/_internal/engine.py +5 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/_internal/version_detect.py +118 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/_internal/version_guard.py +37 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/delta/__init__.py +15 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/delta/pip.py +9 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/delta/pip_utils.py +23 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/delta/tables.py +303 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/errors/__init__.py +47 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/ml/__init__.py +26 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/pandas/__init__.py +123 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/pytest_plugin.py +35 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/rdd.py +28 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/__init__.py +38 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/_map_utils.py +38 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/catalog.py +349 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/column.py +861 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/dataframe.py +2948 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/__init__.py +228 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/aggregates.py +783 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/arrays.py +1014 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/bits.py +239 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/core.py +358 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/crypto.py +57 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/csvs.py +152 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/dates.py +1941 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/generators.py +68 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/geospatial.py +125 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/hashes.py +161 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/jsons.py +243 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/maps.py +424 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/maths.py +691 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/partitioning.py +29 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/predicates.py +261 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/strings.py +1754 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/udfs.py +77 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/urls.py +176 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/variants.py +391 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/windows.py +54 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/xmls.py +182 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions.py +3 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions_all.py +16 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/__init__.py +85 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/_arrow_types.py +174 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/avro.py +130 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/base.py +358 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/binary_file.py +49 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/csv.py +129 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/delta.py +102 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/json.py +122 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/orc.py +61 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/parquet.py +51 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/text.py +74 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/readwriter.py +917 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/row.py +76 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/session.py +354 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/sql_dispatcher.py +1180 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/streaming/__init__.py +15 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/streaming/readwriter.py +13 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/types.py +757 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/utils.py +33 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/window.py +138 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/streaming/__init__.py +15 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/testing/__init__.py +19 -0
- mock_pyspark-0.0.1.dev20260804/mockpyspark/testing/utils.py +150 -0
- mock_pyspark-0.0.1.dev20260804/pyproject.toml +51 -0
- mock_pyspark-0.0.1.dev20260804/setup.cfg +4 -0
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mock-pyspark
|
|
3
|
+
Version: 0.0.1.dev20260804
|
|
4
|
+
Summary: Drop-in mock for Apache PySpark 4.x — Arrow-backed, no JVM required
|
|
5
|
+
Author-email: Peter Dowdy <peter.dowdy@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/PeterDowdy/mock-pyspark
|
|
8
|
+
Project-URL: Repository, https://github.com/PeterDowdy/mock-pyspark
|
|
9
|
+
Project-URL: Issues, https://github.com/PeterDowdy/mock-pyspark/issues
|
|
10
|
+
Keywords: pyspark,mock,testing,arrow,duckdb
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Topic :: Software Development :: Testing
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
Requires-Dist: pyarrow>=14
|
|
20
|
+
Requires-Dist: duckdb>=1.0
|
|
21
|
+
Requires-Dist: pandas>=2.0
|
|
22
|
+
Requires-Dist: numpy>=1.24
|
|
23
|
+
Provides-Extra: avro
|
|
24
|
+
Requires-Dist: fastavro>=1.9; extra == "avro"
|
|
25
|
+
Provides-Extra: delta
|
|
26
|
+
Requires-Dist: deltalake>=1.0; extra == "delta"
|
|
27
|
+
Provides-Extra: all
|
|
28
|
+
Requires-Dist: mock-pyspark[avro,delta]; extra == "all"
|
|
29
|
+
|
|
30
|
+
# mockpyspark
|
|
31
|
+
|
|
32
|
+
A drop-in mock for Apache PySpark 4.x for unit and integration testing. Runs entirely in Python — no JVM, no Hadoop, no cluster — using PyArrow as the compute engine and DuckDB for SQL.
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install mock-pyspark
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
`mockpyspark` (the PyPI distribution is `mock-pyspark`; you still `import mockpyspark`) installs alongside real `pyspark`; there is no namespace collision. Tests opt in via a one-liner so that `from pyspark.sql import ...` in your production code transparently resolves to the mock.
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
## Quick start
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
# conftest.py
|
|
46
|
+
import mockpyspark
|
|
47
|
+
mockpyspark.activate()
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
# tests/test_my_pipeline.py
|
|
52
|
+
from pyspark.sql import SparkSession, functions as F
|
|
53
|
+
|
|
54
|
+
def test_normalize(spark=None):
|
|
55
|
+
spark = SparkSession.builder.getOrCreate()
|
|
56
|
+
df = spark.createDataFrame([
|
|
57
|
+
{"id": 1, "name": " alice ", "score": 10},
|
|
58
|
+
{"id": 2, "name": "BOB", "score": 20},
|
|
59
|
+
])
|
|
60
|
+
out = (
|
|
61
|
+
df.withColumn("name", F.initcap(F.trim(F.col("name"))))
|
|
62
|
+
.filter(F.col("score") > 15)
|
|
63
|
+
.select("id", "name", "score")
|
|
64
|
+
.collect()
|
|
65
|
+
)
|
|
66
|
+
assert out == [(2, "Bob", 20)]
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
No changes to your Spark logic. The alias just makes `pyspark.*` imports resolve to `mockpyspark.*` for the test process.
|
|
70
|
+
|
|
71
|
+
---
|
|
72
|
+
|
|
73
|
+
## Version targeting
|
|
74
|
+
|
|
75
|
+
`mockpyspark` supports both PySpark 4.0.x and 4.1.x. APIs that only exist in 4.1 (e.g. `current_time`, `TimeType`) raise `AttributeError` when running in 4.0 mode.
|
|
76
|
+
|
|
77
|
+
It picks a target version automatically, in order:
|
|
78
|
+
|
|
79
|
+
1. `PYSPARK_VERSION` env var
|
|
80
|
+
2. The `pyspark` pin in `pyproject.toml`, `requirements*.txt`, `poetry.lock`, `uv.lock`, `Pipfile.lock`, or `setup.cfg` (walking up from the working directory)
|
|
81
|
+
3. `SparkSession.builder.config("spark.mock.targetVersion", "4.0").getOrCreate()` — per-session override
|
|
82
|
+
4. If nothing is found, defaults to the latest supported version and emits a `UserWarning`. Set `MOCKPYSPARK_STRICT_VERSION=1` to raise instead.
|
|
83
|
+
|
|
84
|
+
So if your project pins `pyspark==4.0.1` in `pyproject.toml`, the mock will refuse 4.1-only APIs without any configuration on your part.
|
|
85
|
+
|
|
86
|
+
---
|
|
87
|
+
|
|
88
|
+
## Usage modes
|
|
89
|
+
|
|
90
|
+
| Mode | What to do |
|
|
91
|
+
| ---------------------------- | ---------------------------------------------------------------- |
|
|
92
|
+
| **Mock for tests (default)** | `mockpyspark.activate()` in conftest. |
|
|
93
|
+
| **Pytest plugin** | `pytest_plugins = ["mockpyspark.pytest_plugin"]` in conftest. |
|
|
94
|
+
| **Real PySpark for a run** | Pass `--no-mock-pyspark` to pytest, or don't call `activate()`. |
|
|
95
|
+
|
|
96
|
+
The pytest plugin also accepts `--no-mock-pyspark` to bypass activation for a single invocation, useful for running the same suite against real PySpark for integration coverage.
|
|
97
|
+
|
|
98
|
+
See **[docs/usage.md](docs/usage.md)** for the full pattern, including switching between mock and real PySpark in the same suite, and known limitations.
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## Detecting the mock at runtime
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
import pyspark
|
|
106
|
+
if getattr(pyspark, "__mock__", False):
|
|
107
|
+
print("running against mockpyspark")
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
---
|
|
111
|
+
|
|
112
|
+
## Contributing
|
|
113
|
+
|
|
114
|
+
**Run unit tests** (no Java, no Docker):
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
git clone https://github.com/PeterDowdy/mock-pyspark.git && cd mock-pyspark
|
|
118
|
+
pip install pyarrow duckdb pandas numpy pytest
|
|
119
|
+
PYTHONPATH=. pytest -q -m unit
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
**Run integration tests** against real PySpark (requires Java 17):
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
# Via Docker (easiest)
|
|
126
|
+
./start.sh 4.1 --integration-test
|
|
127
|
+
|
|
128
|
+
# Or locally with a PySpark venv
|
|
129
|
+
python -m venv /opt/spark-venv && /opt/spark-venv/bin/pip install pyspark==4.1.0
|
|
130
|
+
PYTHONPATH=. PYSPARK_VERSION=4.1 PYSPARK_PYTHON=/opt/spark-venv/bin/python pytest -q -m integration
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
See **[docs/contributing.md](docs/contributing.md)** for the full setup guide.
|
|
134
|
+
|
|
135
|
+
---
|
|
136
|
+
|
|
137
|
+
## API coverage
|
|
138
|
+
|
|
139
|
+
Supports PySpark 4.0.x and 4.1.x — 439+ functions, full DataFrame/Column/Window/Catalog API, CSV/JSON/Parquet IO.
|
|
140
|
+
|
|
141
|
+
- [STATUS-4.1.md](STATUS-4.1.md) — PySpark 4.1 coverage
|
|
142
|
+
- [STATUS-4.0.md](STATUS-4.0.md) — PySpark 4.0 coverage
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# mockpyspark
|
|
2
|
+
|
|
3
|
+
A drop-in mock for Apache PySpark 4.x for unit and integration testing. Runs entirely in Python — no JVM, no Hadoop, no cluster — using PyArrow as the compute engine and DuckDB for SQL.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install mock-pyspark
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
`mockpyspark` (the PyPI distribution is `mock-pyspark`; you still `import mockpyspark`) installs alongside real `pyspark`; there is no namespace collision. Tests opt in via a one-liner so that `from pyspark.sql import ...` in your production code transparently resolves to the mock.
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## Quick start
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
# conftest.py
|
|
17
|
+
import mockpyspark
|
|
18
|
+
mockpyspark.activate()
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
```python
|
|
22
|
+
# tests/test_my_pipeline.py
|
|
23
|
+
from pyspark.sql import SparkSession, functions as F
|
|
24
|
+
|
|
25
|
+
def test_normalize(spark=None):
|
|
26
|
+
spark = SparkSession.builder.getOrCreate()
|
|
27
|
+
df = spark.createDataFrame([
|
|
28
|
+
{"id": 1, "name": " alice ", "score": 10},
|
|
29
|
+
{"id": 2, "name": "BOB", "score": 20},
|
|
30
|
+
])
|
|
31
|
+
out = (
|
|
32
|
+
df.withColumn("name", F.initcap(F.trim(F.col("name"))))
|
|
33
|
+
.filter(F.col("score") > 15)
|
|
34
|
+
.select("id", "name", "score")
|
|
35
|
+
.collect()
|
|
36
|
+
)
|
|
37
|
+
assert out == [(2, "Bob", 20)]
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
No changes to your Spark logic. The alias just makes `pyspark.*` imports resolve to `mockpyspark.*` for the test process.
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## Version targeting
|
|
45
|
+
|
|
46
|
+
`mockpyspark` supports both PySpark 4.0.x and 4.1.x. APIs that only exist in 4.1 (e.g. `current_time`, `TimeType`) raise `AttributeError` when running in 4.0 mode.
|
|
47
|
+
|
|
48
|
+
It picks a target version automatically, in order:
|
|
49
|
+
|
|
50
|
+
1. `PYSPARK_VERSION` env var
|
|
51
|
+
2. The `pyspark` pin in `pyproject.toml`, `requirements*.txt`, `poetry.lock`, `uv.lock`, `Pipfile.lock`, or `setup.cfg` (walking up from the working directory)
|
|
52
|
+
3. `SparkSession.builder.config("spark.mock.targetVersion", "4.0").getOrCreate()` — per-session override
|
|
53
|
+
4. If nothing is found, defaults to the latest supported version and emits a `UserWarning`. Set `MOCKPYSPARK_STRICT_VERSION=1` to raise instead.
|
|
54
|
+
|
|
55
|
+
So if your project pins `pyspark==4.0.1` in `pyproject.toml`, the mock will refuse 4.1-only APIs without any configuration on your part.
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## Usage modes
|
|
60
|
+
|
|
61
|
+
| Mode | What to do |
|
|
62
|
+
| ---------------------------- | ---------------------------------------------------------------- |
|
|
63
|
+
| **Mock for tests (default)** | `mockpyspark.activate()` in conftest. |
|
|
64
|
+
| **Pytest plugin** | `pytest_plugins = ["mockpyspark.pytest_plugin"]` in conftest. |
|
|
65
|
+
| **Real PySpark for a run** | Pass `--no-mock-pyspark` to pytest, or don't call `activate()`. |
|
|
66
|
+
|
|
67
|
+
The pytest plugin also accepts `--no-mock-pyspark` to bypass activation for a single invocation, useful for running the same suite against real PySpark for integration coverage.
|
|
68
|
+
|
|
69
|
+
See **[docs/usage.md](docs/usage.md)** for the full pattern, including switching between mock and real PySpark in the same suite, and known limitations.
|
|
70
|
+
|
|
71
|
+
---
|
|
72
|
+
|
|
73
|
+
## Detecting the mock at runtime
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
import pyspark
|
|
77
|
+
if getattr(pyspark, "__mock__", False):
|
|
78
|
+
print("running against mockpyspark")
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
---
|
|
82
|
+
|
|
83
|
+
## Contributing
|
|
84
|
+
|
|
85
|
+
**Run unit tests** (no Java, no Docker):
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
git clone https://github.com/PeterDowdy/mock-pyspark.git && cd mock-pyspark
|
|
89
|
+
pip install pyarrow duckdb pandas numpy pytest
|
|
90
|
+
PYTHONPATH=. pytest -q -m unit
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
**Run integration tests** against real PySpark (requires Java 17):
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
# Via Docker (easiest)
|
|
97
|
+
./start.sh 4.1 --integration-test
|
|
98
|
+
|
|
99
|
+
# Or locally with a PySpark venv
|
|
100
|
+
python -m venv /opt/spark-venv && /opt/spark-venv/bin/pip install pyspark==4.1.0
|
|
101
|
+
PYTHONPATH=. PYSPARK_VERSION=4.1 PYSPARK_PYTHON=/opt/spark-venv/bin/python pytest -q -m integration
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
See **[docs/contributing.md](docs/contributing.md)** for the full setup guide.
|
|
105
|
+
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## API coverage
|
|
109
|
+
|
|
110
|
+
Supports PySpark 4.0.x and 4.1.x — 439+ functions, full DataFrame/Column/Window/Catalog API, CSV/JSON/Parquet IO.
|
|
111
|
+
|
|
112
|
+
- [STATUS-4.1.md](STATUS-4.1.md) — PySpark 4.1 coverage
|
|
113
|
+
- [STATUS-4.0.md](STATUS-4.0.md) — PySpark 4.0 coverage
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mock-pyspark
|
|
3
|
+
Version: 0.0.1.dev20260804
|
|
4
|
+
Summary: Drop-in mock for Apache PySpark 4.x — Arrow-backed, no JVM required
|
|
5
|
+
Author-email: Peter Dowdy <peter.dowdy@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/PeterDowdy/mock-pyspark
|
|
8
|
+
Project-URL: Repository, https://github.com/PeterDowdy/mock-pyspark
|
|
9
|
+
Project-URL: Issues, https://github.com/PeterDowdy/mock-pyspark/issues
|
|
10
|
+
Keywords: pyspark,mock,testing,arrow,duckdb
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Topic :: Software Development :: Testing
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
Requires-Dist: pyarrow>=14
|
|
20
|
+
Requires-Dist: duckdb>=1.0
|
|
21
|
+
Requires-Dist: pandas>=2.0
|
|
22
|
+
Requires-Dist: numpy>=1.24
|
|
23
|
+
Provides-Extra: avro
|
|
24
|
+
Requires-Dist: fastavro>=1.9; extra == "avro"
|
|
25
|
+
Provides-Extra: delta
|
|
26
|
+
Requires-Dist: deltalake>=1.0; extra == "delta"
|
|
27
|
+
Provides-Extra: all
|
|
28
|
+
Requires-Dist: mock-pyspark[avro,delta]; extra == "all"
|
|
29
|
+
|
|
30
|
+
# mockpyspark
|
|
31
|
+
|
|
32
|
+
A drop-in mock for Apache PySpark 4.x for unit and integration testing. Runs entirely in Python — no JVM, no Hadoop, no cluster — using PyArrow as the compute engine and DuckDB for SQL.
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install mock-pyspark
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
`mockpyspark` (the PyPI distribution is `mock-pyspark`; you still `import mockpyspark`) installs alongside real `pyspark`; there is no namespace collision. Tests opt in via a one-liner so that `from pyspark.sql import ...` in your production code transparently resolves to the mock.
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
## Quick start
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
# conftest.py
|
|
46
|
+
import mockpyspark
|
|
47
|
+
mockpyspark.activate()
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
# tests/test_my_pipeline.py
|
|
52
|
+
from pyspark.sql import SparkSession, functions as F
|
|
53
|
+
|
|
54
|
+
def test_normalize(spark=None):
|
|
55
|
+
spark = SparkSession.builder.getOrCreate()
|
|
56
|
+
df = spark.createDataFrame([
|
|
57
|
+
{"id": 1, "name": " alice ", "score": 10},
|
|
58
|
+
{"id": 2, "name": "BOB", "score": 20},
|
|
59
|
+
])
|
|
60
|
+
out = (
|
|
61
|
+
df.withColumn("name", F.initcap(F.trim(F.col("name"))))
|
|
62
|
+
.filter(F.col("score") > 15)
|
|
63
|
+
.select("id", "name", "score")
|
|
64
|
+
.collect()
|
|
65
|
+
)
|
|
66
|
+
assert out == [(2, "Bob", 20)]
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
No changes to your Spark logic. The alias just makes `pyspark.*` imports resolve to `mockpyspark.*` for the test process.
|
|
70
|
+
|
|
71
|
+
---
|
|
72
|
+
|
|
73
|
+
## Version targeting
|
|
74
|
+
|
|
75
|
+
`mockpyspark` supports both PySpark 4.0.x and 4.1.x. APIs that only exist in 4.1 (e.g. `current_time`, `TimeType`) raise `AttributeError` when running in 4.0 mode.
|
|
76
|
+
|
|
77
|
+
It picks a target version automatically, in order:
|
|
78
|
+
|
|
79
|
+
1. `PYSPARK_VERSION` env var
|
|
80
|
+
2. The `pyspark` pin in `pyproject.toml`, `requirements*.txt`, `poetry.lock`, `uv.lock`, `Pipfile.lock`, or `setup.cfg` (walking up from the working directory)
|
|
81
|
+
3. `SparkSession.builder.config("spark.mock.targetVersion", "4.0").getOrCreate()` — per-session override
|
|
82
|
+
4. If nothing is found, defaults to the latest supported version and emits a `UserWarning`. Set `MOCKPYSPARK_STRICT_VERSION=1` to raise instead.
|
|
83
|
+
|
|
84
|
+
So if your project pins `pyspark==4.0.1` in `pyproject.toml`, the mock will refuse 4.1-only APIs without any configuration on your part.
|
|
85
|
+
|
|
86
|
+
---
|
|
87
|
+
|
|
88
|
+
## Usage modes
|
|
89
|
+
|
|
90
|
+
| Mode | What to do |
|
|
91
|
+
| ---------------------------- | ---------------------------------------------------------------- |
|
|
92
|
+
| **Mock for tests (default)** | `mockpyspark.activate()` in conftest. |
|
|
93
|
+
| **Pytest plugin** | `pytest_plugins = ["mockpyspark.pytest_plugin"]` in conftest. |
|
|
94
|
+
| **Real PySpark for a run** | Pass `--no-mock-pyspark` to pytest, or don't call `activate()`. |
|
|
95
|
+
|
|
96
|
+
The pytest plugin also accepts `--no-mock-pyspark` to bypass activation for a single invocation, useful for running the same suite against real PySpark for integration coverage.
|
|
97
|
+
|
|
98
|
+
See **[docs/usage.md](docs/usage.md)** for the full pattern, including switching between mock and real PySpark in the same suite, and known limitations.
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## Detecting the mock at runtime
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
import pyspark
|
|
106
|
+
if getattr(pyspark, "__mock__", False):
|
|
107
|
+
print("running against mockpyspark")
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
---
|
|
111
|
+
|
|
112
|
+
## Contributing
|
|
113
|
+
|
|
114
|
+
**Run unit tests** (no Java, no Docker):
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
git clone https://github.com/PeterDowdy/mock-pyspark.git && cd mock-pyspark
|
|
118
|
+
pip install pyarrow duckdb pandas numpy pytest
|
|
119
|
+
PYTHONPATH=. pytest -q -m unit
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
**Run integration tests** against real PySpark (requires Java 17):
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
# Via Docker (easiest)
|
|
126
|
+
./start.sh 4.1 --integration-test
|
|
127
|
+
|
|
128
|
+
# Or locally with a PySpark venv
|
|
129
|
+
python -m venv /opt/spark-venv && /opt/spark-venv/bin/pip install pyspark==4.1.0
|
|
130
|
+
PYTHONPATH=. PYSPARK_VERSION=4.1 PYSPARK_PYTHON=/opt/spark-venv/bin/python pytest -q -m integration
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
See **[docs/contributing.md](docs/contributing.md)** for the full setup guide.
|
|
134
|
+
|
|
135
|
+
---
|
|
136
|
+
|
|
137
|
+
## API coverage
|
|
138
|
+
|
|
139
|
+
Supports PySpark 4.0.x and 4.1.x — 439+ functions, full DataFrame/Column/Window/Catalog API, CSV/JSON/Parquet IO.
|
|
140
|
+
|
|
141
|
+
- [STATUS-4.1.md](STATUS-4.1.md) — PySpark 4.1 coverage
|
|
142
|
+
- [STATUS-4.0.md](STATUS-4.0.md) — PySpark 4.0 coverage
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
./mockpyspark/__init__.py
|
|
4
|
+
./mockpyspark/_aliasing.py
|
|
5
|
+
./mockpyspark/pytest_plugin.py
|
|
6
|
+
./mockpyspark/rdd.py
|
|
7
|
+
./mockpyspark/_internal/__init__.py
|
|
8
|
+
./mockpyspark/_internal/config.py
|
|
9
|
+
./mockpyspark/_internal/engine.py
|
|
10
|
+
./mockpyspark/_internal/version_detect.py
|
|
11
|
+
./mockpyspark/_internal/version_guard.py
|
|
12
|
+
./mockpyspark/delta/__init__.py
|
|
13
|
+
./mockpyspark/delta/pip.py
|
|
14
|
+
./mockpyspark/delta/pip_utils.py
|
|
15
|
+
./mockpyspark/delta/tables.py
|
|
16
|
+
./mockpyspark/errors/__init__.py
|
|
17
|
+
./mockpyspark/ml/__init__.py
|
|
18
|
+
./mockpyspark/pandas/__init__.py
|
|
19
|
+
./mockpyspark/sql/__init__.py
|
|
20
|
+
./mockpyspark/sql/_map_utils.py
|
|
21
|
+
./mockpyspark/sql/catalog.py
|
|
22
|
+
./mockpyspark/sql/column.py
|
|
23
|
+
./mockpyspark/sql/dataframe.py
|
|
24
|
+
./mockpyspark/sql/functions.py
|
|
25
|
+
./mockpyspark/sql/functions_all.py
|
|
26
|
+
./mockpyspark/sql/readwriter.py
|
|
27
|
+
./mockpyspark/sql/row.py
|
|
28
|
+
./mockpyspark/sql/session.py
|
|
29
|
+
./mockpyspark/sql/sql_dispatcher.py
|
|
30
|
+
./mockpyspark/sql/types.py
|
|
31
|
+
./mockpyspark/sql/utils.py
|
|
32
|
+
./mockpyspark/sql/window.py
|
|
33
|
+
./mockpyspark/sql/functions/__init__.py
|
|
34
|
+
./mockpyspark/sql/functions/aggregates.py
|
|
35
|
+
./mockpyspark/sql/functions/arrays.py
|
|
36
|
+
./mockpyspark/sql/functions/bits.py
|
|
37
|
+
./mockpyspark/sql/functions/core.py
|
|
38
|
+
./mockpyspark/sql/functions/crypto.py
|
|
39
|
+
./mockpyspark/sql/functions/csvs.py
|
|
40
|
+
./mockpyspark/sql/functions/dates.py
|
|
41
|
+
./mockpyspark/sql/functions/generators.py
|
|
42
|
+
./mockpyspark/sql/functions/geospatial.py
|
|
43
|
+
./mockpyspark/sql/functions/hashes.py
|
|
44
|
+
./mockpyspark/sql/functions/jsons.py
|
|
45
|
+
./mockpyspark/sql/functions/maps.py
|
|
46
|
+
./mockpyspark/sql/functions/maths.py
|
|
47
|
+
./mockpyspark/sql/functions/partitioning.py
|
|
48
|
+
./mockpyspark/sql/functions/predicates.py
|
|
49
|
+
./mockpyspark/sql/functions/strings.py
|
|
50
|
+
./mockpyspark/sql/functions/udfs.py
|
|
51
|
+
./mockpyspark/sql/functions/urls.py
|
|
52
|
+
./mockpyspark/sql/functions/variants.py
|
|
53
|
+
./mockpyspark/sql/functions/windows.py
|
|
54
|
+
./mockpyspark/sql/functions/xmls.py
|
|
55
|
+
./mockpyspark/sql/io/__init__.py
|
|
56
|
+
./mockpyspark/sql/io/_arrow_types.py
|
|
57
|
+
./mockpyspark/sql/io/avro.py
|
|
58
|
+
./mockpyspark/sql/io/base.py
|
|
59
|
+
./mockpyspark/sql/io/binary_file.py
|
|
60
|
+
./mockpyspark/sql/io/csv.py
|
|
61
|
+
./mockpyspark/sql/io/delta.py
|
|
62
|
+
./mockpyspark/sql/io/json.py
|
|
63
|
+
./mockpyspark/sql/io/orc.py
|
|
64
|
+
./mockpyspark/sql/io/parquet.py
|
|
65
|
+
./mockpyspark/sql/io/text.py
|
|
66
|
+
./mockpyspark/sql/streaming/__init__.py
|
|
67
|
+
./mockpyspark/sql/streaming/readwriter.py
|
|
68
|
+
./mockpyspark/streaming/__init__.py
|
|
69
|
+
./mockpyspark/testing/__init__.py
|
|
70
|
+
./mockpyspark/testing/utils.py
|
|
71
|
+
mock_pyspark.egg-info/PKG-INFO
|
|
72
|
+
mock_pyspark.egg-info/SOURCES.txt
|
|
73
|
+
mock_pyspark.egg-info/dependency_links.txt
|
|
74
|
+
mock_pyspark.egg-info/requires.txt
|
|
75
|
+
mock_pyspark.egg-info/top_level.txt
|
|
76
|
+
mockpyspark/__init__.py
|
|
77
|
+
mockpyspark/_aliasing.py
|
|
78
|
+
mockpyspark/pytest_plugin.py
|
|
79
|
+
mockpyspark/rdd.py
|
|
80
|
+
mockpyspark/_internal/__init__.py
|
|
81
|
+
mockpyspark/_internal/config.py
|
|
82
|
+
mockpyspark/_internal/engine.py
|
|
83
|
+
mockpyspark/_internal/version_detect.py
|
|
84
|
+
mockpyspark/_internal/version_guard.py
|
|
85
|
+
mockpyspark/delta/__init__.py
|
|
86
|
+
mockpyspark/delta/pip.py
|
|
87
|
+
mockpyspark/delta/pip_utils.py
|
|
88
|
+
mockpyspark/delta/tables.py
|
|
89
|
+
mockpyspark/errors/__init__.py
|
|
90
|
+
mockpyspark/ml/__init__.py
|
|
91
|
+
mockpyspark/pandas/__init__.py
|
|
92
|
+
mockpyspark/sql/__init__.py
|
|
93
|
+
mockpyspark/sql/_map_utils.py
|
|
94
|
+
mockpyspark/sql/catalog.py
|
|
95
|
+
mockpyspark/sql/column.py
|
|
96
|
+
mockpyspark/sql/dataframe.py
|
|
97
|
+
mockpyspark/sql/functions.py
|
|
98
|
+
mockpyspark/sql/functions_all.py
|
|
99
|
+
mockpyspark/sql/readwriter.py
|
|
100
|
+
mockpyspark/sql/row.py
|
|
101
|
+
mockpyspark/sql/session.py
|
|
102
|
+
mockpyspark/sql/sql_dispatcher.py
|
|
103
|
+
mockpyspark/sql/types.py
|
|
104
|
+
mockpyspark/sql/utils.py
|
|
105
|
+
mockpyspark/sql/window.py
|
|
106
|
+
mockpyspark/sql/functions/__init__.py
|
|
107
|
+
mockpyspark/sql/functions/aggregates.py
|
|
108
|
+
mockpyspark/sql/functions/arrays.py
|
|
109
|
+
mockpyspark/sql/functions/bits.py
|
|
110
|
+
mockpyspark/sql/functions/core.py
|
|
111
|
+
mockpyspark/sql/functions/crypto.py
|
|
112
|
+
mockpyspark/sql/functions/csvs.py
|
|
113
|
+
mockpyspark/sql/functions/dates.py
|
|
114
|
+
mockpyspark/sql/functions/generators.py
|
|
115
|
+
mockpyspark/sql/functions/geospatial.py
|
|
116
|
+
mockpyspark/sql/functions/hashes.py
|
|
117
|
+
mockpyspark/sql/functions/jsons.py
|
|
118
|
+
mockpyspark/sql/functions/maps.py
|
|
119
|
+
mockpyspark/sql/functions/maths.py
|
|
120
|
+
mockpyspark/sql/functions/partitioning.py
|
|
121
|
+
mockpyspark/sql/functions/predicates.py
|
|
122
|
+
mockpyspark/sql/functions/strings.py
|
|
123
|
+
mockpyspark/sql/functions/udfs.py
|
|
124
|
+
mockpyspark/sql/functions/urls.py
|
|
125
|
+
mockpyspark/sql/functions/variants.py
|
|
126
|
+
mockpyspark/sql/functions/windows.py
|
|
127
|
+
mockpyspark/sql/functions/xmls.py
|
|
128
|
+
mockpyspark/sql/io/__init__.py
|
|
129
|
+
mockpyspark/sql/io/_arrow_types.py
|
|
130
|
+
mockpyspark/sql/io/avro.py
|
|
131
|
+
mockpyspark/sql/io/base.py
|
|
132
|
+
mockpyspark/sql/io/binary_file.py
|
|
133
|
+
mockpyspark/sql/io/csv.py
|
|
134
|
+
mockpyspark/sql/io/delta.py
|
|
135
|
+
mockpyspark/sql/io/json.py
|
|
136
|
+
mockpyspark/sql/io/orc.py
|
|
137
|
+
mockpyspark/sql/io/parquet.py
|
|
138
|
+
mockpyspark/sql/io/text.py
|
|
139
|
+
mockpyspark/sql/streaming/__init__.py
|
|
140
|
+
mockpyspark/sql/streaming/readwriter.py
|
|
141
|
+
mockpyspark/streaming/__init__.py
|
|
142
|
+
mockpyspark/testing/__init__.py
|
|
143
|
+
mockpyspark/testing/utils.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
mockpyspark
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""mockpyspark — drop-in mock for Apache PySpark 4.x for testing.
|
|
2
|
+
|
|
3
|
+
Activate the import alias so production code's `from pyspark.sql import ...`
|
|
4
|
+
resolves to the mock without uninstalling real PySpark::
|
|
5
|
+
|
|
6
|
+
import mockpyspark
|
|
7
|
+
mockpyspark.activate()
|
|
8
|
+
|
|
9
|
+
Or, in pytest, load the bundled plugin::
|
|
10
|
+
|
|
11
|
+
# conftest.py
|
|
12
|
+
pytest_plugins = ["mockpyspark.pytest_plugin"]
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from ._aliasing import activate, deactivate, is_active
|
|
17
|
+
from .rdd import RDD
|
|
18
|
+
from .sql.session import SparkSession
|
|
19
|
+
|
|
20
|
+
__all__ = ["SparkSession", "RDD", "activate", "deactivate", "is_active"]
|
|
21
|
+
|
|
22
|
+
# Sentinel: lets user code detect at runtime that the loaded `pyspark` is the
|
|
23
|
+
# mock without depending on file paths or install layout.
|
|
24
|
+
__mock__ = True
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def __getattr__(name: str):
|
|
28
|
+
# Lazy so `pyspark.__version__` always reflects the current target version,
|
|
29
|
+
# even after it changes via env var or SparkSession builder config.
|
|
30
|
+
if name == "__version__":
|
|
31
|
+
from ._internal.config import CONFIG
|
|
32
|
+
return CONFIG.target_version
|
|
33
|
+
raise AttributeError(f"module 'mockpyspark' has no attribute {name!r}")
|