mock-pyspark 0.0.1.dev20260804__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. mock_pyspark-0.0.1.dev20260804/PKG-INFO +142 -0
  2. mock_pyspark-0.0.1.dev20260804/README.md +113 -0
  3. mock_pyspark-0.0.1.dev20260804/mock_pyspark.egg-info/PKG-INFO +142 -0
  4. mock_pyspark-0.0.1.dev20260804/mock_pyspark.egg-info/SOURCES.txt +143 -0
  5. mock_pyspark-0.0.1.dev20260804/mock_pyspark.egg-info/dependency_links.txt +1 -0
  6. mock_pyspark-0.0.1.dev20260804/mock_pyspark.egg-info/requires.txt +13 -0
  7. mock_pyspark-0.0.1.dev20260804/mock_pyspark.egg-info/top_level.txt +1 -0
  8. mock_pyspark-0.0.1.dev20260804/mockpyspark/__init__.py +33 -0
  9. mock_pyspark-0.0.1.dev20260804/mockpyspark/_aliasing.py +105 -0
  10. mock_pyspark-0.0.1.dev20260804/mockpyspark/_internal/__init__.py +0 -0
  11. mock_pyspark-0.0.1.dev20260804/mockpyspark/_internal/config.py +27 -0
  12. mock_pyspark-0.0.1.dev20260804/mockpyspark/_internal/engine.py +5 -0
  13. mock_pyspark-0.0.1.dev20260804/mockpyspark/_internal/version_detect.py +118 -0
  14. mock_pyspark-0.0.1.dev20260804/mockpyspark/_internal/version_guard.py +37 -0
  15. mock_pyspark-0.0.1.dev20260804/mockpyspark/delta/__init__.py +15 -0
  16. mock_pyspark-0.0.1.dev20260804/mockpyspark/delta/pip.py +9 -0
  17. mock_pyspark-0.0.1.dev20260804/mockpyspark/delta/pip_utils.py +23 -0
  18. mock_pyspark-0.0.1.dev20260804/mockpyspark/delta/tables.py +303 -0
  19. mock_pyspark-0.0.1.dev20260804/mockpyspark/errors/__init__.py +47 -0
  20. mock_pyspark-0.0.1.dev20260804/mockpyspark/ml/__init__.py +26 -0
  21. mock_pyspark-0.0.1.dev20260804/mockpyspark/pandas/__init__.py +123 -0
  22. mock_pyspark-0.0.1.dev20260804/mockpyspark/pytest_plugin.py +35 -0
  23. mock_pyspark-0.0.1.dev20260804/mockpyspark/rdd.py +28 -0
  24. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/__init__.py +38 -0
  25. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/_map_utils.py +38 -0
  26. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/catalog.py +349 -0
  27. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/column.py +861 -0
  28. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/dataframe.py +2948 -0
  29. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/__init__.py +228 -0
  30. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/aggregates.py +783 -0
  31. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/arrays.py +1014 -0
  32. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/bits.py +239 -0
  33. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/core.py +358 -0
  34. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/crypto.py +57 -0
  35. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/csvs.py +152 -0
  36. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/dates.py +1941 -0
  37. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/generators.py +68 -0
  38. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/geospatial.py +125 -0
  39. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/hashes.py +161 -0
  40. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/jsons.py +243 -0
  41. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/maps.py +424 -0
  42. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/maths.py +691 -0
  43. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/partitioning.py +29 -0
  44. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/predicates.py +261 -0
  45. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/strings.py +1754 -0
  46. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/udfs.py +77 -0
  47. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/urls.py +176 -0
  48. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/variants.py +391 -0
  49. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/windows.py +54 -0
  50. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions/xmls.py +182 -0
  51. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions.py +3 -0
  52. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/functions_all.py +16 -0
  53. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/__init__.py +85 -0
  54. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/_arrow_types.py +174 -0
  55. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/avro.py +130 -0
  56. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/base.py +358 -0
  57. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/binary_file.py +49 -0
  58. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/csv.py +129 -0
  59. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/delta.py +102 -0
  60. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/json.py +122 -0
  61. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/orc.py +61 -0
  62. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/parquet.py +51 -0
  63. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/io/text.py +74 -0
  64. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/readwriter.py +917 -0
  65. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/row.py +76 -0
  66. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/session.py +354 -0
  67. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/sql_dispatcher.py +1180 -0
  68. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/streaming/__init__.py +15 -0
  69. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/streaming/readwriter.py +13 -0
  70. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/types.py +757 -0
  71. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/utils.py +33 -0
  72. mock_pyspark-0.0.1.dev20260804/mockpyspark/sql/window.py +138 -0
  73. mock_pyspark-0.0.1.dev20260804/mockpyspark/streaming/__init__.py +15 -0
  74. mock_pyspark-0.0.1.dev20260804/mockpyspark/testing/__init__.py +19 -0
  75. mock_pyspark-0.0.1.dev20260804/mockpyspark/testing/utils.py +150 -0
  76. mock_pyspark-0.0.1.dev20260804/pyproject.toml +51 -0
  77. mock_pyspark-0.0.1.dev20260804/setup.cfg +4 -0
@@ -0,0 +1,142 @@
1
+ Metadata-Version: 2.4
2
+ Name: mock-pyspark
3
+ Version: 0.0.1.dev20260804
4
+ Summary: Drop-in mock for Apache PySpark 4.x — Arrow-backed, no JVM required
5
+ Author-email: Peter Dowdy <peter.dowdy@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/PeterDowdy/mock-pyspark
8
+ Project-URL: Repository, https://github.com/PeterDowdy/mock-pyspark
9
+ Project-URL: Issues, https://github.com/PeterDowdy/mock-pyspark/issues
10
+ Keywords: pyspark,mock,testing,arrow,duckdb
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Topic :: Software Development :: Testing
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Operating System :: OS Independent
17
+ Requires-Python: >=3.9
18
+ Description-Content-Type: text/markdown
19
+ Requires-Dist: pyarrow>=14
20
+ Requires-Dist: duckdb>=1.0
21
+ Requires-Dist: pandas>=2.0
22
+ Requires-Dist: numpy>=1.24
23
+ Provides-Extra: avro
24
+ Requires-Dist: fastavro>=1.9; extra == "avro"
25
+ Provides-Extra: delta
26
+ Requires-Dist: deltalake>=1.0; extra == "delta"
27
+ Provides-Extra: all
28
+ Requires-Dist: mock-pyspark[avro,delta]; extra == "all"
29
+
30
+ # mockpyspark
31
+
32
+ A drop-in mock for Apache PySpark 4.x for unit and integration testing. Runs entirely in Python — no JVM, no Hadoop, no cluster — using PyArrow as the compute engine and DuckDB for SQL.
33
+
34
+ ```bash
35
+ pip install mock-pyspark
36
+ ```
37
+
38
+ `mockpyspark` (the PyPI distribution is `mock-pyspark`; you still `import mockpyspark`) installs alongside real `pyspark`; there is no namespace collision. Tests opt in via a one-liner so that `from pyspark.sql import ...` in your production code transparently resolves to the mock.
39
+
40
+ ---
41
+
42
+ ## Quick start
43
+
44
+ ```python
45
+ # conftest.py
46
+ import mockpyspark
47
+ mockpyspark.activate()
48
+ ```
49
+
50
+ ```python
51
+ # tests/test_my_pipeline.py
52
+ from pyspark.sql import SparkSession, functions as F
53
+
54
+ def test_normalize(spark=None):
55
+ spark = SparkSession.builder.getOrCreate()
56
+ df = spark.createDataFrame([
57
+ {"id": 1, "name": " alice ", "score": 10},
58
+ {"id": 2, "name": "BOB", "score": 20},
59
+ ])
60
+ out = (
61
+ df.withColumn("name", F.initcap(F.trim(F.col("name"))))
62
+ .filter(F.col("score") > 15)
63
+ .select("id", "name", "score")
64
+ .collect()
65
+ )
66
+ assert out == [(2, "Bob", 20)]
67
+ ```
68
+
69
+ No changes to your Spark logic. The alias just makes `pyspark.*` imports resolve to `mockpyspark.*` for the test process.
70
+
71
+ ---
72
+
73
+ ## Version targeting
74
+
75
+ `mockpyspark` supports both PySpark 4.0.x and 4.1.x. APIs that only exist in 4.1 (e.g. `current_time`, `TimeType`) raise `AttributeError` when running in 4.0 mode.
76
+
77
+ It picks a target version automatically, in order:
78
+
79
+ 1. `PYSPARK_VERSION` env var
80
+ 2. The `pyspark` pin in `pyproject.toml`, `requirements*.txt`, `poetry.lock`, `uv.lock`, `Pipfile.lock`, or `setup.cfg` (walking up from the working directory)
81
+ 3. `SparkSession.builder.config("spark.mock.targetVersion", "4.0").getOrCreate()` — per-session override
82
+ 4. If nothing is found, defaults to the latest supported version and emits a `UserWarning`. Set `MOCKPYSPARK_STRICT_VERSION=1` to raise instead.
83
+
84
+ So if your project pins `pyspark==4.0.1` in `pyproject.toml`, the mock will refuse 4.1-only APIs without any configuration on your part.
85
+
86
+ ---
87
+
88
+ ## Usage modes
89
+
90
+ | Mode | What to do |
91
+ | ---------------------------- | ---------------------------------------------------------------- |
92
+ | **Mock for tests (default)** | `mockpyspark.activate()` in conftest. |
93
+ | **Pytest plugin** | `pytest_plugins = ["mockpyspark.pytest_plugin"]` in conftest. |
94
+ | **Real PySpark for a run** | Pass `--no-mock-pyspark` to pytest, or don't call `activate()`. |
95
+
96
+ The pytest plugin also accepts `--no-mock-pyspark` to bypass activation for a single invocation, useful for running the same suite against real PySpark for integration coverage.
97
+
98
+ See **[docs/usage.md](docs/usage.md)** for the full pattern, including switching between mock and real PySpark in the same suite, and known limitations.
99
+
100
+ ---
101
+
102
+ ## Detecting the mock at runtime
103
+
104
+ ```python
105
+ import pyspark
106
+ if getattr(pyspark, "__mock__", False):
107
+ print("running against mockpyspark")
108
+ ```
109
+
110
+ ---
111
+
112
+ ## Contributing
113
+
114
+ **Run unit tests** (no Java, no Docker):
115
+
116
+ ```bash
117
+ git clone https://github.com/PeterDowdy/mock-pyspark.git && cd mock-pyspark
118
+ pip install pyarrow duckdb pandas numpy pytest
119
+ PYTHONPATH=. pytest -q -m unit
120
+ ```
121
+
122
+ **Run integration tests** against real PySpark (requires Java 17):
123
+
124
+ ```bash
125
+ # Via Docker (easiest)
126
+ ./start.sh 4.1 --integration-test
127
+
128
+ # Or locally with a PySpark venv
129
+ python -m venv /opt/spark-venv && /opt/spark-venv/bin/pip install pyspark==4.1.0
130
+ PYTHONPATH=. PYSPARK_VERSION=4.1 PYSPARK_PYTHON=/opt/spark-venv/bin/python pytest -q -m integration
131
+ ```
132
+
133
+ See **[docs/contributing.md](docs/contributing.md)** for the full setup guide.
134
+
135
+ ---
136
+
137
+ ## API coverage
138
+
139
+ Supports PySpark 4.0.x and 4.1.x — 439+ functions, full DataFrame/Column/Window/Catalog API, CSV/JSON/Parquet IO.
140
+
141
+ - [STATUS-4.1.md](STATUS-4.1.md) — PySpark 4.1 coverage
142
+ - [STATUS-4.0.md](STATUS-4.0.md) — PySpark 4.0 coverage
@@ -0,0 +1,113 @@
1
+ # mockpyspark
2
+
3
+ A drop-in mock for Apache PySpark 4.x for unit and integration testing. Runs entirely in Python — no JVM, no Hadoop, no cluster — using PyArrow as the compute engine and DuckDB for SQL.
4
+
5
+ ```bash
6
+ pip install mock-pyspark
7
+ ```
8
+
9
+ `mockpyspark` (the PyPI distribution is `mock-pyspark`; you still `import mockpyspark`) installs alongside real `pyspark`; there is no namespace collision. Tests opt in via a one-liner so that `from pyspark.sql import ...` in your production code transparently resolves to the mock.
10
+
11
+ ---
12
+
13
+ ## Quick start
14
+
15
+ ```python
16
+ # conftest.py
17
+ import mockpyspark
18
+ mockpyspark.activate()
19
+ ```
20
+
21
+ ```python
22
+ # tests/test_my_pipeline.py
23
+ from pyspark.sql import SparkSession, functions as F
24
+
25
+ def test_normalize(spark=None):
26
+ spark = SparkSession.builder.getOrCreate()
27
+ df = spark.createDataFrame([
28
+ {"id": 1, "name": " alice ", "score": 10},
29
+ {"id": 2, "name": "BOB", "score": 20},
30
+ ])
31
+ out = (
32
+ df.withColumn("name", F.initcap(F.trim(F.col("name"))))
33
+ .filter(F.col("score") > 15)
34
+ .select("id", "name", "score")
35
+ .collect()
36
+ )
37
+ assert out == [(2, "Bob", 20)]
38
+ ```
39
+
40
+ No changes to your Spark logic. The alias just makes `pyspark.*` imports resolve to `mockpyspark.*` for the test process.
41
+
42
+ ---
43
+
44
+ ## Version targeting
45
+
46
+ `mockpyspark` supports both PySpark 4.0.x and 4.1.x. APIs that only exist in 4.1 (e.g. `current_time`, `TimeType`) raise `AttributeError` when running in 4.0 mode.
47
+
48
+ It picks a target version automatically, in order:
49
+
50
+ 1. `PYSPARK_VERSION` env var
51
+ 2. The `pyspark` pin in `pyproject.toml`, `requirements*.txt`, `poetry.lock`, `uv.lock`, `Pipfile.lock`, or `setup.cfg` (walking up from the working directory)
52
+ 3. `SparkSession.builder.config("spark.mock.targetVersion", "4.0").getOrCreate()` — per-session override
53
+ 4. If nothing is found, defaults to the latest supported version and emits a `UserWarning`. Set `MOCKPYSPARK_STRICT_VERSION=1` to raise instead.
54
+
55
+ So if your project pins `pyspark==4.0.1` in `pyproject.toml`, the mock will refuse 4.1-only APIs without any configuration on your part.
56
+
57
+ ---
58
+
59
+ ## Usage modes
60
+
61
+ | Mode | What to do |
62
+ | ---------------------------- | ---------------------------------------------------------------- |
63
+ | **Mock for tests (default)** | `mockpyspark.activate()` in conftest. |
64
+ | **Pytest plugin** | `pytest_plugins = ["mockpyspark.pytest_plugin"]` in conftest. |
65
+ | **Real PySpark for a run** | Pass `--no-mock-pyspark` to pytest, or don't call `activate()`. |
66
+
67
+ The pytest plugin also accepts `--no-mock-pyspark` to bypass activation for a single invocation, useful for running the same suite against real PySpark for integration coverage.
68
+
69
+ See **[docs/usage.md](docs/usage.md)** for the full pattern, including switching between mock and real PySpark in the same suite, and known limitations.
70
+
71
+ ---
72
+
73
+ ## Detecting the mock at runtime
74
+
75
+ ```python
76
+ import pyspark
77
+ if getattr(pyspark, "__mock__", False):
78
+ print("running against mockpyspark")
79
+ ```
80
+
81
+ ---
82
+
83
+ ## Contributing
84
+
85
+ **Run unit tests** (no Java, no Docker):
86
+
87
+ ```bash
88
+ git clone https://github.com/PeterDowdy/mock-pyspark.git && cd mock-pyspark
89
+ pip install pyarrow duckdb pandas numpy pytest
90
+ PYTHONPATH=. pytest -q -m unit
91
+ ```
92
+
93
+ **Run integration tests** against real PySpark (requires Java 17):
94
+
95
+ ```bash
96
+ # Via Docker (easiest)
97
+ ./start.sh 4.1 --integration-test
98
+
99
+ # Or locally with a PySpark venv
100
+ python -m venv /opt/spark-venv && /opt/spark-venv/bin/pip install pyspark==4.1.0
101
+ PYTHONPATH=. PYSPARK_VERSION=4.1 PYSPARK_PYTHON=/opt/spark-venv/bin/python pytest -q -m integration
102
+ ```
103
+
104
+ See **[docs/contributing.md](docs/contributing.md)** for the full setup guide.
105
+
106
+ ---
107
+
108
+ ## API coverage
109
+
110
+ Supports PySpark 4.0.x and 4.1.x — 439+ functions, full DataFrame/Column/Window/Catalog API, CSV/JSON/Parquet IO.
111
+
112
+ - [STATUS-4.1.md](STATUS-4.1.md) — PySpark 4.1 coverage
113
+ - [STATUS-4.0.md](STATUS-4.0.md) — PySpark 4.0 coverage
@@ -0,0 +1,142 @@
1
+ Metadata-Version: 2.4
2
+ Name: mock-pyspark
3
+ Version: 0.0.1.dev20260804
4
+ Summary: Drop-in mock for Apache PySpark 4.x — Arrow-backed, no JVM required
5
+ Author-email: Peter Dowdy <peter.dowdy@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/PeterDowdy/mock-pyspark
8
+ Project-URL: Repository, https://github.com/PeterDowdy/mock-pyspark
9
+ Project-URL: Issues, https://github.com/PeterDowdy/mock-pyspark/issues
10
+ Keywords: pyspark,mock,testing,arrow,duckdb
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Topic :: Software Development :: Testing
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Operating System :: OS Independent
17
+ Requires-Python: >=3.9
18
+ Description-Content-Type: text/markdown
19
+ Requires-Dist: pyarrow>=14
20
+ Requires-Dist: duckdb>=1.0
21
+ Requires-Dist: pandas>=2.0
22
+ Requires-Dist: numpy>=1.24
23
+ Provides-Extra: avro
24
+ Requires-Dist: fastavro>=1.9; extra == "avro"
25
+ Provides-Extra: delta
26
+ Requires-Dist: deltalake>=1.0; extra == "delta"
27
+ Provides-Extra: all
28
+ Requires-Dist: mock-pyspark[avro,delta]; extra == "all"
29
+
30
+ # mockpyspark
31
+
32
+ A drop-in mock for Apache PySpark 4.x for unit and integration testing. Runs entirely in Python — no JVM, no Hadoop, no cluster — using PyArrow as the compute engine and DuckDB for SQL.
33
+
34
+ ```bash
35
+ pip install mock-pyspark
36
+ ```
37
+
38
+ `mockpyspark` (the PyPI distribution is `mock-pyspark`; you still `import mockpyspark`) installs alongside real `pyspark`; there is no namespace collision. Tests opt in via a one-liner so that `from pyspark.sql import ...` in your production code transparently resolves to the mock.
39
+
40
+ ---
41
+
42
+ ## Quick start
43
+
44
+ ```python
45
+ # conftest.py
46
+ import mockpyspark
47
+ mockpyspark.activate()
48
+ ```
49
+
50
+ ```python
51
+ # tests/test_my_pipeline.py
52
+ from pyspark.sql import SparkSession, functions as F
53
+
54
+ def test_normalize(spark=None):
55
+ spark = SparkSession.builder.getOrCreate()
56
+ df = spark.createDataFrame([
57
+ {"id": 1, "name": " alice ", "score": 10},
58
+ {"id": 2, "name": "BOB", "score": 20},
59
+ ])
60
+ out = (
61
+ df.withColumn("name", F.initcap(F.trim(F.col("name"))))
62
+ .filter(F.col("score") > 15)
63
+ .select("id", "name", "score")
64
+ .collect()
65
+ )
66
+ assert out == [(2, "Bob", 20)]
67
+ ```
68
+
69
+ No changes to your Spark logic. The alias just makes `pyspark.*` imports resolve to `mockpyspark.*` for the test process.
70
+
71
+ ---
72
+
73
+ ## Version targeting
74
+
75
+ `mockpyspark` supports both PySpark 4.0.x and 4.1.x. APIs that only exist in 4.1 (e.g. `current_time`, `TimeType`) raise `AttributeError` when running in 4.0 mode.
76
+
77
+ It picks a target version automatically, in order:
78
+
79
+ 1. `PYSPARK_VERSION` env var
80
+ 2. The `pyspark` pin in `pyproject.toml`, `requirements*.txt`, `poetry.lock`, `uv.lock`, `Pipfile.lock`, or `setup.cfg` (walking up from the working directory)
81
+ 3. `SparkSession.builder.config("spark.mock.targetVersion", "4.0").getOrCreate()` — per-session override
82
+ 4. If nothing is found, defaults to the latest supported version and emits a `UserWarning`. Set `MOCKPYSPARK_STRICT_VERSION=1` to raise instead.
83
+
84
+ So if your project pins `pyspark==4.0.1` in `pyproject.toml`, the mock will refuse 4.1-only APIs without any configuration on your part.
85
+
86
+ ---
87
+
88
+ ## Usage modes
89
+
90
+ | Mode | What to do |
91
+ | ---------------------------- | ---------------------------------------------------------------- |
92
+ | **Mock for tests (default)** | `mockpyspark.activate()` in conftest. |
93
+ | **Pytest plugin** | `pytest_plugins = ["mockpyspark.pytest_plugin"]` in conftest. |
94
+ | **Real PySpark for a run** | Pass `--no-mock-pyspark` to pytest, or don't call `activate()`. |
95
+
96
+ The pytest plugin also accepts `--no-mock-pyspark` to bypass activation for a single invocation, useful for running the same suite against real PySpark for integration coverage.
97
+
98
+ See **[docs/usage.md](docs/usage.md)** for the full pattern, including switching between mock and real PySpark in the same suite, and known limitations.
99
+
100
+ ---
101
+
102
+ ## Detecting the mock at runtime
103
+
104
+ ```python
105
+ import pyspark
106
+ if getattr(pyspark, "__mock__", False):
107
+ print("running against mockpyspark")
108
+ ```
109
+
110
+ ---
111
+
112
+ ## Contributing
113
+
114
+ **Run unit tests** (no Java, no Docker):
115
+
116
+ ```bash
117
+ git clone https://github.com/PeterDowdy/mock-pyspark.git && cd mock-pyspark
118
+ pip install pyarrow duckdb pandas numpy pytest
119
+ PYTHONPATH=. pytest -q -m unit
120
+ ```
121
+
122
+ **Run integration tests** against real PySpark (requires Java 17):
123
+
124
+ ```bash
125
+ # Via Docker (easiest)
126
+ ./start.sh 4.1 --integration-test
127
+
128
+ # Or locally with a PySpark venv
129
+ python -m venv /opt/spark-venv && /opt/spark-venv/bin/pip install pyspark==4.1.0
130
+ PYTHONPATH=. PYSPARK_VERSION=4.1 PYSPARK_PYTHON=/opt/spark-venv/bin/python pytest -q -m integration
131
+ ```
132
+
133
+ See **[docs/contributing.md](docs/contributing.md)** for the full setup guide.
134
+
135
+ ---
136
+
137
+ ## API coverage
138
+
139
+ Supports PySpark 4.0.x and 4.1.x — 439+ functions, full DataFrame/Column/Window/Catalog API, CSV/JSON/Parquet IO.
140
+
141
+ - [STATUS-4.1.md](STATUS-4.1.md) — PySpark 4.1 coverage
142
+ - [STATUS-4.0.md](STATUS-4.0.md) — PySpark 4.0 coverage
@@ -0,0 +1,143 @@
1
+ README.md
2
+ pyproject.toml
3
+ ./mockpyspark/__init__.py
4
+ ./mockpyspark/_aliasing.py
5
+ ./mockpyspark/pytest_plugin.py
6
+ ./mockpyspark/rdd.py
7
+ ./mockpyspark/_internal/__init__.py
8
+ ./mockpyspark/_internal/config.py
9
+ ./mockpyspark/_internal/engine.py
10
+ ./mockpyspark/_internal/version_detect.py
11
+ ./mockpyspark/_internal/version_guard.py
12
+ ./mockpyspark/delta/__init__.py
13
+ ./mockpyspark/delta/pip.py
14
+ ./mockpyspark/delta/pip_utils.py
15
+ ./mockpyspark/delta/tables.py
16
+ ./mockpyspark/errors/__init__.py
17
+ ./mockpyspark/ml/__init__.py
18
+ ./mockpyspark/pandas/__init__.py
19
+ ./mockpyspark/sql/__init__.py
20
+ ./mockpyspark/sql/_map_utils.py
21
+ ./mockpyspark/sql/catalog.py
22
+ ./mockpyspark/sql/column.py
23
+ ./mockpyspark/sql/dataframe.py
24
+ ./mockpyspark/sql/functions.py
25
+ ./mockpyspark/sql/functions_all.py
26
+ ./mockpyspark/sql/readwriter.py
27
+ ./mockpyspark/sql/row.py
28
+ ./mockpyspark/sql/session.py
29
+ ./mockpyspark/sql/sql_dispatcher.py
30
+ ./mockpyspark/sql/types.py
31
+ ./mockpyspark/sql/utils.py
32
+ ./mockpyspark/sql/window.py
33
+ ./mockpyspark/sql/functions/__init__.py
34
+ ./mockpyspark/sql/functions/aggregates.py
35
+ ./mockpyspark/sql/functions/arrays.py
36
+ ./mockpyspark/sql/functions/bits.py
37
+ ./mockpyspark/sql/functions/core.py
38
+ ./mockpyspark/sql/functions/crypto.py
39
+ ./mockpyspark/sql/functions/csvs.py
40
+ ./mockpyspark/sql/functions/dates.py
41
+ ./mockpyspark/sql/functions/generators.py
42
+ ./mockpyspark/sql/functions/geospatial.py
43
+ ./mockpyspark/sql/functions/hashes.py
44
+ ./mockpyspark/sql/functions/jsons.py
45
+ ./mockpyspark/sql/functions/maps.py
46
+ ./mockpyspark/sql/functions/maths.py
47
+ ./mockpyspark/sql/functions/partitioning.py
48
+ ./mockpyspark/sql/functions/predicates.py
49
+ ./mockpyspark/sql/functions/strings.py
50
+ ./mockpyspark/sql/functions/udfs.py
51
+ ./mockpyspark/sql/functions/urls.py
52
+ ./mockpyspark/sql/functions/variants.py
53
+ ./mockpyspark/sql/functions/windows.py
54
+ ./mockpyspark/sql/functions/xmls.py
55
+ ./mockpyspark/sql/io/__init__.py
56
+ ./mockpyspark/sql/io/_arrow_types.py
57
+ ./mockpyspark/sql/io/avro.py
58
+ ./mockpyspark/sql/io/base.py
59
+ ./mockpyspark/sql/io/binary_file.py
60
+ ./mockpyspark/sql/io/csv.py
61
+ ./mockpyspark/sql/io/delta.py
62
+ ./mockpyspark/sql/io/json.py
63
+ ./mockpyspark/sql/io/orc.py
64
+ ./mockpyspark/sql/io/parquet.py
65
+ ./mockpyspark/sql/io/text.py
66
+ ./mockpyspark/sql/streaming/__init__.py
67
+ ./mockpyspark/sql/streaming/readwriter.py
68
+ ./mockpyspark/streaming/__init__.py
69
+ ./mockpyspark/testing/__init__.py
70
+ ./mockpyspark/testing/utils.py
71
+ mock_pyspark.egg-info/PKG-INFO
72
+ mock_pyspark.egg-info/SOURCES.txt
73
+ mock_pyspark.egg-info/dependency_links.txt
74
+ mock_pyspark.egg-info/requires.txt
75
+ mock_pyspark.egg-info/top_level.txt
76
+ mockpyspark/__init__.py
77
+ mockpyspark/_aliasing.py
78
+ mockpyspark/pytest_plugin.py
79
+ mockpyspark/rdd.py
80
+ mockpyspark/_internal/__init__.py
81
+ mockpyspark/_internal/config.py
82
+ mockpyspark/_internal/engine.py
83
+ mockpyspark/_internal/version_detect.py
84
+ mockpyspark/_internal/version_guard.py
85
+ mockpyspark/delta/__init__.py
86
+ mockpyspark/delta/pip.py
87
+ mockpyspark/delta/pip_utils.py
88
+ mockpyspark/delta/tables.py
89
+ mockpyspark/errors/__init__.py
90
+ mockpyspark/ml/__init__.py
91
+ mockpyspark/pandas/__init__.py
92
+ mockpyspark/sql/__init__.py
93
+ mockpyspark/sql/_map_utils.py
94
+ mockpyspark/sql/catalog.py
95
+ mockpyspark/sql/column.py
96
+ mockpyspark/sql/dataframe.py
97
+ mockpyspark/sql/functions.py
98
+ mockpyspark/sql/functions_all.py
99
+ mockpyspark/sql/readwriter.py
100
+ mockpyspark/sql/row.py
101
+ mockpyspark/sql/session.py
102
+ mockpyspark/sql/sql_dispatcher.py
103
+ mockpyspark/sql/types.py
104
+ mockpyspark/sql/utils.py
105
+ mockpyspark/sql/window.py
106
+ mockpyspark/sql/functions/__init__.py
107
+ mockpyspark/sql/functions/aggregates.py
108
+ mockpyspark/sql/functions/arrays.py
109
+ mockpyspark/sql/functions/bits.py
110
+ mockpyspark/sql/functions/core.py
111
+ mockpyspark/sql/functions/crypto.py
112
+ mockpyspark/sql/functions/csvs.py
113
+ mockpyspark/sql/functions/dates.py
114
+ mockpyspark/sql/functions/generators.py
115
+ mockpyspark/sql/functions/geospatial.py
116
+ mockpyspark/sql/functions/hashes.py
117
+ mockpyspark/sql/functions/jsons.py
118
+ mockpyspark/sql/functions/maps.py
119
+ mockpyspark/sql/functions/maths.py
120
+ mockpyspark/sql/functions/partitioning.py
121
+ mockpyspark/sql/functions/predicates.py
122
+ mockpyspark/sql/functions/strings.py
123
+ mockpyspark/sql/functions/udfs.py
124
+ mockpyspark/sql/functions/urls.py
125
+ mockpyspark/sql/functions/variants.py
126
+ mockpyspark/sql/functions/windows.py
127
+ mockpyspark/sql/functions/xmls.py
128
+ mockpyspark/sql/io/__init__.py
129
+ mockpyspark/sql/io/_arrow_types.py
130
+ mockpyspark/sql/io/avro.py
131
+ mockpyspark/sql/io/base.py
132
+ mockpyspark/sql/io/binary_file.py
133
+ mockpyspark/sql/io/csv.py
134
+ mockpyspark/sql/io/delta.py
135
+ mockpyspark/sql/io/json.py
136
+ mockpyspark/sql/io/orc.py
137
+ mockpyspark/sql/io/parquet.py
138
+ mockpyspark/sql/io/text.py
139
+ mockpyspark/sql/streaming/__init__.py
140
+ mockpyspark/sql/streaming/readwriter.py
141
+ mockpyspark/streaming/__init__.py
142
+ mockpyspark/testing/__init__.py
143
+ mockpyspark/testing/utils.py
@@ -0,0 +1,13 @@
1
+ pyarrow>=14
2
+ duckdb>=1.0
3
+ pandas>=2.0
4
+ numpy>=1.24
5
+
6
+ [all]
7
+ mock-pyspark[avro,delta]
8
+
9
+ [avro]
10
+ fastavro>=1.9
11
+
12
+ [delta]
13
+ deltalake>=1.0
@@ -0,0 +1,33 @@
1
+ """mockpyspark — drop-in mock for Apache PySpark 4.x for testing.
2
+
3
+ Activate the import alias so production code's `from pyspark.sql import ...`
4
+ resolves to the mock without uninstalling real PySpark::
5
+
6
+ import mockpyspark
7
+ mockpyspark.activate()
8
+
9
+ Or, in pytest, load the bundled plugin::
10
+
11
+ # conftest.py
12
+ pytest_plugins = ["mockpyspark.pytest_plugin"]
13
+ """
14
+ from __future__ import annotations
15
+
16
+ from ._aliasing import activate, deactivate, is_active
17
+ from .rdd import RDD
18
+ from .sql.session import SparkSession
19
+
20
+ __all__ = ["SparkSession", "RDD", "activate", "deactivate", "is_active"]
21
+
22
+ # Sentinel: lets user code detect at runtime that the loaded `pyspark` is the
23
+ # mock without depending on file paths or install layout.
24
+ __mock__ = True
25
+
26
+
27
+ def __getattr__(name: str):
28
+ # Lazy so `pyspark.__version__` always reflects the current target version,
29
+ # even after it changes via env var or SparkSession builder config.
30
+ if name == "__version__":
31
+ from ._internal.config import CONFIG
32
+ return CONFIG.target_version
33
+ raise AttributeError(f"module 'mockpyspark' has no attribute {name!r}")