sparkless-testing 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sparkless_testing/__init__.py +87 -0
- sparkless_testing/conftest.py +39 -0
- sparkless_testing/engine.py +306 -0
- sparkless_testing/parametrize.py +150 -0
- sparkless_testing/protocols.py +145 -0
- sparkless_testing/py.typed +0 -0
- sparkless_testing/pytest_fixtures.py +235 -0
- sparkless_testing/session.py +262 -0
- sparkless_testing/utils.py +209 -0
- sparkless_testing-0.1.0.dist-info/METADATA +445 -0
- sparkless_testing-0.1.0.dist-info/RECORD +14 -0
- sparkless_testing-0.1.0.dist-info/WHEEL +5 -0
- sparkless_testing-0.1.0.dist-info/licenses/LICENSE +22 -0
- sparkless_testing-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Test utilities for sparkless-testing.
|
|
3
|
+
|
|
4
|
+
Helper functions for common test patterns and compatibility handling.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from .engine import get_engine
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def detect_spark_type(spark: Any) -> str:
|
|
16
|
+
"""
|
|
17
|
+
Detect if spark session is PySpark or mock.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
spark: SparkSession instance to check
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
'pyspark', 'mock', or 'unknown'
|
|
24
|
+
"""
|
|
25
|
+
# Fast-path: PySpark sessions have a JVM bridge
|
|
26
|
+
if hasattr(spark, "sparkContext") and hasattr(spark.sparkContext, "_jsc"):
|
|
27
|
+
return "pyspark"
|
|
28
|
+
|
|
29
|
+
try:
|
|
30
|
+
spark_module = type(spark).__module__
|
|
31
|
+
if "pyspark" in spark_module:
|
|
32
|
+
return "pyspark"
|
|
33
|
+
# Detect sparkless/mock sessions by module path
|
|
34
|
+
if "sparkless" in spark_module or "mock" in spark_module:
|
|
35
|
+
return "mock"
|
|
36
|
+
except Exception:
|
|
37
|
+
pass
|
|
38
|
+
|
|
39
|
+
# Fallback to engine name if available
|
|
40
|
+
try:
|
|
41
|
+
engine_config = get_engine()
|
|
42
|
+
engine_name = engine_config.engine_name
|
|
43
|
+
if engine_name in {"mock", "sparkless"}:
|
|
44
|
+
return "mock"
|
|
45
|
+
if engine_name == "pyspark":
|
|
46
|
+
return "pyspark"
|
|
47
|
+
except Exception:
|
|
48
|
+
pass
|
|
49
|
+
|
|
50
|
+
return "unknown"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def is_dataframe_like(obj: Any) -> bool:
|
|
54
|
+
"""
|
|
55
|
+
Check if object is DataFrame-like using structural typing.
|
|
56
|
+
|
|
57
|
+
Checks for essential DataFrame methods: count, columns (property), filter.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
obj: Object to check
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
True if object has DataFrame-like interface, False otherwise
|
|
64
|
+
"""
|
|
65
|
+
# columns is typically a property (not callable), count and filter are methods
|
|
66
|
+
return (
|
|
67
|
+
hasattr(obj, "count")
|
|
68
|
+
and hasattr(obj, "columns")
|
|
69
|
+
and hasattr(obj, "filter")
|
|
70
|
+
and callable(getattr(obj, "count", None))
|
|
71
|
+
and callable(getattr(obj, "filter", None))
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def create_test_dataframe(
|
|
76
|
+
spark: Any,
|
|
77
|
+
data: Any,
|
|
78
|
+
schema: Any | None = None,
|
|
79
|
+
**kwargs: Any,
|
|
80
|
+
) -> Any:
|
|
81
|
+
"""
|
|
82
|
+
High-level helper for creating test DataFrames.
|
|
83
|
+
|
|
84
|
+
Provides a consistent API for creating DataFrames in tests.
|
|
85
|
+
Handles PySpark 3.5+ schema argument position differences.
|
|
86
|
+
|
|
87
|
+
Args:
|
|
88
|
+
spark: SparkSession instance (PySpark or mock-spark)
|
|
89
|
+
data: Data to create DataFrame from (list of tuples, list of dicts, etc.)
|
|
90
|
+
schema: Schema definition (list of strings, StructType, or None)
|
|
91
|
+
**kwargs: Additional arguments passed to createDataFrame
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
DataFrame instance
|
|
95
|
+
|
|
96
|
+
Raises:
|
|
97
|
+
TypeError: If spark is not a valid SparkSession
|
|
98
|
+
ValueError: If data is invalid or schema is incompatible
|
|
99
|
+
RuntimeError: If DataFrame creation fails after all retry attempts
|
|
100
|
+
"""
|
|
101
|
+
if not hasattr(spark, "createDataFrame"):
|
|
102
|
+
raise TypeError(
|
|
103
|
+
f"spark parameter must have a createDataFrame method, got {type(spark).__name__}"
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
create_df = spark.createDataFrame
|
|
107
|
+
last_error: Exception | None = None
|
|
108
|
+
|
|
109
|
+
# Call createDataFrame method
|
|
110
|
+
# Handle PySpark 3.5+ schema argument issues
|
|
111
|
+
if schema is None:
|
|
112
|
+
try:
|
|
113
|
+
return create_df(data, **kwargs)
|
|
114
|
+
except (TypeError, ValueError) as e:
|
|
115
|
+
raise ValueError(
|
|
116
|
+
f"Failed to create DataFrame from data: {e}. Data type: {type(data).__name__}"
|
|
117
|
+
) from e
|
|
118
|
+
except Exception as e:
|
|
119
|
+
raise RuntimeError(f"Unexpected error creating DataFrame: {e}") from e
|
|
120
|
+
else:
|
|
121
|
+
# Try different calling patterns to handle PySpark version differences
|
|
122
|
+
# Pattern 1: Positional schema (PySpark 3.5+)
|
|
123
|
+
try:
|
|
124
|
+
if kwargs:
|
|
125
|
+
return create_df(data, schema, **kwargs)
|
|
126
|
+
else:
|
|
127
|
+
return create_df(data, schema)
|
|
128
|
+
except (TypeError, ValueError) as e:
|
|
129
|
+
error_str = str(e)
|
|
130
|
+
last_error = e
|
|
131
|
+
|
|
132
|
+
# Check if this is the PySpark StructType error (known PySpark 3.5+ bug)
|
|
133
|
+
if "NOT_LIST_OR_NONE_OR_STRUCT" in error_str:
|
|
134
|
+
# Try without kwargs
|
|
135
|
+
try:
|
|
136
|
+
return create_df(data, schema)
|
|
137
|
+
except (TypeError, ValueError) as e2:
|
|
138
|
+
last_error = e2
|
|
139
|
+
# Last resort: try with schema as keyword
|
|
140
|
+
try:
|
|
141
|
+
return create_df(data, schema=schema, **kwargs)
|
|
142
|
+
except (TypeError, ValueError) as e3:
|
|
143
|
+
last_error = e3
|
|
144
|
+
# Final fallback: try without kwargs and keyword schema
|
|
145
|
+
try:
|
|
146
|
+
return create_df(data, schema=schema)
|
|
147
|
+
except (TypeError, ValueError) as e4:
|
|
148
|
+
last_error = e4
|
|
149
|
+
else:
|
|
150
|
+
# For other errors, try keyword argument (older PySpark)
|
|
151
|
+
try:
|
|
152
|
+
return create_df(data, schema=schema, **kwargs)
|
|
153
|
+
except (TypeError, ValueError) as e2:
|
|
154
|
+
last_error = e2
|
|
155
|
+
# Final fallback: try without kwargs and keyword schema
|
|
156
|
+
try:
|
|
157
|
+
return create_df(data, schema=schema)
|
|
158
|
+
except (TypeError, ValueError) as e3:
|
|
159
|
+
last_error = e3
|
|
160
|
+
|
|
161
|
+
# If we get here, all attempts failed
|
|
162
|
+
if last_error:
|
|
163
|
+
raise ValueError(
|
|
164
|
+
f"Failed to create DataFrame with schema after multiple attempts. "
|
|
165
|
+
f"Last error: {last_error}. "
|
|
166
|
+
f"Data type: {type(data).__name__}, Schema type: {type(schema).__name__}"
|
|
167
|
+
) from last_error
|
|
168
|
+
else:
|
|
169
|
+
raise RuntimeError("Failed to create DataFrame: unknown error occurred")
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def get_spark_mode() -> str:
|
|
173
|
+
"""
|
|
174
|
+
Get SPARK_MODE from environment variable.
|
|
175
|
+
|
|
176
|
+
Returns:
|
|
177
|
+
'mock', 'pyspark', or 'auto' (default)
|
|
178
|
+
"""
|
|
179
|
+
return os.environ.get("SPARK_MODE", "auto").lower()
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def get_warehouse_dir() -> str | None:
|
|
183
|
+
"""
|
|
184
|
+
Get custom warehouse directory from environment variable.
|
|
185
|
+
|
|
186
|
+
Returns:
|
|
187
|
+
Warehouse directory path or None
|
|
188
|
+
"""
|
|
189
|
+
return os.environ.get("SPARK_TEST_WAREHOUSE_DIR")
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def get_app_name_prefix() -> str:
|
|
193
|
+
"""
|
|
194
|
+
Get app name prefix from environment variable.
|
|
195
|
+
|
|
196
|
+
Returns:
|
|
197
|
+
App name prefix (default: 'sparkless-testing')
|
|
198
|
+
"""
|
|
199
|
+
return os.environ.get("SPARK_TEST_APP_NAME", "sparkless-testing")
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
__all__ = [
|
|
203
|
+
"detect_spark_type",
|
|
204
|
+
"is_dataframe_like",
|
|
205
|
+
"create_test_dataframe",
|
|
206
|
+
"get_spark_mode",
|
|
207
|
+
"get_warehouse_dir",
|
|
208
|
+
"get_app_name_prefix",
|
|
209
|
+
]
|
|
@@ -0,0 +1,445 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sparkless-testing
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Easy testing with sparkless or PySpark on demand
|
|
5
|
+
Author-email: Odos Matthews <odosmatthews@gmail.com>
|
|
6
|
+
Maintainer-email: Odos Matthews <odosmatthews@gmail.com>
|
|
7
|
+
License: MIT
|
|
8
|
+
Project-URL: Homepage, https://github.com/eddiethedean/sparkless-testing
|
|
9
|
+
Project-URL: Repository, https://github.com/eddiethedean/sparkless-testing
|
|
10
|
+
Project-URL: Issues, https://github.com/eddiethedean/sparkless-testing/issues
|
|
11
|
+
Keywords: spark,pyspark,testing,pytest,sparkless,data-engineering,unit-testing,fixtures
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Topic :: Software Development :: Testing
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Requires-Python: >=3.9
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: pytest>=7.0.0
|
|
26
|
+
Provides-Extra: mock
|
|
27
|
+
Requires-Dist: sparkless>=3.19.0; extra == "mock"
|
|
28
|
+
Provides-Extra: pyspark
|
|
29
|
+
Requires-Dist: pyspark<3.6.0,>=3.5.0; extra == "pyspark"
|
|
30
|
+
Provides-Extra: delta
|
|
31
|
+
Requires-Dist: delta-spark<4.0.0,>=3.0.0; extra == "delta"
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
34
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
|
35
|
+
Requires-Dist: mypy>=1.0.0; extra == "dev"
|
|
36
|
+
Requires-Dist: ruff>=0.4.0; extra == "dev"
|
|
37
|
+
Requires-Dist: sparkless>=3.19.0; extra == "dev"
|
|
38
|
+
Requires-Dist: pyspark<3.6.0,>=3.5.0; extra == "dev"
|
|
39
|
+
Requires-Dist: delta-spark<4.0.0,>=3.0.0; extra == "dev"
|
|
40
|
+
Dynamic: license-file
|
|
41
|
+
|
|
42
|
+
# sparkless-testing
|
|
43
|
+
|
|
44
|
+
> **Easy testing with sparkless or PySpark on demand**
|
|
45
|
+
|
|
46
|
+
A Python package that simplifies running tests with either **sparkless** (mock) or **PySpark** (real) engines on demand. Write tests once, run them with either engine seamlessly.
|
|
47
|
+
|
|
48
|
+
## Why sparkless-testing?
|
|
49
|
+
|
|
50
|
+
- 🚀 **Fast Mock Testing**: Use `sparkless` for lightning-fast unit tests without JVM overhead
|
|
51
|
+
- 🔄 **Real Engine Validation**: Test against real PySpark to catch integration issues
|
|
52
|
+
- 🎯 **Single Test Suite**: Write tests once, run with both engines automatically
|
|
53
|
+
- 🔧 **Zero Configuration**: Automatic engine detection and session management
|
|
54
|
+
- ⚡ **Parallel Ready**: Optimized for parallel test execution with pytest-xdist
|
|
55
|
+
|
|
56
|
+
## Features
|
|
57
|
+
|
|
58
|
+
- **Automatic Engine Detection** - Automatically detects and configures available engines
|
|
59
|
+
- **Pytest Fixtures** - Ready-to-use fixtures for both engines
|
|
60
|
+
- **Session Management** - Automatic session creation, cleanup, and isolation
|
|
61
|
+
- **Test Utilities** - Helpers for common test patterns
|
|
62
|
+
- **Parametrization Support** - Run tests with both engines automatically
|
|
63
|
+
- **Parallel Testing Support** - Optimized for parallel test execution with pytest-xdist
|
|
64
|
+
|
|
65
|
+
## Installation
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
pip install sparkless-testing
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
For specific engines:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
# Install with mock engine (sparkless)
|
|
75
|
+
pip install sparkless-testing[mock]
|
|
76
|
+
|
|
77
|
+
# Install with PySpark engine
|
|
78
|
+
pip install sparkless-testing[pyspark]
|
|
79
|
+
|
|
80
|
+
# Install with Delta Lake support
|
|
81
|
+
pip install sparkless-testing[delta]
|
|
82
|
+
|
|
83
|
+
# Install with all engines
|
|
84
|
+
pip install sparkless-testing[dev]
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## Quick Start
|
|
88
|
+
|
|
89
|
+
### Basic Usage
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
import pytest
|
|
93
|
+
from sparkless_testing import pytest_fixtures
|
|
94
|
+
|
|
95
|
+
# Use the spark_session fixture
|
|
96
|
+
def test_my_function(spark_session):
|
|
97
|
+
df = spark_session.createDataFrame([{"id": 1, "name": "Alice"}])
|
|
98
|
+
assert df.count() == 1
|
|
99
|
+
assert df.collect()[0]["name"] == "Alice"
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### Explicit Engine Selection
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
def test_mock_only(mock_spark_session):
|
|
106
|
+
# Only runs with sparkless
|
|
107
|
+
df = mock_spark_session.createDataFrame([{"id": 1}])
|
|
108
|
+
assert df.count() == 1
|
|
109
|
+
|
|
110
|
+
def test_pyspark_only(pyspark_session):
|
|
111
|
+
# Only runs with PySpark
|
|
112
|
+
df = pyspark_session.createDataFrame([{"id": 1}])
|
|
113
|
+
assert df.count() == 1
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
### Using Functions and Types
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
def test_with_functions(spark_session, spark_functions, spark_types):
|
|
120
|
+
F = spark_functions
|
|
121
|
+
Types = spark_types
|
|
122
|
+
|
|
123
|
+
df = spark_session.createDataFrame([{"name": "Alice"}])
|
|
124
|
+
result = df.select(F.upper(F.col("name"))).collect()
|
|
125
|
+
assert result[0][0] == "ALICE"
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
**Note**: Always use `spark_functions` together with `spark_session`. PySpark functions like `F.col()` require an active SparkContext, which is provided by the `spark_session` fixture.
|
|
129
|
+
|
|
130
|
+
### Parametrized Tests (Both Engines)
|
|
131
|
+
|
|
132
|
+
You can run tests with both engines in two ways:
|
|
133
|
+
|
|
134
|
+
**Option 1: Using the decorator**
|
|
135
|
+
```python
|
|
136
|
+
@pytest.mark.parametrize_engines
|
|
137
|
+
def test_both_engines(spark_session):
|
|
138
|
+
# Runs with both engines automatically
|
|
139
|
+
df = spark_session.createDataFrame([{"id": 1}])
|
|
140
|
+
assert df.count() == 1
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
**Option 2: Using the `both_engines` fixture**
|
|
144
|
+
```python
|
|
145
|
+
def test_both_engines(both_engines):
|
|
146
|
+
spark, engine_type = both_engines
|
|
147
|
+
# Runs with both engines automatically
|
|
148
|
+
df = spark.createDataFrame([{"id": 1}])
|
|
149
|
+
assert df.count() == 1
|
|
150
|
+
# engine_type will be EngineType.MOCK or EngineType.PYSPARK
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
### Environment-Based Configuration
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
# Run tests with mock engine
|
|
157
|
+
SPARK_MODE=mock pytest tests/
|
|
158
|
+
|
|
159
|
+
# Run tests with PySpark engine
|
|
160
|
+
SPARK_MODE=pyspark pytest tests/
|
|
161
|
+
|
|
162
|
+
# Auto-detect (default)
|
|
163
|
+
SPARK_MODE=auto pytest tests/
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
### Pytest Markers
|
|
167
|
+
|
|
168
|
+
```python
|
|
169
|
+
@pytest.mark.spark_engine("mock")
|
|
170
|
+
def test_mock_specific(spark_session):
|
|
171
|
+
# Forces mock engine
|
|
172
|
+
pass
|
|
173
|
+
|
|
174
|
+
@pytest.mark.spark_engine("pyspark")
|
|
175
|
+
def test_pyspark_specific(spark_session):
|
|
176
|
+
# Forces PySpark engine
|
|
177
|
+
pass
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
## Configuration
|
|
181
|
+
|
|
182
|
+
### Environment Variables
|
|
183
|
+
|
|
184
|
+
- `SPARK_MODE`: `mock`, `pyspark`, or `auto` (default: `auto`)
|
|
185
|
+
- `SPARK_TEST_WAREHOUSE_DIR`: Custom warehouse directory for PySpark tests
|
|
186
|
+
- `SPARK_TEST_APP_NAME`: Custom app name prefix (default: `sparkless-testing`)
|
|
187
|
+
|
|
188
|
+
### Programmatic Configuration
|
|
189
|
+
|
|
190
|
+
```python
|
|
191
|
+
from sparkless_testing import (
|
|
192
|
+
EngineType,
|
|
193
|
+
SessionConfig,
|
|
194
|
+
auto_configure_engine,
|
|
195
|
+
create_session,
|
|
196
|
+
create_pyspark_session,
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
# Auto-configure based on availability
|
|
200
|
+
auto_configure_engine()
|
|
201
|
+
|
|
202
|
+
# Or specify preferred engine
|
|
203
|
+
auto_configure_engine(EngineType.MOCK)
|
|
204
|
+
|
|
205
|
+
# Create session manually
|
|
206
|
+
spark = create_session(engine_type=EngineType.MOCK)
|
|
207
|
+
|
|
208
|
+
# Create session with custom configuration
|
|
209
|
+
config = SessionConfig(
|
|
210
|
+
app_name="my-test",
|
|
211
|
+
enable_delta=True,
|
|
212
|
+
shuffle_partitions=2,
|
|
213
|
+
)
|
|
214
|
+
spark = create_pyspark_session(config=config)
|
|
215
|
+
# ... use spark ...
|
|
216
|
+
spark.stop()
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
## API Reference
|
|
220
|
+
|
|
221
|
+
### Engine Configuration
|
|
222
|
+
|
|
223
|
+
- `EngineType`: Enum for engine types (`MOCK`, `PYSPARK`, `AUTO`)
|
|
224
|
+
- `configure_engine()`: Manually configure engine components
|
|
225
|
+
- `get_engine()`: Get current engine configuration
|
|
226
|
+
- `detect_available_engines()`: Check which engines are installed
|
|
227
|
+
- `auto_configure_engine()`: Automatically configure based on availability
|
|
228
|
+
|
|
229
|
+
### Session Factory
|
|
230
|
+
|
|
231
|
+
- `create_mock_session()`: Create sparkless session
|
|
232
|
+
- `create_pyspark_session()`: Create PySpark session
|
|
233
|
+
- `create_session()`: Factory function that creates appropriate session
|
|
234
|
+
- `SessionConfig`: Configuration dataclass for session creation
|
|
235
|
+
- `app_name`: Application name for the Spark session
|
|
236
|
+
- `warehouse_dir`: Custom warehouse directory (optional)
|
|
237
|
+
- `enable_delta`: Enable Delta Lake support (default: `True` for programmatic use, `False` in test fixtures)
|
|
238
|
+
- `enable_ui`: Enable Spark UI (default: `False`)
|
|
239
|
+
- `shuffle_partitions`: Number of shuffle partitions (default: `1`)
|
|
240
|
+
- `parallelism`: Default parallelism (default: `1`)
|
|
241
|
+
- `adaptive_enabled`: Enable adaptive query execution (default: `False`)
|
|
242
|
+
|
|
243
|
+
### Test Utilities
|
|
244
|
+
|
|
245
|
+
- `detect_spark_type(spark)`: Detect if session is PySpark or mock
|
|
246
|
+
- `create_test_dataframe(spark, data, schema)`: Compatibility wrapper for DataFrame creation
|
|
247
|
+
- `is_dataframe_like(obj)`: Check if object is DataFrame-like
|
|
248
|
+
|
|
249
|
+
### Pytest Fixtures
|
|
250
|
+
|
|
251
|
+
- `spark_session`: Main fixture (auto-detects engine based on `SPARK_MODE` or markers)
|
|
252
|
+
- `mock_spark_session`: Explicitly use sparkless (mock) engine
|
|
253
|
+
- `pyspark_session`: Explicitly use PySpark (real) engine
|
|
254
|
+
- `both_engines`: Fixture that yields `(spark, engine_type)` for both engines (parametrized)
|
|
255
|
+
- `spark_functions`: Functions module (F) for current engine
|
|
256
|
+
- `spark_types`: Types module for current engine
|
|
257
|
+
- `spark_engine_type`: Current engine type as string (`"mock"` or `"pyspark"`)
|
|
258
|
+
|
|
259
|
+
**Note**: Test fixtures automatically disable Delta Lake for parallel testing compatibility. To use Delta Lake, create sessions programmatically with `enable_delta=True` in `SessionConfig`.
|
|
260
|
+
|
|
261
|
+
## Advanced Usage
|
|
262
|
+
|
|
263
|
+
### Custom Session Configuration
|
|
264
|
+
|
|
265
|
+
```python
|
|
266
|
+
from sparkless_testing import SessionConfig, create_pyspark_session
|
|
267
|
+
|
|
268
|
+
def test_with_custom_config():
|
|
269
|
+
config = SessionConfig(
|
|
270
|
+
app_name="custom-test",
|
|
271
|
+
warehouse_dir="/tmp/my-warehouse",
|
|
272
|
+
enable_delta=True,
|
|
273
|
+
enable_ui=True,
|
|
274
|
+
shuffle_partitions=4,
|
|
275
|
+
parallelism=4,
|
|
276
|
+
)
|
|
277
|
+
spark = create_pyspark_session(config=config)
|
|
278
|
+
# ... your test code ...
|
|
279
|
+
spark.stop()
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
### Parallel Test Execution
|
|
283
|
+
|
|
284
|
+
```bash
|
|
285
|
+
# Run tests in parallel with pytest-xdist
|
|
286
|
+
pytest -n 2 # Use 2 workers
|
|
287
|
+
pytest -n auto # Auto-detect worker count
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
**Best Practices for Parallel Testing:**
|
|
291
|
+
- Mock (sparkless) tests can run with high parallelism (`-n 10+`)
|
|
292
|
+
- PySpark tests work best with fewer workers (`-n 2` or `-n 4`)
|
|
293
|
+
- Test fixtures automatically disable Delta Lake to avoid conflicts
|
|
294
|
+
- Each test gets a unique session name and warehouse directory
|
|
295
|
+
|
|
296
|
+
### Using Test Utilities
|
|
297
|
+
|
|
298
|
+
```python
|
|
299
|
+
from sparkless_testing.utils import create_test_dataframe, detect_spark_type
|
|
300
|
+
|
|
301
|
+
def test_with_utilities(spark_session):
|
|
302
|
+
# Create DataFrame with schema handling
|
|
303
|
+
data = [("Alice", 25), ("Bob", 30)]
|
|
304
|
+
schema = ["name", "age"]
|
|
305
|
+
df = create_test_dataframe(spark_session, data, schema)
|
|
306
|
+
|
|
307
|
+
# Detect engine type
|
|
308
|
+
engine = detect_spark_type(spark_session)
|
|
309
|
+
assert engine in ("mock", "pyspark")
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
## Examples
|
|
313
|
+
|
|
314
|
+
### Complete Test Example
|
|
315
|
+
|
|
316
|
+
```python
|
|
317
|
+
import pytest
|
|
318
|
+
from sparkless_testing.pytest_fixtures import spark_session, spark_functions
|
|
319
|
+
|
|
320
|
+
def test_data_transformation(spark_session, spark_functions):
|
|
321
|
+
F = spark_functions
|
|
322
|
+
|
|
323
|
+
# Create test data
|
|
324
|
+
data = [
|
|
325
|
+
{"id": 1, "name": "Alice", "age": 25},
|
|
326
|
+
{"id": 2, "name": "Bob", "age": 30},
|
|
327
|
+
]
|
|
328
|
+
df = spark_session.createDataFrame(data)
|
|
329
|
+
|
|
330
|
+
# Transform data
|
|
331
|
+
result = (
|
|
332
|
+
df.filter(F.col("age") > 25)
|
|
333
|
+
.select("name", "age")
|
|
334
|
+
.collect()
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
# Assertions
|
|
338
|
+
assert len(result) == 1
|
|
339
|
+
assert result[0]["name"] == "Bob"
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
### Using with Both Engines
|
|
343
|
+
|
|
344
|
+
```python
|
|
345
|
+
@pytest.mark.parametrize_engines
|
|
346
|
+
def test_compatibility(spark_session, spark_engine_type):
|
|
347
|
+
# This test runs with both engines
|
|
348
|
+
df = spark_session.createDataFrame([{"value": 42}])
|
|
349
|
+
|
|
350
|
+
# Engine-specific logic if needed
|
|
351
|
+
if spark_engine_type == "mock":
|
|
352
|
+
# Mock-specific assertions
|
|
353
|
+
pass
|
|
354
|
+
else:
|
|
355
|
+
# PySpark-specific assertions
|
|
356
|
+
pass
|
|
357
|
+
|
|
358
|
+
assert df.count() == 1
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
## Troubleshooting
|
|
362
|
+
|
|
363
|
+
### Tests Hang with Parallel Execution
|
|
364
|
+
|
|
365
|
+
If tests hang when using `pytest -n`, try:
|
|
366
|
+
- Use fewer workers: `pytest -n 2` instead of `pytest -n 10`
|
|
367
|
+
- Run PySpark tests sequentially: `pytest -n 0` for PySpark-only tests
|
|
368
|
+
- Use mock engine for faster parallel execution: `SPARK_MODE=mock pytest -n 10`
|
|
369
|
+
|
|
370
|
+
### Delta Lake Not Available
|
|
371
|
+
|
|
372
|
+
Delta Lake is disabled by default in test fixtures. To enable:
|
|
373
|
+
```python
|
|
374
|
+
from sparkless_testing import SessionConfig, create_pyspark_session
|
|
375
|
+
|
|
376
|
+
config = SessionConfig(app_name="test", enable_delta=True)
|
|
377
|
+
spark = create_pyspark_session(config=config)
|
|
378
|
+
```
|
|
379
|
+
|
|
380
|
+
### SparkContext Errors
|
|
381
|
+
|
|
382
|
+
Always use `spark_functions` together with `spark_session`:
|
|
383
|
+
```python
|
|
384
|
+
def test_correct(spark_session, spark_functions): # ✅ Correct
|
|
385
|
+
F = spark_functions
|
|
386
|
+
df = spark_session.createDataFrame([{"x": 1}])
|
|
387
|
+
result = df.select(F.col("x")).collect()
|
|
388
|
+
|
|
389
|
+
def test_incorrect(spark_functions): # ❌ May fail with PySpark
|
|
390
|
+
F = spark_functions
|
|
391
|
+
# Missing active SparkContext
|
|
392
|
+
```
|
|
393
|
+
|
|
394
|
+
## Migration Guide
|
|
395
|
+
|
|
396
|
+
### From Manual Engine Switching
|
|
397
|
+
|
|
398
|
+
**Before:**
|
|
399
|
+
```python
|
|
400
|
+
import os
|
|
401
|
+
from sparkless import SparkSession as MockSparkSession
|
|
402
|
+
from pyspark.sql import SparkSession as PySparkSession
|
|
403
|
+
|
|
404
|
+
def test_my_function():
|
|
405
|
+
if os.environ.get("SPARK_MODE") == "mock":
|
|
406
|
+
spark = MockSparkSession("test")
|
|
407
|
+
else:
|
|
408
|
+
spark = PySparkSession.builder.appName("test").getOrCreate()
|
|
409
|
+
# ... test code ...
|
|
410
|
+
spark.stop()
|
|
411
|
+
```
|
|
412
|
+
|
|
413
|
+
**After:**
|
|
414
|
+
```python
|
|
415
|
+
from sparkless_testing.pytest_fixtures import spark_session
|
|
416
|
+
|
|
417
|
+
def test_my_function(spark_session):
|
|
418
|
+
# ... test code ...
|
|
419
|
+
# spark_session is automatically cleaned up
|
|
420
|
+
```
|
|
421
|
+
|
|
422
|
+
## Requirements
|
|
423
|
+
|
|
424
|
+
- Python 3.9+
|
|
425
|
+
- pytest 7.0+
|
|
426
|
+
|
|
427
|
+
Optional:
|
|
428
|
+
- sparkless>=3.19.0 (for mock engine)
|
|
429
|
+
- pyspark>=3.5.0 (for real engine)
|
|
430
|
+
- delta-spark>=3.0.0 (for Delta Lake support)
|
|
431
|
+
- pytest-xdist (for parallel test execution)
|
|
432
|
+
|
|
433
|
+
## License
|
|
434
|
+
|
|
435
|
+
MIT License - see [LICENSE](LICENSE) file for details.
|
|
436
|
+
|
|
437
|
+
## Links
|
|
438
|
+
|
|
439
|
+
- **GitHub**: [github.com/eddiethedean/sparkless-testing](https://github.com/eddiethedean/sparkless-testing)
|
|
440
|
+
- **Issues**: [github.com/eddiethedean/sparkless-testing/issues](https://github.com/eddiethedean/sparkless-testing/issues)
|
|
441
|
+
|
|
442
|
+
---
|
|
443
|
+
|
|
444
|
+
**Made with ❤️ for the data engineering community**
|
|
445
|
+
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
sparkless_testing/__init__.py,sha256=cXNVuRYhRyk2BgIpwGHDMu1lLVwnF3JC6MPaSUxEiuA,1839
|
|
2
|
+
sparkless_testing/conftest.py,sha256=TsZDs4Y_-4bQawVrirKpOvHjqN6SDuMwYK9GZ8XOBqc,887
|
|
3
|
+
sparkless_testing/engine.py,sha256=lLlhiCwJzabA4qEfhNgY6_pEVjHiGNvk8qgU-Wcg-3Q,9634
|
|
4
|
+
sparkless_testing/parametrize.py,sha256=Pqbvp2AlsioiuGVJVOlKd5Ldw397tBjlEtkCR9dgTrs,5315
|
|
5
|
+
sparkless_testing/protocols.py,sha256=SzGWpy5extMKxznvT9cyviWwKSJd8ZnH21nCpIq8nSQ,4321
|
|
6
|
+
sparkless_testing/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
sparkless_testing/pytest_fixtures.py,sha256=WvxFqxHabid4gRISliZU9QaohMfUZHx9FCwcXejwCVI,7312
|
|
8
|
+
sparkless_testing/session.py,sha256=UoQT5Kkp2bhiLMfQRw6qDxkQhIItW4h79EOLbHG9l3I,8597
|
|
9
|
+
sparkless_testing/utils.py,sha256=Km1tKXGvhwtdcDfPgvLYZWhzObylObkIJpeD2Bdi-Ks,6561
|
|
10
|
+
sparkless_testing-0.1.0.dist-info/licenses/LICENSE,sha256=7d2e2UstJQ5UphPnHa3djO30rYc3Q92LascNZuSM4J0,1071
|
|
11
|
+
sparkless_testing-0.1.0.dist-info/METADATA,sha256=0-4Pw54wBWkR88E8mpIgGrQ2jcwUbc3RsLcQFS0gFCg,13313
|
|
12
|
+
sparkless_testing-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
13
|
+
sparkless_testing-0.1.0.dist-info/top_level.txt,sha256=hyXY8iujz_qcsS1amyKStVYg7Y0DqmBURJP_EqYKnzk,18
|
|
14
|
+
sparkless_testing-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Odos Matthews
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
22
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
sparkless_testing
|