sparkless-testing 0.1.0__tar.gz → 0.2.0__tar.gz

{sparkless_testing-0.1.0/sparkless_testing.egg-info → sparkless_testing-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sparkless-testing
-Version: 0.1.0
+Version: 0.2.0
 Summary: Easy testing with sparkless or PySpark on demand
 Author-email: Odos Matthews <odosmatthews@gmail.com>
 Maintainer-email: Odos Matthews <odosmatthews@gmail.com>
@@ -27,8 +27,6 @@ Provides-Extra: mock
 Requires-Dist: sparkless>=3.19.0; extra == "mock"
 Provides-Extra: pyspark
 Requires-Dist: pyspark<3.6.0,>=3.5.0; extra == "pyspark"
-Provides-Extra: delta
-Requires-Dist: delta-spark<4.0.0,>=3.0.0; extra == "delta"
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
@@ -36,7 +34,6 @@ Requires-Dist: mypy>=1.0.0; extra == "dev"
 Requires-Dist: ruff>=0.4.0; extra == "dev"
 Requires-Dist: sparkless>=3.19.0; extra == "dev"
 Requires-Dist: pyspark<3.6.0,>=3.5.0; extra == "dev"
-Requires-Dist: delta-spark<4.0.0,>=3.0.0; extra == "dev"
 Dynamic: license-file
 
 # sparkless-testing
@@ -77,9 +74,6 @@ pip install sparkless-testing[mock]
 # Install with PySpark engine
 pip install sparkless-testing[pyspark]
 
-# Install with Delta Lake support
-pip install sparkless-testing[delta]
-
 # Install with all engines
 pip install sparkless-testing[dev]
 ```
@@ -208,7 +202,6 @@ spark = create_session(engine_type=EngineType.MOCK)
 # Create session with custom configuration
 config = SessionConfig(
     app_name="my-test",
-    enable_delta=True,
     shuffle_partitions=2,
 )
 spark = create_pyspark_session(config=config)
@@ -234,7 +227,6 @@ spark.stop()
 - `SessionConfig`: Configuration dataclass for session creation
   - `app_name`: Application name for the Spark session
   - `warehouse_dir`: Custom warehouse directory (optional)
-  - `enable_delta`: Enable Delta Lake support (default: `True` for programmatic use, `False` in test fixtures)
   - `enable_ui`: Enable Spark UI (default: `False`)
   - `shuffle_partitions`: Number of shuffle partitions (default: `1`)
   - `parallelism`: Default parallelism (default: `1`)
@@ -256,8 +248,6 @@ spark.stop()
 - `spark_types`: Types module for current engine
 - `spark_engine_type`: Current engine type as string (`"mock"` or `"pyspark"`)
 
-**Note**: Test fixtures automatically disable Delta Lake for parallel testing compatibility. To use Delta Lake, create sessions programmatically with `enable_delta=True` in `SessionConfig`.
-
 ## Advanced Usage
 
 ### Custom Session Configuration
@@ -269,7 +259,6 @@ def test_with_custom_config():
     config = SessionConfig(
         app_name="custom-test",
         warehouse_dir="/tmp/my-warehouse",
-        enable_delta=True,
         enable_ui=True,
         shuffle_partitions=4,
         parallelism=4,
@@ -290,7 +279,6 @@ pytest -n auto  # Auto-detect worker count
 **Best Practices for Parallel Testing:**
 - Mock (sparkless) tests can run with high parallelism (`-n 10+`)
 - PySpark tests work best with fewer workers (`-n 2` or `-n 4`)
-- Test fixtures automatically disable Delta Lake to avoid conflicts
 - Each test gets a unique session name and warehouse directory
 
 ### Using Test Utilities
@@ -367,15 +355,6 @@ If tests hang when using `pytest -n`, try:
 - Run PySpark tests sequentially: `pytest -n 0` for PySpark-only tests
 - Use mock engine for faster parallel execution: `SPARK_MODE=mock pytest -n 10`
 
-### Delta Lake Not Available
-
-Delta Lake is disabled by default in test fixtures. To enable:
-```python
-from sparkless_testing import SessionConfig, create_pyspark_session
-
-config = SessionConfig(app_name="test", enable_delta=True)
-spark = create_pyspark_session(config=config)
-```
 
 ### SparkContext Errors
 
@@ -427,7 +406,6 @@ def test_my_function(spark_session):
 Optional:
 - sparkless>=3.19.0 (for mock engine)
 - pyspark>=3.5.0 (for real engine)
-- delta-spark>=3.0.0 (for Delta Lake support)
 - pytest-xdist (for parallel test execution)
 
 ## License

{sparkless_testing-0.1.0 → sparkless_testing-0.2.0}/README.md

@@ -36,9 +36,6 @@ pip install sparkless-testing[mock]
 # Install with PySpark engine
 pip install sparkless-testing[pyspark]
 
-# Install with Delta Lake support
-pip install sparkless-testing[delta]
-
 # Install with all engines
 pip install sparkless-testing[dev]
 ```
@@ -167,7 +164,6 @@ spark = create_session(engine_type=EngineType.MOCK)
 # Create session with custom configuration
 config = SessionConfig(
     app_name="my-test",
-    enable_delta=True,
     shuffle_partitions=2,
 )
 spark = create_pyspark_session(config=config)
@@ -193,7 +189,6 @@ spark.stop()
 - `SessionConfig`: Configuration dataclass for session creation
   - `app_name`: Application name for the Spark session
   - `warehouse_dir`: Custom warehouse directory (optional)
-  - `enable_delta`: Enable Delta Lake support (default: `True` for programmatic use, `False` in test fixtures)
   - `enable_ui`: Enable Spark UI (default: `False`)
   - `shuffle_partitions`: Number of shuffle partitions (default: `1`)
   - `parallelism`: Default parallelism (default: `1`)
@@ -215,8 +210,6 @@ spark.stop()
 - `spark_types`: Types module for current engine
 - `spark_engine_type`: Current engine type as string (`"mock"` or `"pyspark"`)
 
-**Note**: Test fixtures automatically disable Delta Lake for parallel testing compatibility. To use Delta Lake, create sessions programmatically with `enable_delta=True` in `SessionConfig`.
-
 ## Advanced Usage
 
 ### Custom Session Configuration
@@ -228,7 +221,6 @@ def test_with_custom_config():
     config = SessionConfig(
         app_name="custom-test",
         warehouse_dir="/tmp/my-warehouse",
-        enable_delta=True,
         enable_ui=True,
         shuffle_partitions=4,
         parallelism=4,
@@ -249,7 +241,6 @@ pytest -n auto  # Auto-detect worker count
 **Best Practices for Parallel Testing:**
 - Mock (sparkless) tests can run with high parallelism (`-n 10+`)
 - PySpark tests work best with fewer workers (`-n 2` or `-n 4`)
-- Test fixtures automatically disable Delta Lake to avoid conflicts
 - Each test gets a unique session name and warehouse directory
 
 ### Using Test Utilities
@@ -326,15 +317,6 @@ If tests hang when using `pytest -n`, try:
 - Run PySpark tests sequentially: `pytest -n 0` for PySpark-only tests
 - Use mock engine for faster parallel execution: `SPARK_MODE=mock pytest -n 10`
 
-### Delta Lake Not Available
-
-Delta Lake is disabled by default in test fixtures. To enable:
-```python
-from sparkless_testing import SessionConfig, create_pyspark_session
-
-config = SessionConfig(app_name="test", enable_delta=True)
-spark = create_pyspark_session(config=config)
-```
 
 ### SparkContext Errors
 
@@ -386,7 +368,6 @@ def test_my_function(spark_session):
 Optional:
 - sparkless>=3.19.0 (for mock engine)
 - pyspark>=3.5.0 (for real engine)
-- delta-spark>=3.0.0 (for Delta Lake support)
 - pytest-xdist (for parallel test execution)
 
 ## License

{sparkless_testing-0.1.0 → sparkless_testing-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sparkless-testing"
-version = "0.1.0"
+version = "0.2.0"
 description = "Easy testing with sparkless or PySpark on demand"
 readme = "README.md"
 license = {text = "MIT"}
@@ -42,9 +42,6 @@ mock = [
 pyspark = [
     "pyspark>=3.5.0,<3.6.0",
 ]
-delta = [
-    "delta-spark>=3.0.0,<4.0.0",
-]
 dev = [
     "pytest>=7.0.0",
     "pytest-cov>=4.0.0",
@@ -52,7 +49,6 @@ dev = [
     "ruff>=0.4.0",
     "sparkless>=3.19.0",
     "pyspark>=3.5.0,<3.6.0",
-    "delta-spark>=3.0.0,<4.0.0",
 ]
 
 [project.urls]
@@ -92,17 +88,18 @@ warn_unreachable = true
 strict_equality = true
 show_error_codes = true
 exclude = [
-    "venv/",
-    ".venv/",
-    "build/",
-    "dist/",
+    "venv/.*",
+    ".venv/.*",
+    "build/.*",
+    "dist/.*",
+    ".*/venv/.*",
+    ".*/site-packages/.*",
 ]
 
 [[tool.mypy.overrides]]
 module = [
     "pytest.*",
     "pyspark.*",
-    "delta.*",
     "sparkless.*",
 ]
 ignore_missing_imports = true

{sparkless_testing-0.1.0 → sparkless_testing-0.2.0}/sparkless_testing/__init__.py

@@ -5,7 +5,7 @@ A Python package that simplifies running tests with either sparkless (mock)
 or PySpark (real) engines on demand.
 """
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"
 __author__ = "Odos Matthews"
 __email__ = "odosmatthews@gmail.com"
 

{sparkless_testing-0.1.0 → sparkless_testing-0.2.0}/sparkless_testing/parametrize.py

@@ -55,9 +55,8 @@ def both_engines(request: pytest.FixtureRequest) -> Generator[tuple[Any, EngineT
         engine_type = EngineType.MOCK
 
     # Configure and create session for the specified engine
-    # Disable Delta Lake for parallel testing compatibility
     unique_id = f"{os.getpid()}-{uuid.uuid4().hex[:8]}"
-    config = SessionConfig(app_name=f"test-{unique_id}", enable_delta=False)
+    config = SessionConfig(app_name=f"test-{unique_id}")
     auto_configure_engine(engine_type)
     spark = create_session(engine_type=engine_type, config=config)
 

{sparkless_testing-0.1.0 → sparkless_testing-0.2.0}/sparkless_testing/pytest_fixtures.py

@@ -85,10 +85,9 @@ def spark_session(request: pytest.FixtureRequest) -> Generator[Any, None, None]:
         else:
             auto_configure_engine(engine_type)
 
-    # Create session with Delta Lake disabled for parallel testing compatibility
     # Use process ID and UUID for unique session names (pytest-xdist uses separate processes)
     unique_id = f"{os.getpid()}-{uuid.uuid4().hex[:8]}"
-    config = SessionConfig(app_name=f"test-{unique_id}", enable_delta=False)
+    config = SessionConfig(app_name=f"test-{unique_id}")
     spark = create_session(engine_type=engine_type, config=config)
 
     yield spark
@@ -145,9 +144,8 @@ def pyspark_session() -> Generator[Any, None, None]:
     except RuntimeError:
         auto_configure_engine(EngineType.PYSPARK)
 
-    # Disable Delta Lake for parallel testing compatibility
     unique_id = f"{os.getpid()}-{uuid.uuid4().hex[:8]}"
-    config = SessionConfig(app_name=f"test-{unique_id}", enable_delta=False)
+    config = SessionConfig(app_name=f"test-{unique_id}")
     spark = create_pyspark_session(config=config)
 
     yield spark

{sparkless_testing-0.1.0 → sparkless_testing-0.2.0}/sparkless_testing/session.py

@@ -23,7 +23,6 @@ class SessionConfig:
 
     app_name: str
     warehouse_dir: str | None = None
-    enable_delta: bool = True
     enable_ui: bool = False
     shuffle_partitions: int = 1
     parallelism: int = 1
@@ -78,7 +77,7 @@ def create_pyspark_session(
     config: SessionConfig | None = None,
 ) -> Any:
     """
-    Create a PySpark SparkSession with optional Delta Lake support.
+    Create a PySpark SparkSession.
 
     Args:
         app_name: Application name (defaults to auto-generated)
@@ -146,23 +145,6 @@ def create_pyspark_session(
         if config.adaptive_enabled:
             builder = builder.config("spark.sql.adaptive.enabled", "true")
 
-    # Add Delta Lake support if enabled and available
-    enable_delta = config.enable_delta if config else True
-    if enable_delta:
-        try:
-            from delta import configure_spark_with_delta_pip
-
-            builder = builder.config(
-                "spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension"
-            ).config(
-                "spark.sql.catalog.spark_catalog",
-                "org.apache.spark.sql.delta.catalog.DeltaCatalog",
-            )
-            builder = configure_spark_with_delta_pip(builder)
-        except ImportError:
-            # Delta Lake not available, continue without it
-            pass
-
     # Stop any existing session in this process first to ensure clean state
     # This prevents conflicts when multiple tests run in the same process
     try:

{sparkless_testing-0.1.0 → sparkless_testing-0.2.0/sparkless_testing.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sparkless-testing
-Version: 0.1.0
+Version: 0.2.0
 Summary: Easy testing with sparkless or PySpark on demand
 Author-email: Odos Matthews <odosmatthews@gmail.com>
 Maintainer-email: Odos Matthews <odosmatthews@gmail.com>
@@ -27,8 +27,6 @@ Provides-Extra: mock
 Requires-Dist: sparkless>=3.19.0; extra == "mock"
 Provides-Extra: pyspark
 Requires-Dist: pyspark<3.6.0,>=3.5.0; extra == "pyspark"
-Provides-Extra: delta
-Requires-Dist: delta-spark<4.0.0,>=3.0.0; extra == "delta"
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
@@ -36,7 +34,6 @@ Requires-Dist: mypy>=1.0.0; extra == "dev"
 Requires-Dist: ruff>=0.4.0; extra == "dev"
 Requires-Dist: sparkless>=3.19.0; extra == "dev"
 Requires-Dist: pyspark<3.6.0,>=3.5.0; extra == "dev"
-Requires-Dist: delta-spark<4.0.0,>=3.0.0; extra == "dev"
 Dynamic: license-file
 
 # sparkless-testing
@@ -77,9 +74,6 @@ pip install sparkless-testing[mock]
 # Install with PySpark engine
 pip install sparkless-testing[pyspark]
 
-# Install with Delta Lake support
-pip install sparkless-testing[delta]
-
 # Install with all engines
 pip install sparkless-testing[dev]
 ```
@@ -208,7 +202,6 @@ spark = create_session(engine_type=EngineType.MOCK)
 # Create session with custom configuration
 config = SessionConfig(
     app_name="my-test",
-    enable_delta=True,
     shuffle_partitions=2,
 )
 spark = create_pyspark_session(config=config)
@@ -234,7 +227,6 @@ spark.stop()
 - `SessionConfig`: Configuration dataclass for session creation
   - `app_name`: Application name for the Spark session
   - `warehouse_dir`: Custom warehouse directory (optional)
-  - `enable_delta`: Enable Delta Lake support (default: `True` for programmatic use, `False` in test fixtures)
   - `enable_ui`: Enable Spark UI (default: `False`)
   - `shuffle_partitions`: Number of shuffle partitions (default: `1`)
   - `parallelism`: Default parallelism (default: `1`)
@@ -256,8 +248,6 @@ spark.stop()
 - `spark_types`: Types module for current engine
 - `spark_engine_type`: Current engine type as string (`"mock"` or `"pyspark"`)
 
-**Note**: Test fixtures automatically disable Delta Lake for parallel testing compatibility. To use Delta Lake, create sessions programmatically with `enable_delta=True` in `SessionConfig`.
-
 ## Advanced Usage
 
 ### Custom Session Configuration
@@ -269,7 +259,6 @@ def test_with_custom_config():
     config = SessionConfig(
         app_name="custom-test",
         warehouse_dir="/tmp/my-warehouse",
-        enable_delta=True,
         enable_ui=True,
         shuffle_partitions=4,
         parallelism=4,
@@ -290,7 +279,6 @@ pytest -n auto  # Auto-detect worker count
 **Best Practices for Parallel Testing:**
 - Mock (sparkless) tests can run with high parallelism (`-n 10+`)
 - PySpark tests work best with fewer workers (`-n 2` or `-n 4`)
-- Test fixtures automatically disable Delta Lake to avoid conflicts
 - Each test gets a unique session name and warehouse directory
 
 ### Using Test Utilities
@@ -367,15 +355,6 @@ If tests hang when using `pytest -n`, try:
 - Run PySpark tests sequentially: `pytest -n 0` for PySpark-only tests
 - Use mock engine for faster parallel execution: `SPARK_MODE=mock pytest -n 10`
 
-### Delta Lake Not Available
-
-Delta Lake is disabled by default in test fixtures. To enable:
-```python
-from sparkless_testing import SessionConfig, create_pyspark_session
-
-config = SessionConfig(app_name="test", enable_delta=True)
-spark = create_pyspark_session(config=config)
-```
 
 ### SparkContext Errors
 
@@ -427,7 +406,6 @@ def test_my_function(spark_session):
 Optional:
 - sparkless>=3.19.0 (for mock engine)
 - pyspark>=3.5.0 (for real engine)
-- delta-spark>=3.0.0 (for Delta Lake support)
 - pytest-xdist (for parallel test execution)
 
 ## License

{sparkless_testing-0.1.0 → sparkless_testing-0.2.0}/sparkless_testing.egg-info/requires.txt

@@ -1,8 +1,5 @@
 pytest>=7.0.0
 
-[delta]
-delta-spark<4.0.0,>=3.0.0
-
 [dev]
 pytest>=7.0.0
 pytest-cov>=4.0.0
@@ -10,7 +7,6 @@ mypy>=1.0.0
 ruff>=0.4.0
 sparkless>=3.19.0
 pyspark<3.6.0,>=3.5.0
-delta-spark<4.0.0,>=3.0.0
 
 [mock]
 sparkless>=3.19.0

{sparkless_testing-0.1.0 → sparkless_testing-0.2.0}/tests/test_session.py

@@ -59,11 +59,9 @@ def test_session_config():
     config = SessionConfig(
         app_name="test",
         warehouse_dir="/tmp/test",
-        enable_delta=False,
     )
     assert config.app_name == "test"
     assert config.warehouse_dir == "/tmp/test"
-    assert config.enable_delta is False
 
 
 def test_create_mock_session_with_app_name():
@@ -84,7 +82,6 @@ def test_create_pyspark_session_with_config():
         config = SessionConfig(
             app_name="test-config",
             warehouse_dir=None,  # Will use default with UUID
-            enable_delta=False,
             enable_ui=False,
             shuffle_partitions=2,
             parallelism=2,
@@ -358,84 +355,6 @@ def test_create_pyspark_session_stop_existing_error(monkeypatch):
         pytest.skip("PySpark not available")
 
 
-def test_create_pyspark_session_delta_import_error(monkeypatch):
-    """Test Delta Lake import error handling (lines 185-191)."""
-    import builtins
-    import sys
-
-    try:
-        # Save original import
-        original_import = builtins.__import__
-
-        # Mock delta import to raise ImportError
-        def mock_import(name, *args, **kwargs):
-            if name == "delta" or name.startswith("delta."):
-                raise ImportError("No module named 'delta'")
-            return original_import(name, *args, **kwargs)
-
-        monkeypatch.setattr(builtins, "__import__", mock_import)
-
-        # Clear cached imports
-        if "delta" in sys.modules:
-            monkeypatch.delattr(sys.modules, "delta", raising=False)
-
-        config = SessionConfig(
-            app_name="test-delta-error",
-            enable_delta=True,
-        )
-
-        # Should continue without Delta support
-        spark = create_pyspark_session(config=config)
-        assert spark is not None
-        spark.stop()
-    except RuntimeError:
-        pytest.skip("PySpark not available")
-
-
-def test_create_pyspark_session_delta_configuration(monkeypatch):
-    """Test Delta Lake configuration when available (lines 185-191)."""
-    try:
-        config = SessionConfig(
-            app_name="test-delta-config",
-            enable_delta=True,
-        )
-
-        # Should configure Delta if available
-        spark = create_pyspark_session(config=config)
-        assert spark is not None
-
-        # Verify Delta extensions are configured if delta is available
-        # Check spark config for Delta extensions
-        try:
-            extensions = spark.conf.get("spark.sql.extensions", "")
-            if "delta" in extensions.lower():
-                # Delta is configured
-                assert True
-        except Exception:
-            # Delta might not be available, that's okay
-            pass
-
-        spark.stop()
-    except RuntimeError:
-        pytest.skip("PySpark not available")
-
-
-def test_create_pyspark_session_delta_disabled(monkeypatch):
-    """Test Delta Lake when disabled in config."""
-    try:
-        config = SessionConfig(
-            app_name="test-delta-disabled",
-            enable_delta=False,
-        )
-
-        # Should not configure Delta
-        spark = create_pyspark_session(config=config)
-        assert spark is not None
-        spark.stop()
-    except RuntimeError:
-        pytest.skip("PySpark not available")
-
-
 def test_create_pyspark_session_log_level_error(monkeypatch):
     """Test log level setting exception (lines 202-203)."""
     import warnings