sparkless-testing 0.1.0__py3-none-any.whl

sparkless_testing/__init__.py

@@ -0,0 +1,87 @@
+"""
+sparkless-testing - Easy testing with sparkless or PySpark.
+
+A Python package that simplifies running tests with either sparkless (mock)
+or PySpark (real) engines on demand.
+"""
+
+__version__ = "0.1.0"
+__author__ = "Odos Matthews"
+__email__ = "odosmatthews@gmail.com"
+
+# Core engine configuration
+from .engine import (
+    EngineConfig,
+    EngineType,
+    auto_configure_engine,
+    configure_engine,
+    detect_available_engines,
+    get_engine,
+    reset_engine_state,
+)
+
+# Protocols (for type checking)
+from .protocols import (
+    AnalysisExceptionProtocol,
+    ColumnProtocol,
+    DataFrameProtocol,
+    FunctionsProtocol,
+    SparkSessionProtocol,
+    TypesProtocol,
+    WindowProtocol,
+)
+
+# Session factory
+from .session import (
+    SessionConfig,
+    create_mock_session,
+    create_pyspark_session,
+    create_session,
+    stop_all_sessions,
+)
+
+# Test utilities
+from .utils import (
+    create_test_dataframe,
+    detect_spark_type,
+    get_app_name_prefix,
+    get_spark_mode,
+    get_warehouse_dir,
+    is_dataframe_like,
+)
+
+__all__ = [
+    # Package metadata
+    "__version__",
+    "__author__",
+    "__email__",
+    # Engine configuration
+    "EngineType",
+    "EngineConfig",
+    "configure_engine",
+    "get_engine",
+    "reset_engine_state",
+    "detect_available_engines",
+    "auto_configure_engine",
+    # Session factory
+    "SessionConfig",
+    "create_mock_session",
+    "create_pyspark_session",
+    "create_session",
+    "stop_all_sessions",
+    # Test utilities
+    "detect_spark_type",
+    "is_dataframe_like",
+    "create_test_dataframe",
+    "get_spark_mode",
+    "get_warehouse_dir",
+    "get_app_name_prefix",
+    # Protocols
+    "ColumnProtocol",
+    "DataFrameProtocol",
+    "FunctionsProtocol",
+    "TypesProtocol",
+    "WindowProtocol",
+    "AnalysisExceptionProtocol",
+    "SparkSessionProtocol",
+]

sparkless_testing/conftest.py

@@ -0,0 +1,39 @@
+"""
+Auto-configure pytest fixtures for sparkless-testing.
+
+This module automatically registers pytest fixtures and hooks when imported.
+Import this in your conftest.py or test files to enable sparkless-testing fixtures.
+"""
+
+from __future__ import annotations
+
+# Import parametrization hooks
+from .parametrize import (  # noqa: F401
+    both_engines,
+    parametrize_engines,
+    pytest_configure,
+    pytest_generate_tests,
+)
+
+# Import fixtures to register them
+from .pytest_fixtures import (  # noqa: F401
+    mock_spark_session,
+    pyspark_session,
+    spark_engine_type,
+    spark_functions,
+    spark_session,
+    spark_types,
+)
+
+__all__ = [
+    "spark_session",
+    "mock_spark_session",
+    "pyspark_session",
+    "spark_functions",
+    "spark_types",
+    "spark_engine_type",
+    "both_engines",
+    "parametrize_engines",
+    "pytest_configure",
+    "pytest_generate_tests",
+]

sparkless_testing/engine.py

@@ -0,0 +1,306 @@
+"""
+Engine detection and configuration for sparkless-testing.
+
+This module provides centralized engine detection and configuration,
+allowing tests to work with either sparkless (mock) or PySpark (real) engines.
+"""
+
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any
+
+from .protocols import (
+    AnalysisExceptionProtocol,
+    FunctionsProtocol,
+    TypesProtocol,
+    WindowProtocol,
+)
+
+
+class EngineType(Enum):
+    """Engine type enumeration."""
+
+    MOCK = "mock"
+    PYSPARK = "pyspark"
+    AUTO = "auto"
+
+
+@dataclass
+class EngineConfig:
+    """Configuration for a Spark engine."""
+
+    functions: FunctionsProtocol
+    types: TypesProtocol
+    analysis_exception: type[BaseException] | AnalysisExceptionProtocol
+    window: WindowProtocol | None = None
+    desc: Any | None = None
+    engine_name: str = "unknown"
+    engine_type: EngineType = EngineType.AUTO
+    dataframe_cls: Any | None = None
+    spark_session_cls: Any | None = None
+    column_cls: Any | None = None
+
+
+# Global engine state (for backward compatibility)
+_engine: EngineConfig | None = None
+
+# Thread-local engine state (for parallel test isolation)
+_thread_local = threading.local()
+
+
+def configure_engine(
+    *,
+    functions: FunctionsProtocol,
+    types: TypesProtocol,
+    analysis_exception: type[BaseException] | AnalysisExceptionProtocol,
+    window: WindowProtocol | None = None,
+    desc: Any | None = None,
+    engine_name: str = "unknown",
+    engine_type: EngineType = EngineType.AUTO,
+    dataframe_cls: Any | None = None,
+    spark_session_cls: Any | None = None,
+    column_cls: Any | None = None,
+) -> None:
+    """
+    Inject engine components.
+
+    Sets both thread-local and global engine state for backward compatibility.
+    Thread-local state takes precedence in get_engine() for parallel test isolation.
+
+    **Thread-Local vs Global State:**
+
+    - **Thread-local state**: Used for parallel test execution (pytest-xdist).
+      Each thread maintains its own engine configuration, ensuring test isolation.
+      This is checked first by get_engine().
+
+    - **Global state**: Used for backward compatibility and single-threaded execution.
+      Falls back to this if thread-local state is not set.
+
+    When configure_engine() is called, it sets BOTH thread-local and global state.
+    This ensures compatibility with both parallel and sequential test execution.
+
+    Args:
+        functions: Functions module (F) from the engine
+        types: Types module from the engine
+        analysis_exception: AnalysisException class from the engine
+        window: Window class from the engine (optional)
+        desc: desc function from the engine (optional)
+        engine_name: Name of the engine (e.g., "mock", "pyspark")
+        engine_type: EngineType enum value
+        dataframe_cls: DataFrame class from the engine (optional)
+        spark_session_cls: SparkSession class from the engine (optional)
+        column_cls: Column class from the engine (optional)
+    """
+    global _engine
+    engine_config = EngineConfig(
+        functions=functions,
+        types=types,
+        analysis_exception=analysis_exception,
+        window=window,
+        desc=desc,
+        engine_name=engine_name,
+        engine_type=engine_type,
+        dataframe_cls=dataframe_cls,
+        spark_session_cls=spark_session_cls,
+        column_cls=column_cls,
+    )
+
+    # Set global state (for backward compatibility)
+    _engine = engine_config
+
+    # Set thread-local state (for parallel test isolation)
+    _thread_local.engine = engine_config
+
+
+def get_engine() -> EngineConfig:
+    """
+    Get the current engine config, raising if not configured.
+
+    **Priority Order:**
+    1. Thread-local state (checked first for parallel test isolation)
+    2. Global state (fallback for backward compatibility)
+
+    This ensures that in parallel test execution, each thread uses its own
+    engine configuration, while single-threaded execution uses the global state.
+
+    Returns:
+        EngineConfig instance
+
+    Raises:
+        RuntimeError: If engine is not configured in either thread-local or global state
+    """
+    # Try thread-local first (for parallel test isolation)
+    if hasattr(_thread_local, "engine") and _thread_local.engine is not None:
+        return _thread_local.engine  # type: ignore[no-any-return]
+
+    # Fallback to global state (for backward compatibility)
+    if _engine is None:
+        raise RuntimeError(
+            "Engine not configured. Call configure_engine(...) before using sparkless-testing."
+        )
+    return _engine
+
+
+def reset_engine_state() -> None:
+    """
+    Reset thread-local engine state.
+
+    This is useful for test isolation - clears the thread-local engine
+    so the next get_engine() call will use global state or raise an error.
+
+    **Note:** This only resets thread-local state, not global state.
+    After calling this, get_engine() will fall back to global state if available,
+    or raise RuntimeError if global state is also not configured.
+    """
+    if hasattr(_thread_local, "engine"):
+        delattr(_thread_local, "engine")
+
+
+def detect_available_engines() -> set[EngineType]:
+    """
+    Detect which engines are available (installed).
+
+    Returns:
+        Set of available EngineType values
+    """
+    available: set[EngineType] = set()
+
+    # Check for sparkless (mock)
+    try:
+        import sparkless  # noqa: F401
+
+        available.add(EngineType.MOCK)
+    except ImportError:
+        pass
+
+    # Check for PySpark
+    try:
+        import pyspark  # noqa: F401
+
+        available.add(EngineType.PYSPARK)
+    except ImportError:
+        pass
+
+    return available
+
+
+def auto_configure_engine(
+    preferred_engine: EngineType | None = None,
+) -> EngineConfig:
+    """
+    Automatically configure engine based on availability and preference.
+
+    Args:
+        preferred_engine: Preferred engine type (defaults to MOCK if available)
+
+    Returns:
+        EngineConfig instance
+
+    Raises:
+        RuntimeError: If no engines are available
+    """
+    available = detect_available_engines()
+
+    if not available:
+        raise RuntimeError("No Spark engines available. Install either 'sparkless' or 'pyspark'.")
+
+    # Determine which engine to use
+    if preferred_engine is None:
+        # Default preference: MOCK > PYSPARK
+        if EngineType.MOCK in available:
+            preferred_engine = EngineType.MOCK
+        elif EngineType.PYSPARK in available:
+            preferred_engine = EngineType.PYSPARK
+        else:
+            # Use first available
+            preferred_engine = next(iter(available))
+
+    if preferred_engine not in available:
+        raise RuntimeError(
+            f"Preferred engine '{preferred_engine.value}' is not available. "
+            f"Available engines: {[e.value for e in available]}"
+        )
+
+    # Configure based on engine type
+    if preferred_engine == EngineType.MOCK:
+        return _configure_mock_engine()
+    elif preferred_engine == EngineType.PYSPARK:
+        return _configure_pyspark_engine()
+    else:
+        raise RuntimeError(f"Unknown engine type: {preferred_engine}")
+
+
+def _configure_mock_engine() -> EngineConfig:
+    """Configure sparkless (mock) engine."""
+    try:
+        from sparkless import AnalysisException as MockAnalysisException
+        from sparkless import Column as MockColumn
+        from sparkless import DataFrame as MockDataFrame
+        from sparkless import SparkSession as MockSparkSession
+        from sparkless import Window as MockWindow
+        from sparkless import functions as mock_functions
+        from sparkless import spark_types as mock_types
+        from sparkless.functions import desc as mock_desc
+    except ImportError as e:
+        raise RuntimeError(
+            "sparkless is not installed. Install it with: pip install sparkless"
+        ) from e
+
+    configure_engine(
+        functions=mock_functions,
+        types=mock_types,
+        analysis_exception=MockAnalysisException,
+        window=MockWindow,
+        desc=mock_desc,
+        engine_name="mock",
+        engine_type=EngineType.MOCK,
+        dataframe_cls=MockDataFrame,
+        spark_session_cls=MockSparkSession,
+        column_cls=MockColumn,
+    )
+
+    return get_engine()
+
+
+def _configure_pyspark_engine() -> EngineConfig:
+    """Configure PySpark engine."""
+    try:
+        from pyspark.sql import Column as PySparkColumn
+        from pyspark.sql import DataFrame as PySparkDataFrame
+        from pyspark.sql import SparkSession as PySparkSparkSession
+        from pyspark.sql import functions as pyspark_functions
+        from pyspark.sql import types as pyspark_types
+        from pyspark.sql.functions import desc as pyspark_desc
+        from pyspark.sql.utils import AnalysisException as PySparkAnalysisException
+        from pyspark.sql.window import Window as PySparkWindow
+    except ImportError as e:
+        raise RuntimeError("PySpark is not installed. Install it with: pip install pyspark") from e
+
+    configure_engine(
+        functions=pyspark_functions,
+        types=pyspark_types,
+        analysis_exception=PySparkAnalysisException,
+        window=PySparkWindow,
+        desc=pyspark_desc,
+        engine_name="pyspark",
+        engine_type=EngineType.PYSPARK,
+        dataframe_cls=PySparkDataFrame,
+        spark_session_cls=PySparkSparkSession,
+        column_cls=PySparkColumn,
+    )
+
+    return get_engine()
+
+
+__all__ = [
+    "EngineType",
+    "EngineConfig",
+    "configure_engine",
+    "get_engine",
+    "reset_engine_state",
+    "detect_available_engines",
+    "auto_configure_engine",
+]

sparkless_testing/parametrize.py

@@ -0,0 +1,150 @@
+"""
+Parametrization support for running tests with both engines.
+
+This module provides utilities for running tests with both sparkless (mock)
+and PySpark (real) engines automatically.
+"""
+
+from __future__ import annotations
+
+import os
+import uuid
+from collections.abc import Generator
+from typing import Any
+
+import pytest
+
+from .engine import EngineType, auto_configure_engine, detect_available_engines
+from .session import SessionConfig, create_session
+
+
+def pytest_configure(config: Any) -> None:
+    """Register custom pytest markers."""
+    config.addinivalue_line(
+        "markers",
+        "spark_engine(engine): Mark test to use specific engine (mock or pyspark)",
+    )
+    config.addinivalue_line(
+        "markers",
+        "parametrize_engines: Run test with both available engines",
+    )
+
+
+@pytest.fixture(scope="function")  # type: ignore[untyped-decorator]
+def both_engines(request: pytest.FixtureRequest) -> Generator[tuple[Any, EngineType], None, None]:
+    """
+    Fixture that yields both available engines via parametrization.
+
+    This fixture is automatically parametrized to run with both engines.
+    The test will be called twice - once with mock engine, once with PySpark.
+
+    Example:
+        def test_both_engines(both_engines):
+            spark, engine_type = both_engines
+            df = spark.createDataFrame([{"id": 1}])
+            assert df.count() == 1
+    """
+    # Get the parametrized engine type
+    engine_type = getattr(request, "param", None)
+    if engine_type is None:
+        # If not parametrized, check availability and use default
+        available = detect_available_engines()
+        if EngineType.MOCK not in available or EngineType.PYSPARK not in available:
+            pytest.skip(f"Both engines not available. Available: {[e.value for e in available]}")
+        # Default to MOCK if not parametrized
+        engine_type = EngineType.MOCK
+
+    # Configure and create session for the specified engine
+    # Disable Delta Lake for parallel testing compatibility
+    unique_id = f"{os.getpid()}-{uuid.uuid4().hex[:8]}"
+    config = SessionConfig(app_name=f"test-{unique_id}", enable_delta=False)
+    auto_configure_engine(engine_type)
+    spark = create_session(engine_type=engine_type, config=config)
+
+    yield spark, engine_type
+
+    # Cleanup
+    try:
+        spark.stop()
+    except Exception:
+        pass
+
+
+# Auto-parametrize both_engines fixture if both engines are available
+def pytest_generate_tests(metafunc: Any) -> None:
+    """Generate test parameters for parametrize_engines marker and both_engines fixture."""
+    # Handle both_engines fixture parametrization
+    if "both_engines" in metafunc.fixturenames:
+        available = detect_available_engines()
+        if EngineType.MOCK in available and EngineType.PYSPARK in available:
+            # Parametrize the both_engines fixture
+            metafunc.parametrize(
+                "both_engines",
+                [EngineType.MOCK, EngineType.PYSPARK],
+                ids=["mock", "pyspark"],
+                indirect=True,
+            )
+        elif EngineType.MOCK in available or EngineType.PYSPARK in available:
+            # Only one engine available, parametrize with just that one
+            available_list = list(available)
+            metafunc.parametrize(
+                "both_engines",
+                available_list,
+                ids=[e.value for e in available_list],
+                indirect=True,
+            )
+
+    # Handle spark_session fixture parametrization for parametrize_engines marker
+    if "spark_session" in metafunc.fixturenames:
+        # Check if test has parametrize_engines marker
+        marker = metafunc.definition.get_closest_marker("parametrize_engines")
+        if marker:
+            available = detect_available_engines()
+
+            if EngineType.MOCK in available and EngineType.PYSPARK in available:
+                # Parametrize the spark_session fixture
+                metafunc.parametrize(
+                    "spark_session",
+                    [EngineType.MOCK, EngineType.PYSPARK],
+                    ids=["mock", "pyspark"],
+                    indirect=True,
+                )
+
+
+def parametrize_engines(*args: Any, **kwargs: Any) -> Any:
+    """
+    Decorator to parametrize a test to run with both engines.
+
+    This is a convenience decorator that uses pytest.mark.parametrize
+    to run the test with both mock and pyspark engines.
+
+    Example:
+        @parametrize_engines
+        def test_my_function(spark_session):
+            df = spark_session.createDataFrame([{"id": 1}])
+            assert df.count() == 1
+
+    Note: This decorator requires both engines to be available.
+    """
+    available = detect_available_engines()
+
+    if EngineType.MOCK not in available or EngineType.PYSPARK not in available:
+        # Skip parametrization if both engines not available
+        return pytest.mark.skip(
+            f"Both engines not available. Available: {[e.value for e in available]}"
+        )(*args, **kwargs)
+
+    # Create parametrize marker
+    return pytest.mark.parametrize(
+        "engine_type",
+        [EngineType.MOCK, EngineType.PYSPARK],
+        ids=["mock", "pyspark"],
+    )(*args, **kwargs)
+
+
+__all__ = [
+    "pytest_configure",
+    "both_engines",
+    "parametrize_engines",
+    "pytest_generate_tests",
+]