streamcase 0.1.0a1__py3-none-any.whl

streamcase/_spark_source.py

@@ -0,0 +1,90 @@
+"""Private construction of the runner-owned Spark JSON input source."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING
+
+from streamcase._directories import RunDirectories
+
+if TYPE_CHECKING:
+    from pyspark.sql import DataFrame, SparkSession
+    from pyspark.sql.types import StructType
+
+
+_RESERVED_SOURCE_OPTIONS = frozenset(
+    {
+        "badrecordspath",
+        "basepath",
+        "cleansource",
+        "columnnameofcorruptrecord",
+        "discardcachedinputratio",
+        "encoding",
+        "filenameonly",
+        "ignorecorruptfiles",
+        "ignoremissingfiles",
+        "inferschema",
+        "latestfirst",
+        "linesep",
+        "maxbytespertrigger",
+        "maxcachedfiles",
+        "maxfileage",
+        "maxfilespertrigger",
+        "mode",
+        "modifiedafter",
+        "modifiedbefore",
+        "multiline",
+        "path",
+        "pathglobfilter",
+        "paths",
+        "recursivefilelookup",
+        "samplingratio",
+        "schema",
+        "sourcearchivedir",
+        "wholefile",
+    }
+)
+
+
+def _copy_source_options(source_options: Mapping[str, str] | None) -> dict[str, str]:
+    copied = {} if source_options is None else dict(source_options)
+    conflicts: list[str] = []
+
+    for name, value in copied.items():
+        if not isinstance(name, str):
+            raise TypeError("Source option names must be strings.")
+        if not isinstance(value, str):
+            raise TypeError(f"Source option {name!r} must have a string value.")
+        if name.casefold() in _RESERVED_SOURCE_OPTIONS:
+            conflicts.append(name)
+
+    if conflicts:
+        rendered = ", ".join(repr(name) for name in sorted(conflicts, key=str.casefold))
+        raise ValueError(f"Runner-owned source options cannot be overridden: {rendered}.")
+
+    return copied
+
+
+def _build_json_stream(
+    spark: SparkSession,
+    directories: RunDirectories,
+    *,
+    schema: StructType | str,
+    source_options: Mapping[str, str] | None = None,
+) -> DataFrame:
+    """Build a deterministic streaming JSON source without starting a query."""
+    if schema is None:
+        raise ValueError("An explicit Spark input schema is required.")
+    if isinstance(schema, str) and not schema.strip():
+        raise ValueError("The Spark input schema must not be empty.")
+
+    approved_options = _copy_source_options(source_options)
+    reader = (
+        spark.readStream.schema(schema)
+        .option("maxFilesPerTrigger", 1)
+        .option("multiLine", "false")
+        .option("mode", "FAILFAST")
+        .options(**approved_options)
+    )
+    stream: DataFrame = reader.json(str(directories.input_dir))
+    return stream

streamcase/_version.py

@@ -0,0 +1,3 @@
+"""Streamcase package version."""
+
+__version__ = "0.1.0a1"

streamcase/actions.py

@@ -0,0 +1,51 @@
+"""Scenario actions for describing streaming test inputs."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
+
+
+@dataclass(frozen=True, slots=True, init=False)
+class Batch:
+    """One non-empty logical batch of input rows.
+
+    Row mappings are copied when the batch is created and exposed as read-only
+    mappings. Every row key must be a string. Values inside a row are not
+    recursively copied or frozen.
+    """
+
+    rows: tuple[Mapping[str, object], ...]
+
+    def __init__(self, rows: Iterable[Mapping[str, object]]) -> None:
+        """Create a batch from rows while preserving their insertion order."""
+        snapshots = tuple(MappingProxyType(dict(row)) for row in rows)
+        if not snapshots:
+            raise ValueError("Batch must contain at least one row.")
+
+        for row_index, row in enumerate(snapshots):
+            for key in row:
+                if not isinstance(key, str):
+                    message = (
+                        f"Batch row at index {row_index} contains a non-string key "
+                        f"of type {type(key).__name__}."
+                    )
+                    raise TypeError(message)
+
+        object.__setattr__(self, "rows", snapshots)
+
+
+@dataclass(frozen=True, slots=True)
+class Restart:
+    """A checkpoint-preserving streaming-query restart boundary."""
+
+
+def batch(*rows: Mapping[str, object]) -> Batch:
+    """Create a :class:`Batch` from one or more input rows."""
+    return Batch(rows)
+
+
+def restart() -> Restart:
+    """Create a checkpoint-preserving restart action."""
+    return Restart()

streamcase/assertions.py

@@ -0,0 +1,153 @@
+"""Backend-independent assertions for captured streaming results."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Iterable, Mapping
+from typing import cast
+
+from streamcase._diagnostics import (
+    _format_batch_count_mismatch,
+    _format_duplicate_keys,
+    _format_missing_key_fields,
+    _format_rows_mismatch,
+)
+from streamcase.results import CapturedBatch, ScenarioResult
+
+
+def assert_batch_count(result: ScenarioResult, expected_count: int) -> None:
+    """Assert the exact number of captured output batches."""
+    if not isinstance(result, ScenarioResult):
+        message = f"result must be a ScenarioResult; got {type(result).__name__}."
+        raise TypeError(message)
+    if isinstance(expected_count, bool) or not isinstance(expected_count, int):
+        message = f"expected_count must be an int; got {type(expected_count).__name__}."
+        raise TypeError(message)
+    if expected_count < 0:
+        raise ValueError("expected_count must be non-negative.")
+
+    actual_count = len(result.batches)
+    if actual_count != expected_count:
+        raise AssertionError(_format_batch_count_mismatch(expected_count, actual_count))
+
+
+def _values_equal(left: object, right: object) -> bool:
+    if type(left) is not type(right):
+        return False
+
+    if isinstance(left, float):
+        right_float = cast(float, right)
+        if math.isnan(left) and math.isnan(right_float):
+            return True
+        return left == right_float
+
+    if isinstance(left, Mapping):
+        right_mapping = cast(Mapping[object, object], right)
+        if set(left) != set(right_mapping):
+            return False
+        for key, left_value in left.items():
+            if not _values_equal(left_value, right_mapping[key]):
+                return False
+        return True
+
+    if isinstance(left, tuple):
+        right_tuple = cast(tuple[object, ...], right)
+        if len(left) != len(right_tuple):
+            return False
+        for left_value, right_value in zip(left, right_tuple, strict=True):
+            if not _values_equal(left_value, right_value):
+                return False
+        return True
+
+    return left == right
+
+
+def assert_unique_keys(result: ScenarioResult, *fields: str) -> None:
+    """Assert that selected fields form unique keys across all result rows."""
+    if not isinstance(result, ScenarioResult):
+        message = f"result must be a ScenarioResult; got {type(result).__name__}."
+        raise TypeError(message)
+    if not fields:
+        raise ValueError("At least one key field is required.")
+
+    seen_fields: set[str] = set()
+    for index, field in enumerate(fields):
+        if not isinstance(field, str):
+            message = f"Key field at index {index} must be a string; got {type(field).__name__}."
+            raise TypeError(message)
+        if field in seen_fields:
+            raise ValueError(f"Key fields must be unique; repeated {field!r}.")
+        seen_fields.add(field)
+
+    groups: list[tuple[tuple[object, ...], list[tuple[int, int]]]] = []
+    for captured_batch in result.batches:
+        for row_index, row in enumerate(captured_batch.rows):
+            missing_fields = tuple(field for field in fields if field not in row)
+            if missing_fields:
+                raise AssertionError(
+                    _format_missing_key_fields(
+                        captured_batch.batch_id,
+                        row_index,
+                        missing_fields,
+                    ),
+                )
+
+            key = tuple(row[field] for field in fields)
+            matched_locations: list[tuple[int, int]] | None = None
+            for existing_key, locations in groups:
+                if _values_equal(key, existing_key):
+                    matched_locations = locations
+                    break
+
+            location = (captured_batch.batch_id, row_index)
+            if matched_locations is None:
+                groups.append((key, [location]))
+            else:
+                matched_locations.append(location)
+
+    duplicates: list[tuple[tuple[str, ...], tuple[object, ...], tuple[tuple[int, int], ...]]] = []
+    for key, locations in groups:
+        if len(locations) < 2:
+            continue
+        duplicates.append((fields, key, tuple(locations)))
+
+    if duplicates:
+        raise AssertionError(_format_duplicate_keys(duplicates))
+
+
+def assert_rows_equal(
+    result: ScenarioResult,
+    expected_rows: Iterable[Mapping[str, object]],
+) -> None:
+    """Assert duplicate-aware equality of unordered flattened result rows."""
+    if not isinstance(result, ScenarioResult):
+        message = f"result must be a ScenarioResult; got {type(result).__name__}."
+        raise TypeError(message)
+
+    expected = CapturedBatch(0, expected_rows).rows
+    unmatched_actual = list(result.rows)
+    missing: list[Mapping[str, object]] = []
+
+    for expected_row in expected:
+        matched_index: int | None = None
+        for index, actual_row in enumerate(unmatched_actual):
+            if _values_equal(expected_row, actual_row):
+                matched_index = index
+                break
+
+        if matched_index is None:
+            missing.append(expected_row)
+        else:
+            unmatched_actual.pop(matched_index)
+
+    if not missing and not unmatched_actual:
+        return
+
+    raise AssertionError(
+        _format_rows_mismatch(
+            len(expected),
+            len(result.rows),
+            missing,
+            unmatched_actual,
+        ),
+    )

streamcase/py.typed

@@ -0,0 +1 @@
+

streamcase/results.py

@@ -0,0 +1,91 @@
+"""Immutable backend-independent captured result models."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass, field
+from datetime import date
+from decimal import Decimal
+from types import MappingProxyType
+
+
+def _freeze_mapping(value: Mapping[object, object], path: str) -> Mapping[str, object]:
+    frozen: dict[str, object] = {}
+    for key, nested_value in value.items():
+        if not isinstance(key, str):
+            message = f"Captured mapping at {path} must use string keys; got {type(key).__name__}."
+            raise TypeError(message)
+        frozen[key] = _freeze_value(nested_value, f"{path}.{key}")
+    return MappingProxyType(frozen)
+
+
+def _freeze_value(value: object, path: str) -> object:
+    if value is None:
+        return None
+    if isinstance(value, (bool, int, float, str, bytes, Decimal, date)):
+        return value
+    if isinstance(value, Mapping):
+        return _freeze_mapping(value, path)
+    if isinstance(value, (list, tuple)):
+        return tuple(
+            _freeze_value(nested_value, f"{path}[{index}]")
+            for index, nested_value in enumerate(value)
+        )
+
+    message = f"Captured value at {path} has unsupported type {type(value).__name__}."
+    raise TypeError(message)
+
+
+def _freeze_row(row: object, row_index: int) -> Mapping[str, object]:
+    if not isinstance(row, Mapping):
+        message = f"Captured row at index {row_index} must be a mapping; got {type(row).__name__}."
+        raise TypeError(message)
+    return _freeze_mapping(row, f"row[{row_index}]")
+
+
+@dataclass(frozen=True, slots=True, init=False)
+class CapturedBatch:
+    """One immutable output batch captured independently of a streaming backend."""
+
+    batch_id: int
+    rows: tuple[Mapping[str, object], ...]
+
+    def __init__(self, batch_id: int, rows: Iterable[Mapping[str, object]]) -> None:
+        """Snapshot an output batch and recursively freeze its rows."""
+        if isinstance(batch_id, bool) or not isinstance(batch_id, int):
+            message = f"Captured batch ID must be an int; got {type(batch_id).__name__}."
+            raise TypeError(message)
+        if batch_id < 0:
+            raise ValueError("Captured batch ID must be non-negative.")
+
+        snapshots = tuple(_freeze_row(row, index) for index, row in enumerate(rows))
+        object.__setattr__(self, "batch_id", batch_id)
+        object.__setattr__(self, "rows", snapshots)
+
+
+@dataclass(frozen=True, slots=True, init=False)
+class ScenarioResult:
+    """An immutable ordered snapshot of captured output batches."""
+
+    batches: tuple[CapturedBatch, ...]
+    _rows: tuple[Mapping[str, object], ...] = field(repr=False, compare=False)
+
+    def __init__(self, batches: Iterable[CapturedBatch]) -> None:
+        """Snapshot captured batches and expose their rows in execution order."""
+        snapshot = tuple(batches)
+        for index, captured_batch in enumerate(snapshot):
+            if not isinstance(captured_batch, CapturedBatch):
+                message = (
+                    f"Scenario result batch at index {index} must be a CapturedBatch; "
+                    f"got {type(captured_batch).__name__}."
+                )
+                raise TypeError(message)
+
+        rows = tuple(row for captured_batch in snapshot for row in captured_batch.rows)
+        object.__setattr__(self, "batches", snapshot)
+        object.__setattr__(self, "_rows", rows)
+
+    @property
+    def rows(self) -> tuple[Mapping[str, object], ...]:
+        """Return captured rows flattened in batch and row order."""
+        return self._rows

streamcase/scenario.py

@@ -0,0 +1,58 @@
+"""Immutable scenario model for ordered streaming-test actions."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import TypeAlias
+
+from streamcase.actions import Batch, Restart
+
+Action: TypeAlias = Batch | Restart
+
+
+@dataclass(frozen=True, slots=True, init=False)
+class Scenario:
+    """A non-empty, ordered sequence of supported scenario actions.
+
+    Restart actions must appear between batch actions and cannot be consecutive.
+    """
+
+    actions: tuple[Action, ...]
+
+    def __init__(self, actions: Iterable[Action]) -> None:
+        """Snapshot and validate an ordered iterable of actions."""
+        snapshot = tuple(actions)
+        if not snapshot:
+            raise ValueError("Scenario must contain at least one action.")
+
+        for index, action in enumerate(snapshot):
+            if not isinstance(action, (Batch, Restart)):
+                message = (
+                    f"Scenario action at index {index} must be a Batch or Restart; "
+                    f"got {type(action).__name__}."
+                )
+                raise TypeError(message)
+
+        if isinstance(snapshot[0], Restart):
+            raise ValueError("Scenario Restart action at index 0 cannot be first.")
+
+        last_index = len(snapshot) - 1
+        if isinstance(snapshot[last_index], Restart):
+            message = f"Scenario Restart action at index {last_index} cannot be last."
+            raise ValueError(message)
+
+        for index in range(1, len(snapshot)):
+            if isinstance(snapshot[index - 1], Restart) and isinstance(snapshot[index], Restart):
+                message = (
+                    f"Scenario Restart action at index {index} cannot immediately "
+                    "follow another Restart."
+                )
+                raise ValueError(message)
+
+        object.__setattr__(self, "actions", snapshot)
+
+
+def scenario(*actions: Action) -> Scenario:
+    """Create a :class:`Scenario` from one or more ordered actions."""
+    return Scenario(actions)

streamcase/spark.py

@@ -0,0 +1,93 @@
+"""Public Apache Spark Structured Streaming runner."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Callable, Mapping
+from typing import Literal
+
+try:
+    from pyspark.sql import DataFrame, SparkSession
+    from pyspark.sql.types import StructType
+except ModuleNotFoundError as error:
+    if (error.name or "").partition(".")[0] != "pyspark":
+        raise
+    raise ImportError(
+        'PySpark is required for streamcase.spark; install "streamcase[spark]".'
+    ) from error
+
+from streamcase._directories import RunDirectories
+from streamcase._spark_managed import _run_managed_batches
+from streamcase._spark_query_options import _prepare_query_configuration
+from streamcase._spark_source import _build_json_stream, _copy_source_options
+from streamcase.results import ScenarioResult
+from streamcase.scenario import Scenario
+
+__all__ = ["run_scenario"]
+
+
+def _validate_output_frame(output: object, spark: SparkSession) -> DataFrame:
+    if not isinstance(output, DataFrame):
+        raise TypeError("Spark transformation must return a DataFrame.")
+    if not output.isStreaming:
+        raise ValueError("Spark transformation must return a streaming DataFrame.")
+    if output.sparkSession is not spark:
+        raise ValueError("Spark transformation output must belong to the supplied session.")
+
+    seen: set[str] = set()
+    duplicates: set[str] = set()
+    for name in output.columns:
+        normalized = name.casefold()
+        if normalized in seen:
+            duplicates.add(name)
+        seen.add(normalized)
+    if duplicates:
+        rendered = ", ".join(repr(name) for name in sorted(duplicates, key=str.casefold))
+        raise ValueError(f"Spark transformation output has duplicate column names: {rendered}.")
+    return output
+
+
+def run_scenario(
+    spark: SparkSession,
+    scenario: Scenario,
+    *,
+    schema: StructType | str,
+    transform: Callable[[DataFrame], DataFrame],
+    output_mode: Literal["append", "complete", "update"] = "append",
+    source_options: Mapping[str, str] | None = None,
+    query_options: Mapping[str, str] | None = None,
+    base_dir: str | os.PathLike[str] | None = None,
+    retain_artifacts: bool = False,
+) -> ScenarioResult:
+    """Run a scenario using a caller-owned Spark session."""
+    if not isinstance(spark, SparkSession):
+        raise TypeError("spark must be a PySpark SparkSession.")
+    if not isinstance(scenario, Scenario):
+        raise TypeError("scenario must be a Streamcase Scenario.")
+    if not isinstance(schema, (StructType, str)):
+        raise TypeError("schema must be a Spark StructType or DDL string.")
+    if isinstance(schema, str) and not schema.strip():
+        raise ValueError("The Spark input schema must not be empty.")
+    if not callable(transform):
+        raise TypeError("transform must be callable.")
+
+    approved_source_options = _copy_source_options(source_options)
+    approved_mode, approved_query_options = _prepare_query_configuration(output_mode, query_options)
+
+    def build_stream(directories: RunDirectories) -> DataFrame:
+        source = _build_json_stream(
+            spark,
+            directories,
+            schema=schema,
+            source_options=approved_source_options,
+        )
+        return _validate_output_frame(transform(source), spark)
+
+    return _run_managed_batches(
+        scenario,
+        build_stream,
+        base_dir=base_dir,
+        retain_artifacts=retain_artifacts,
+        output_mode=approved_mode,
+        query_options=approved_query_options,
+    )