Samara 0.2__tar.gz → 0.3__tar.gz

{samara-0.2 → samara-0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Samara
-Version: 0.2
+Version: 0.3
 Summary: Config Driven ETL Framework
 License-File: LICENSE
 Author: Krijn van der Burg
@@ -22,6 +22,7 @@ Requires-Dist: pyjson5 (>=1.6.9,<2.0.0)
 Requires-Dist: pyspark (>=4.0.1,<5.0.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
 Requires-Dist: requests (>=2.32.5,<3.0.0)
+Requires-Dist: rich (>=14.3.3,<15.0.0)
 Requires-Dist: structlog (>=25.4.0,<26.0.0)
 Description-Content-Type: text/markdown
 

{samara-0.2 → samara-0.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "Samara"
-version = "0.2"
+version = "0.3"
 description = "Config Driven ETL Framework"
 authors = ["Krijn van der Burg"]
 readme = "README.md"
@@ -24,6 +24,7 @@ click = "^8.1.0"
 opentelemetry-api = "^1.37.0"
 opentelemetry-sdk = "^1.37.0"
 opentelemetry-exporter-otlp = "^1.37.0"
+rich = "^14.3.3"
 
 [tool.poetry.group.test.dependencies]
 pre_commit = "*"

{samara-0.2 → samara-0.3}/src/samara/__init__.py

@@ -7,7 +7,6 @@ engines and extensible components.
 
 Key capabilities:
     - Define pipelines via configuration (sources, transforms, destinations)
-    - Multi-engine architecture (Pandas, Polars, and more)
     - Configurable alert system with multiple notification channels
     - Event-triggered custom actions at pipeline stages
     - Engine-agnostic configuration supporting different backends

{samara-0.2 → samara-0.3}/src/samara/exceptions.py

@@ -14,9 +14,6 @@ meaningful error states to the operating system.
 """
 
 import enum
-from typing import TypeVar
-
-K = TypeVar("K")  # Key type
 
 
 class ExitCode(enum.IntEnum):

{samara-0.2 → samara-0.3}/src/samara/utils/file.py

@@ -51,9 +51,7 @@ class FileHandler(ABC):
             File access is deferred until read operations are performed.
             Validation occurs at read time, not during initialization.
         """
-        logger.debug("Initializing %s for path: %s", self.__class__.__name__, str(filepath))
         self.filepath = filepath
-        logger.debug("%s initialized successfully for: %s", self.__class__.__name__, str(filepath))
 
     def _file_exists(self) -> None:
         """Verify the file exists.
@@ -62,11 +60,9 @@ class FileHandler(ABC):
             FileNotFoundError: If the file does not exist.
             OSError: If a system-level error occurs while checking existence.
         """
-        logger.debug("Checking file existence: %s", str(self.filepath))
         if not self.filepath.exists():
             logger.error("File not found: %s", str(self.filepath))
             raise FileNotFoundError(f"File not found: {self.filepath}")
-        logger.debug("File exists: %s", str(self.filepath))
 
     def _is_file(self) -> None:
         """Verify the path points to a regular file (not a directory).
@@ -75,11 +71,9 @@ class FileHandler(ABC):
             IsADirectoryError: If the path is a directory.
             OSError: If the path is not a regular file or a system-level error occurs.
         """
-        logger.debug("Checking if path is a regular file: %s", str(self.filepath))
         if not self.filepath.is_file():
             logger.error("Path is not a regular file: %s", str(self.filepath))
             raise OSError(f"Expected a file but found directory or invalid path: '{self.filepath}'")
-        logger.debug("Path is a regular file: %s", str(self.filepath))
 
     def _read_permission(self) -> None:
         """Verify the file is readable by the current process.
@@ -88,11 +82,9 @@ class FileHandler(ABC):
             PermissionError: If the file is not readable.
             OSError: If a system-level error occurs while checking permissions.
         """
-        logger.debug("Checking read permissions for file: %s", str(self.filepath))
         if not os.access(self.filepath, os.R_OK):
             logger.error("Read permission denied for file: %s", str(self.filepath))
             raise PermissionError(f"Permission denied: Cannot read file '{self.filepath}'")
-        logger.debug("Read permissions validated for file: %s", str(self.filepath))
 
     def _file_not_empty(self) -> None:
         """Verify the file contains data (not empty).
@@ -100,12 +92,10 @@ class FileHandler(ABC):
         Raises:
             OSError: If the file is empty or a system-level error occurs accessing metadata.
         """
-        logger.debug("Checking if file is empty: %s", str(self.filepath))
         file_size = self.filepath.stat().st_size
         if file_size == 0:
             logger.error("File is empty: %s", str(self.filepath))
             raise OSError(f"File is empty: {self.filepath}")
-        logger.debug("File not empty: %s (size: %d bytes)", str(self.filepath), file_size)
 
     def _file_size_limits(self, max_size: int = DEFAULT_MAX_SIZE) -> None:
         """Verify file size is within specified limits.
@@ -116,7 +106,6 @@ class FileHandler(ABC):
         Raises:
             OSError: If the file exceeds size limits or a system-level error occurs.
         """
-        logger.debug("Checking file size limits for: %s (max allowed: %d bytes)", str(self.filepath), max_size)
         file_size = self.filepath.stat().st_size
 
         if file_size > max_size:
@@ -128,10 +117,6 @@ class FileHandler(ABC):
             )
             raise OSError(f"File too large: '{self.filepath}' ({file_size:,} bytes exceeds {max_size:,} bytes limit)")
 
-        logger.debug(
-            "File size within limits: %s (size: %d bytes, max: %d bytes)", str(self.filepath), file_size, max_size
-        )
-
     def _text_file(self) -> None:
         """Verify the file contains readable text (not binary data).
 
@@ -139,7 +124,6 @@ class FileHandler(ABC):
             OSError: If the file contains binary content or has encoding issues.
             PermissionError: If permission is denied while reading the file.
         """
-        logger.debug("Validating file is readable text: %s", str(self.filepath))
         try:
             with self.filepath.open("r", encoding=self.ENCODING) as file:
                 # Read first 512 bytes to check for binary content
@@ -147,7 +131,6 @@ class FileHandler(ABC):
                 if "\x00" in sample:
                     logger.error("File contains binary content: %s", str(self.filepath))
                     raise OSError(f"Invalid file format: '{self.filepath}' contains binary data, expected text file")
-                logger.debug("Text file validation passed: %s", str(self.filepath))
         except UnicodeDecodeError as e:
             logger.error("File encoding error (not valid UTF-8): %s - %s", str(self.filepath), e)
             raise OSError(f"Invalid file encoding: '{self.filepath}' is not valid UTF-8") from e
@@ -169,8 +152,7 @@ class FileHandler(ABC):
             OSError: If the file fails validation (empty, too large, binary, wrong type).
             NotImplementedError: If the subclass does not implement `_read()`.
         """
-        logger.info("Starting file validation and reading: %s", str(self.filepath))
-        logger.debug("Running validation checks for file: %s", str(self.filepath))
+        logger.info("Reading file: %s", str(self.filepath))
 
         self._file_exists()
         self._is_file()
@@ -179,11 +161,8 @@ class FileHandler(ABC):
         self._file_size_limits()
         self._text_file()
 
-        logger.info("All validation checks passed for file: %s", str(self.filepath))
-
-        logger.debug("Reading file content: %s", str(self.filepath))
         data = self._read()
-        logger.info("File successfully read and parsed: %s", str(self.filepath))
+        logger.info("Successfully parsed file: %s", str(self.filepath))
         return data
 
     @abstractmethod
@@ -222,14 +201,9 @@ class FileYamlHandler(FileHandler):
             FileNotFoundError: If the file does not exist.
             PermissionError: If the file cannot be read due to permission restrictions.
         """
-        logger.info("Reading YAML file: %s", str(self.filepath))
-
         try:
-            logger.debug("Opening YAML file for reading: %s", str(self.filepath))
             with open(file=self.filepath, mode="r", encoding="utf-8") as file:
                 data = yaml.safe_load(file)
-                logger.info("Successfully parsed YAML file: %s", str(self.filepath))
-                logger.debug("YAML data structure type: %s", type(data))
                 return data
         except yaml.YAMLError as e:
             logger.error("YAML parsing error in file '%s': %s", str(self.filepath), e)
@@ -256,15 +230,10 @@ class FileJsonHandler(FileHandler):
             FileNotFoundError: If the file does not exist.
             PermissionError: If the file cannot be read due to permission restrictions.
         """
-        logger.info("Reading JSON file: %s", str(self.filepath))
-
         try:
-            logger.debug("Opening JSON file for reading: %s", str(self.filepath))
             with open(file=self.filepath, mode="r", encoding="utf-8") as file:
                 content = file.read()
                 data = json.loads(content)
-                logger.info("Successfully parsed JSON file: %s", str(self.filepath))
-                logger.debug("JSON data structure type: %s", type(data))
                 return data
         except json.Json5DecoderException as e:
             logger.error("JSON parsing error in file '%s': %s", str(self.filepath), e)
@@ -316,9 +285,7 @@ class FileHandlerContext:
             >>> yaml_handler = FileHandlerContext.from_filepath(Path("pipeline.yaml"))
             >>> json_handler = FileHandlerContext.from_filepath(Path("config.jsonc"))
         """
-        logger.debug("Creating file handler for path: %s", str(filepath))
         _, file_extension = os.path.splitext(filepath)
-        logger.debug("Detected file extension: %s", file_extension)
 
         handler_class = cls.SUPPORTED_EXTENSIONS.get(file_extension)
 
@@ -335,7 +302,6 @@ class FileHandlerContext:
                 f"Supported formats: {supported_extensions}"
             )
 
-        logger.debug("Selected handler class: %s for extension: %s", handler_class.__name__, file_extension)
         handler = handler_class(filepath=filepath)
         logger.info("Created %s for file: %s", handler_class.__name__, str(filepath))
         return handler

{samara-0.2 → samara-0.3}/src/samara/utils/logger.py

@@ -25,6 +25,7 @@ Typical Usage:
 import logging
 
 import structlog
+from rich.logging import RichHandler
 
 
 def set_logger(level: str = "INFO") -> None:
@@ -77,22 +78,33 @@ def set_logger(level: str = "INFO") -> None:
     See Also:
         get_logger: Create logger instances for specific modules
     """
-    # Configure standard logging for third-party libraries and OTLP handler
-    logging.basicConfig(
-        format="%(message)s",
-        level=level,
-        force=True,
+    # Configure standard logging for pretty console output
+    root_logger = logging.getLogger()
+    root_logger.setLevel(level)
+
+    # Remove existing handlers to avoid duplicates on reconfiguration
+    root_logger.handlers.clear()
+
+    # Use Rich for pretty console output with structlog formatting
+    console_handler = RichHandler(rich_tracebacks=True, tracebacks_show_locals=True, markup=True)
+    console_handler.setLevel(level)
+    console_handler.setFormatter(
+        structlog.stdlib.ProcessorFormatter(
+            processors=[
+                structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+                structlog.dev.ConsoleRenderer(colors=False),
+            ],
+        ),
     )
+    root_logger.addHandler(console_handler)
 
     # Configure structlog
     structlog.configure(
         processors=[
             structlog.contextvars.merge_contextvars,
             structlog.stdlib.add_log_level,
-            structlog.stdlib.add_logger_name,
-            structlog.processors.TimeStamper(fmt="iso"),
+            structlog.stdlib.PositionalArgumentsFormatter(),
             structlog.processors.StackInfoRenderer(),
-            structlog.processors.format_exc_info,
             structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
         ],
         wrapper_class=structlog.stdlib.BoundLogger,

{samara-0.2 → samara-0.3}/src/samara/workflow/controller.py

@@ -188,7 +188,6 @@ class WorkflowController(BaseModel):
             validating external configuration sources, or providing IDE hints
             for configuration files.
         """
-        logger.debug("Exporting WorkflowController JSON schema")
         return cls.model_json_schema(schema_generator=PreserveFieldOrderJsonSchema)
 
     @trace_span("workflow_controller.execute_all")

{samara-0.2 → samara-0.3}/src/samara/workflow/jobs/models/model_extract.py

@@ -8,7 +8,7 @@ how data should be read from various sources with type validation.
 from enum import Enum
 from typing import Literal
 
-from pydantic import Field, FilePath
+from pydantic import Field
 
 from samara import BaseModel
 from samara.utils.logger import get_logger
@@ -74,7 +74,7 @@ class ExtractModel(BaseModel):
     id_: str = Field(..., alias="id", description="Identifier for this extraction operation", min_length=1)
     method: ExtractMethod = Field(..., description="Method of extraction (batch or streaming)")
     data_format: str = Field(..., description="Format of the data to extract (parquet, json, csv, etc.)")
-    schema_: str | FilePath = Field(..., alias="schema", description="Schema definition - can be a file path or string")
+    schema_: str = Field(..., alias="schema", description="Schema definition - file path or inline JSON string")
 
 
 class ExtractFileModel(ExtractModel):

{samara-0.2 → samara-0.3}/src/samara/workflow/jobs/models/model_load.py

@@ -109,9 +109,11 @@ class LoadModel(BaseModel, ABC):
     upstream_id: str = Field(..., description="Identifier of the upstream component providing data", min_length=1)
     method: LoadMethod = Field(..., description="Loading method (batch or streaming)")
     location: str = Field(
-        ..., description="URI that identifies where to load data in the modelified format.", min_length=1
+        ..., description="URI that identifies where to load data in the specified format.", min_length=1
+    )
+    schema_export: str = Field(
+        ..., description="URI where the schema is exported. Use an empty string to skip schema export."
     )
-    schema_export: str = Field(..., description="URI that identifies where to load schema.")
 
 
 class LoadModelFile(LoadModel):

{samara-0.2 → samara-0.3}/src/samara/workflow/jobs/models/model_transform.py

@@ -34,7 +34,6 @@ class ArgsModel(BaseModel, ABC):
 
 
 ArgsT = TypeVar("ArgsT", bound=ArgsModel)
-FunctionNameT = TypeVar("FunctionNameT", bound=str)
 
 
 class FunctionModel(BaseModel, Generic[ArgsT], ABC):

{samara-0.2 → samara-0.3}/src/samara/workflow/jobs/spark/extract.py

@@ -7,10 +7,9 @@ schema parsing and PySpark configuration management.
 """
 
 from abc import ABC, abstractmethod
-from pathlib import Path
 from typing import Any, Literal, Self
 
-from pydantic import Field, model_validator
+from pydantic import Field, PrivateAttr, model_validator
 from pyspark.sql import DataFrame
 from pyspark.sql.types import StructType
 
@@ -18,7 +17,7 @@ from samara.telemetry import trace_span
 from samara.types import DataFrameRegistry
 from samara.utils.logger import get_logger
 from samara.workflow.jobs.models.model_extract import ExtractFileModel, ExtractMethod, ExtractModel
-from samara.workflow.jobs.spark.schema import SchemaFilepathHandler, SchemaStringHandler
+from samara.workflow.jobs.spark.schema import SchemaHandler
 from samara.workflow.jobs.spark.session import SparkHandler
 
 logger = get_logger(__name__)
@@ -67,27 +66,14 @@ class ExtractSpark(ExtractModel, ABC):
         extraction mode based on the configured method.
     """
 
-    model_config = {"arbitrary_types_allowed": True, "extra": "allow"}
+    model_config = {"arbitrary_types_allowed": True, "extra": "forbid"}
+
+    _data_registry: DataFrameRegistry = PrivateAttr(default_factory=DataFrameRegistry)
+    _spark: SparkHandler = PrivateAttr(default_factory=SparkHandler)
 
     _schema_parsed: StructType
     options: dict[str, Any] = Field(..., description="PySpark reader options as key-value pairs")
 
-    def __init__(self, **data: Any) -> None:
-        """Initialize the extractor with configuration data and workflow components.
-
-        Sets up the Pydantic model with provided configuration data and initializes
-        workflow components (DataFrameRegistry for storing extracted data and
-        SparkHandler for managing the Spark session).
-
-        Args:
-            **data: Configuration data for model initialization. Should include
-                required fields like id_, method, schema_, and extract-specific fields.
-        """
-        super().__init__(**data)
-        # Set up non-Pydantic attributes that shouldn't be in schema
-        self.data_registry: DataFrameRegistry = DataFrameRegistry()
-        self.spark: SparkHandler = SparkHandler()
-
     @model_validator(mode="after")
     @trace_span("extract_spark.parse_schema")
     def parse_schema(self) -> Self:
@@ -103,21 +89,13 @@ class ExtractSpark(ExtractModel, ABC):
 
         Note:
             If schema_ is empty or None, returns early without parsing.
-            File path detection relies on the .json extension.
+            Detection uses JSON parsing: valid JSON is treated as inline,
+            anything else is treated as a file path.
         """
         if not self.schema_:
             return self
 
-        # Convert to string for processing
-        schema_str = str(self.schema_).strip()
-
-        # Detect if it's a file path or JSON string
-        if schema_str.endswith(".json"):
-            # File path - use FilepathHandler
-            self._schema_parsed = SchemaFilepathHandler.parse(schema=Path(schema_str))
-        else:
-            # JSON string - use StringHandler
-            self._schema_parsed = SchemaStringHandler.parse(schema=schema_str)
+        self._schema_parsed = SchemaHandler.parse_schema_string(schema=str(self.schema_).strip())
 
         return self
 
@@ -139,19 +117,15 @@ class ExtractSpark(ExtractModel, ABC):
         """
         logger.info("Starting extraction for source: %s using method: %s", self.id_, self.method.value)
 
-        logger.debug("Adding Spark configurations: %s", self.options)
-        self.spark.add_configs(options=self.options)
-
-        if self.method == ExtractMethod.BATCH:
-            logger.debug("Performing batch extraction for: %s", self.id_)
-            self.data_registry[self.id_] = self._extract_batch()
-            logger.info("Batch extraction completed successfully for: %s", self.id_)
-        elif self.method == ExtractMethod.STREAMING:
-            logger.debug("Performing streaming extraction for: %s", self.id_)
-            self.data_registry[self.id_] = self._extract_streaming()
-            logger.info("Streaming extraction completed successfully for: %s", self.id_)
-        else:
-            raise ValueError(f"Extraction method {self.method} is not supported for PySpark")
+        with self._spark.scoped_configs(options=self.options):
+            if self.method == ExtractMethod.BATCH:
+                self._data_registry[self.id_] = self._extract_batch()
+                logger.info("Batch extraction completed for: %s", self.id_)
+            elif self.method == ExtractMethod.STREAMING:
+                self._data_registry[self.id_] = self._extract_streaming()
+                logger.info("Streaming extraction completed for: %s", self.id_)
+            else:
+                raise ValueError(f"Extraction method {self.method} is not supported for PySpark")
 
     @abstractmethod
     def _extract_batch(self) -> DataFrame:
@@ -245,14 +219,13 @@ class ExtractFileSpark(ExtractSpark, ExtractFileModel):
         """
         logger.debug("Reading files in batch mode - path: %s, format: %s", self.location, self.data_format)
 
-        dataframe = self.spark.session.read.load(
+        dataframe = self._spark.session.read.load(
             path=self.location,
             format=self.data_format,
             schema=self._schema_parsed,
             **self.options,
         )
-        row_count = dataframe.count()
-        logger.info("Batch extraction successful - loaded %d rows from %s", row_count, self.location)
+        logger.info("Batch extraction successful - loaded data from %s", self.location)
         return dataframe
 
     @trace_span("extract_file_spark._extract_streaming")
@@ -272,7 +245,7 @@ class ExtractFileSpark(ExtractSpark, ExtractFileModel):
         """
         logger.debug("Reading files in streaming mode - path: %s, format: %s", self.location, self.data_format)
 
-        dataframe = self.spark.session.readStream.load(
+        dataframe = self._spark.session.readStream.load(
             path=self.location,
             format=self.data_format,
             schema=self._schema_parsed,

{samara-0.2 → samara-0.3}/src/samara/workflow/jobs/spark/job.py

@@ -16,6 +16,7 @@ from samara.utils.logger import get_logger
 from samara.workflow.jobs.models.model_job import JobEngine, JobModel
 from samara.workflow.jobs.spark.extract import ExtractSparkUnion
 from samara.workflow.jobs.spark.load import LoadSparkUnion
+from samara.workflow.jobs.spark.session import SparkHandler
 from samara.workflow.jobs.spark.transform import TransformSparkUnion
 
 logger = get_logger(__name__)
@@ -159,7 +160,7 @@ class JobSpark(JobModel[ExtractSparkUnion, TransformSparkUnion, LoadSparkUnion])
         logger.info("Starting extract phase with %d extractors", len(self.extracts))
         start_time = time.time()
 
-        for i, extract in enumerate(self.extracts):
+        for i, extract in enumerate(self.extracts, 1):
             extract_start_time = time.time()
             logger.debug("Running extractor %d/%d: %s", i, len(self.extracts), extract.id_)
             extract.extract()
@@ -189,7 +190,7 @@ class JobSpark(JobModel[ExtractSparkUnion, TransformSparkUnion, LoadSparkUnion])
         logger.info("Starting transform phase with %d transformers", len(self.transforms))
         start_time = time.time()
 
-        for i, transform in enumerate(self.transforms):
+        for i, transform in enumerate(self.transforms, 1):
             transform_start_time = time.time()
             logger.debug("Running transformer %d/%d: %s", i, len(self.transforms), transform.id_)
             transform.transform()
@@ -220,7 +221,7 @@ class JobSpark(JobModel[ExtractSparkUnion, TransformSparkUnion, LoadSparkUnion])
         logger.info("Starting load phase with %d loaders", len(self.loads))
         start_time = time.time()
 
-        for i, load in enumerate(self.loads):
+        for i, load in enumerate(self.loads, 1):
             load_start_time = time.time()
             logger.debug("Running loader %d/%d: %s", i, len(self.loads), load.id_)
             load.load()
@@ -232,15 +233,13 @@ class JobSpark(JobModel[ExtractSparkUnion, TransformSparkUnion, LoadSparkUnion])
 
     @override
     def _clear(self) -> None:
-        """Free resources by clearing Spark-specific registries.
+        """Free resources by clearing Spark-specific registries and stopping the session.
 
         Clears the DataFrameRegistry and StreamingQueryRegistry after job execution
-        completes. This prevents memory leaks and ensures clean state for subsequent
-        jobs, particularly important in long-running processes or batch environments
-        where multiple jobs execute sequentially.
+        completes, then stops the SparkSession to release JVM resources. This prevents
+        memory leaks and ensures clean state for subsequent jobs, particularly important
+        in long-running processes or containerized environments.
         """
-        logger.debug("Clearing DataFrameRegistry after job: %s", self.id_)
         DataFrameRegistry().clear()
-
-        logger.debug("Clearing StreamingQueryRegistry after job: %s", self.id_)
         StreamingQueryRegistry().clear()
+        SparkHandler().stop_session()

{samara-0.2 → samara-0.3}/src/samara/workflow/jobs/spark/load.py

@@ -9,7 +9,7 @@ import json
 from abc import ABC, abstractmethod
 from typing import Any, Literal
 
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 from pyspark.sql.streaming.query import StreamingQuery
 
 from samara.telemetry import trace_span
@@ -89,29 +89,13 @@ class LoadSpark(LoadModel, ABC):
         Spark application.
     """
 
-    model_config = {"arbitrary_types_allowed": True, "extra": "allow"}
+    model_config = {"arbitrary_types_allowed": True, "extra": "forbid"}
 
-    options: dict[str, Any] = Field(..., description="Options for the sink input.")
-
-    def __init__(self, **data: Any) -> None:
-        """Initialize the load component from pipeline configuration.
-
-        Creates a Pydantic model instance from provided configuration and
-        initializes workflow registries and Spark handler. Establishes the
-        component's ability to manage DataFrames, streaming queries, and
-        Spark session settings during the load operation.
+    _data_registry: DataFrameRegistry = PrivateAttr(default_factory=DataFrameRegistry)
+    _streaming_query_registry: StreamingQueryRegistry = PrivateAttr(default_factory=StreamingQueryRegistry)
+    _spark: SparkHandler = PrivateAttr(default_factory=SparkHandler)
 
-        Args:
-            **data: Configuration fields from pipeline definition. Expected
-                fields include: id_, upstream_id, method, location,
-                data_format, mode, options, and optionally schema_export.
-                See class Example for full configuration structure.
-        """
-        super().__init__(**data)
-        # Set up non-Pydantic attributes that shouldn't be in schema
-        self.data_registry: DataFrameRegistry = DataFrameRegistry()
-        self.streaming_query_registry: StreamingQueryRegistry = StreamingQueryRegistry()
-        self.spark: SparkHandler = SparkHandler()
+    options: dict[str, Any] = Field(..., description="Options for the sink input.")
 
     @abstractmethod
     def _load_batch(self) -> None:
@@ -181,29 +165,21 @@ class LoadSpark(LoadModel, ABC):
             self.method.value,
         )
 
-        logger.debug("Adding Spark configurations: %s", self.options)
-        self.spark.add_configs(options=self.options)
-
-        logger.debug("Copying dataframe from %s to %s", self.upstream_id, self.id_)
-        self.data_registry[self.id_] = self.data_registry[self.upstream_id]
+        with self._spark.scoped_configs(options=self.options):
+            self._data_registry[self.id_] = self._data_registry[self.upstream_id]
 
-        if self.method == LoadMethod.BATCH:
-            logger.debug("Performing batch load for: %s", self.id_)
-            self._load_batch()
-            logger.info("Batch load completed successfully for: %s", self.id_)
-        elif self.method == LoadMethod.STREAMING:
-            logger.debug("Performing streaming load for: %s", self.id_)
-            self.streaming_query_registry[self.id_] = self._load_streaming()
-            logger.info("Streaming load started successfully for: %s", self.id_)
-        else:
-            raise ValueError(f"Loading method {self.method} is not supported for PySpark")
+            if self.method == LoadMethod.BATCH:
+                self._load_batch()
+                logger.info("Batch load completed for: %s", self.id_)
+            elif self.method == LoadMethod.STREAMING:
+                self._streaming_query_registry[self.id_] = self._load_streaming()
+                logger.info("Streaming load started for: %s", self.id_)
+            else:
+                raise ValueError(f"Loading method {self.method} is not supported for PySpark")
 
-        # Export schema if location is specified
-        if self.schema_export:
-            schema_json = json.dumps(self.data_registry[self.id_].schema.jsonValue())
-            self._export_schema(schema_json, self.schema_export)
-
-        logger.info("Load operation completed successfully for: %s", self.id_)
+            if self.schema_export:
+                schema_json = json.dumps(self._data_registry[self.id_].schema.jsonValue())
+                self._export_schema(schema_json, self.schema_export)
 
 
 class LoadFileSpark(LoadSpark, LoadModelFile):
@@ -289,17 +265,14 @@ class LoadFileSpark(LoadSpark, LoadModelFile):
             self.mode,
         )
 
-        row_count = self.data_registry[self.id_].count()
-        logger.debug("Writing %d rows to %s", row_count, self.location)
-
-        self.data_registry[self.id_].write.save(
+        self._data_registry[self.id_].write.save(
             path=self.location,
             format=self.data_format,
             mode=self.mode,
             **self.options,
         )
 
-        logger.info("Batch write successful - wrote %d rows to %s", row_count, self.location)
+        logger.info("Batch write successful - wrote data to %s", self.location)
 
     @trace_span("load_file_spark._load_streaming")
     def _load_streaming(self) -> StreamingQuery:
@@ -320,7 +293,7 @@ class LoadFileSpark(LoadSpark, LoadModelFile):
             self.mode,
         )
 
-        streaming_query = self.data_registry[self.id_].writeStream.start(
+        streaming_query = self._data_registry[self.id_].writeStream.start(
             path=self.location,
             format=self.data_format,
             outputMode=self.mode,

{samara-0.2 → samara-0.3}/src/samara/workflow/jobs/spark/schema.py

@@ -53,6 +53,24 @@ class SchemaHandler(ABC):
             NotImplementedError: If not implemented by a subclass.
         """
 
+    @staticmethod
+    def parse_schema_string(schema: str) -> StructType:
+        """Parse a schema string, auto-detecting whether it is inline JSON or a file path.
+
+        A PySpark schema definition is always a JSON object starting with '{'.
+        Anything else is treated as a file path.
+
+        Args:
+            schema: A schema string that is either a JSON object defining an
+                inline schema or a file path pointing to a schema definition file.
+
+        Returns:
+            StructType: A fully configured PySpark StructType schema.
+        """
+        if schema.startswith("{"):
+            return SchemaStringHandler.parse(schema=schema)
+        return SchemaFilepathHandler.parse(schema=Path(schema))
+
 
 class SchemaDictHandler(SchemaHandler):
     """Convert dictionary schemas to PySpark StructType.
@@ -101,9 +119,7 @@ class SchemaDictHandler(SchemaHandler):
 
         try:
             struct_type = StructType.fromJson(json=schema)
-            field_count = len(struct_type.fields)
-            logger.info("Successfully parsed schema from dictionary - %d fields", field_count)
-            logger.debug("Schema fields: %s", [f.name for f in struct_type.fields])
+            logger.info("Successfully parsed schema from dictionary - %d fields", len(struct_type.fields))
             return struct_type
         except (ValueError, TypeError, KeyError) as e:
             raise ValueError(f"Failed to convert dictionary to schema: {e}") from e
@@ -151,15 +167,9 @@ class SchemaStringHandler(SchemaHandler):
                 valid schema structure.
             json.JSONDecodeError: If the string is not valid JSON.
         """
-        logger.debug("Parsing schema from JSON string (length: %d)", len(schema))
-
         try:
-            logger.debug("Parsing JSON string to dictionary")
             parsed_json = json.loads(s=schema)
-            logger.debug("Successfully parsed JSON string")
-
             result = SchemaDictHandler.parse(schema=parsed_json)
-            logger.info("Successfully parsed schema from JSON string")
             return result
 
         except json.JSONDecodeError as e:
@@ -211,13 +221,8 @@ class SchemaFilepathHandler(SchemaHandler):
         logger.info("Parsing schema from file: %s", str(schema))
 
         try:
-            logger.debug("Creating file handler for schema file: %s", str(schema))
             file_handler: FileHandler = FileHandlerContext.from_filepath(filepath=schema)
-
-            logger.debug("Reading schema file content")
             file_content = file_handler.read()
-
-            logger.debug("Converting file content to schema")
             result = SchemaDictHandler.parse(schema=file_content)
 
             logger.info("Successfully parsed schema from file: %s", str(schema))

{samara-0.2 → samara-0.3}/src/samara/workflow/jobs/spark/session.py

@@ -8,12 +8,15 @@ Key features:
 - Lazy initialization of Spark sessions (only created when needed)
 - Automatic resource cleanup and management
 - Centralized configuration handling for Spark parameters
+- Stage-scoped configuration to prevent config leaking between pipeline stages
 - Seamless integration with Samara's configuration-driven pipeline model
 
 The SparkHandler singleton ensures efficient resource usage across the entire
 pipeline lifecycle, whether running locally for testing or on distributed clusters.
 """
 
+from collections.abc import Iterator
+from contextlib import contextmanager
 from typing import Any
 
 from pyspark.sql import SparkSession
@@ -67,7 +70,6 @@ class SparkHandler(metaclass=Singleton):
             options: Spark configuration from your pipeline definition as key-value
                 pairs (e.g., {"spark.executor.memory": "4g"}). Optional.
         """
-        logger.debug("Configuring SparkHandler with app_name: %s (lazy initialization)", app_name)
         self._session = None
         self._app_name = app_name
         self._init_options = options or {}
@@ -84,7 +86,7 @@ class SparkHandler(metaclass=Singleton):
             The Spark session ready to execute your transformations
         """
         if self._session is None:
-            logger.debug("Creating SparkSession on first access - app_name: %s", self._app_name)
+            logger.debug("Creating SparkSession - app_name: %s", self._app_name)
 
             builder = SparkSession.Builder().appName(name=self._app_name)
 
@@ -93,11 +95,9 @@ class SparkHandler(metaclass=Singleton):
                     logger.debug("Setting Spark config: %s = %s", key, value)
                     builder = builder.config(key=key, value=value)
 
-            logger.debug("Creating/retrieving SparkSession")
             self._session = builder.getOrCreate()
-            logger.info("SparkHandler initialized successfully with app: %s", self._app_name)
+            logger.info("SparkSession initialized with app: %s", self._app_name)
 
-        logger.debug("Accessing SparkSession instance")
         return self._session
 
     @session.setter
@@ -126,6 +126,14 @@ class SparkHandler(metaclass=Singleton):
         Use this to manually clean up if you need to restart Spark during a
         pipeline's lifecycle.
         """
+        self.stop_session()
+
+    def stop_session(self) -> None:
+        """Stop the active Spark session and release all associated resources.
+
+        Safely shuts down the Spark session if one is active. Idempotent:
+        calling on an already-stopped or never-started session is a no-op.
+        """
         if self._session is not None:
             logger.info("Stopping SparkSession: %s", self._session.sparkContext.appName)
             self._session.stop()
@@ -149,6 +157,10 @@ class SparkHandler(metaclass=Singleton):
             Some Spark settings cannot be changed after initialization. For
             pre-execution configuration, define settings in your pipeline's
             engine configuration instead.
+
+            Configs applied here persist for the lifetime of the SparkSession.
+            For stage-scoped configs that automatically revert, use
+            scoped_configs() instead.
         """
         logger.debug("Adding %d configuration options to SparkSession", len(options))
 
@@ -157,3 +169,40 @@ class SparkHandler(metaclass=Singleton):
             self.session.conf.set(key=key, value=value)
 
         logger.info("Successfully applied %d configuration options", len(options))
+
+    @contextmanager
+    def scoped_configs(self, options: dict[str, Any]) -> Iterator[None]:
+        """Apply Spark settings for the duration of a pipeline stage, then restore previous values.
+
+        Context manager that temporarily applies Spark configuration options and
+        automatically reverts them when the stage completes. This prevents configs
+        set by one stage from leaking into subsequent stages.
+
+        Args:
+            options: Configuration settings as key-value pairs to apply
+                temporarily (e.g., {"spark.sql.shuffle.partitions": "200"})
+
+        Yields:
+            None: Control returns to the caller with configs applied.
+        """
+        if not options:
+            yield
+            return
+
+        logger.debug("Applying %d scoped configuration options", len(options))
+
+        previous_values: dict[str, str | None] = {}
+        for key in options:
+            previous_values[key] = self.session.conf.get(key, default=None)
+
+        self.add_configs(options)
+
+        try:
+            yield
+        finally:
+            logger.debug("Restoring %d configuration options to previous values", len(previous_values))
+            for key, old_value in previous_values.items():
+                if old_value is None:
+                    self.session.conf.unset(key)
+                else:
+                    self.session.conf.set(key=key, value=old_value)

{samara-0.2 → samara-0.3}/src/samara/workflow/jobs/spark/transform.py

@@ -8,7 +8,7 @@ through structured configuration rather than code.
 
 from typing import Any
 
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 
 from samara.telemetry import trace_span
 from samara.types import DataFrameRegistry
@@ -89,28 +89,12 @@ class TransformSpark(TransformModel[TransformFunctionSparkUnion]):
         function modifies the dataframe in place within the registry, so order matters.
     """
 
-    model_config = {"arbitrary_types_allowed": True, "extra": "allow"}
+    model_config = {"arbitrary_types_allowed": True, "extra": "forbid"}
 
-    options: dict[str, Any] = Field(..., description="Transformation options as key-value pairs")
-
-    def __init__(self, **data: Any) -> None:
-        """Initialize TransformSpark with configuration data.
-
-        Creates a Pydantic model instance from the provided configuration data and then
-        initializes workflow attributes for managing dataframes and Spark sessions. This
-        two-stage initialization separates Pydantic model validation from workflow setup.
-
-        Args:
-            **data: Configuration data for initializing the Pydantic model. Should include
-                `id_`, `upstream_id`, `functions`, and `options` keys at minimum.
+    _data_registry: DataFrameRegistry = PrivateAttr(default_factory=DataFrameRegistry)
+    _spark: SparkHandler = PrivateAttr(default_factory=SparkHandler)
 
-        Returns:
-            None
-        """
-        super().__init__(**data)
-        # Set up non-Pydantic attributes that shouldn't be in schema
-        self.data_registry: DataFrameRegistry = DataFrameRegistry()
-        self.spark: SparkHandler = SparkHandler()
+    options: dict[str, Any] = Field(..., description="Transformation options as key-value pairs")
 
     @trace_span("transform_spark.transform")
     def transform(self) -> None:
@@ -133,29 +117,18 @@ class TransformSpark(TransformModel[TransformFunctionSparkUnion]):
         """
         logger.info("Starting transformation for: %s from upstream: %s", self.id_, self.upstream_id)
 
-        logger.debug("Adding Spark configurations: %s", self.options)
-        self.spark.add_configs(options=self.options)
-
-        # Copy the dataframe from upstream to current id
-        logger.debug("Copying dataframe from %s to %s", self.upstream_id, self.id_)
-        self.data_registry[self.id_] = self.data_registry[self.upstream_id]
-
-        # Apply transformations
-        logger.debug("Applying %d transformation functions", len(self.functions))
-        for i, function in enumerate(self.functions):
-            logger.debug("Applying function %d/%d: %s", i, len(self.functions), function.function_type)
+        with self._spark.scoped_configs(options=self.options):
+            self._data_registry[self.id_] = self._data_registry[self.upstream_id]
 
-            original_count = self.data_registry[self.id_].count()
-            callable_ = function.transform()
-            self.data_registry[self.id_] = callable_(df=self.data_registry[self.id_])
+            for i, function in enumerate(self.functions, 1):
+                logger.debug("Applying function %d/%d: %s", i, len(self.functions), function.function_type)
 
-            new_count = self.data_registry[self.id_].count()
+                callable_ = function.transform()
+                self._data_registry[self.id_] = callable_(df=self._data_registry[self.id_])
 
-            logger.info(
-                "Function %s applied - rows changed from %d to %d", function.function_type, original_count, new_count
-            )
+                logger.info("Function %s applied successfully", function.function_type)
 
-        logger.info("Transformation completed successfully for: %s", self.id_)
+            logger.info("Transformation completed successfully for: %s", self.id_)
 
 
 TransformSparkUnion = TransformSpark

{samara-0.2 → samara-0.3}/src/samara/workflow/jobs/spark/transforms/base.py

@@ -4,7 +4,7 @@ This module provides the base class for all Spark-specific transformation functi
 enabling shared access to the DataFrame registry across transformation operations.
 """
 
-from typing import ClassVar
+from pydantic import PrivateAttr
 
 from samara.types import DataFrameRegistry
 from samara.workflow.jobs.models.model_transform import ArgsT, FunctionModel
@@ -20,14 +20,14 @@ class FunctionSpark(FunctionModel[ArgsT]):
     registry access throughout the transformation execution.
 
     Attributes:
-        data_registry: Shared class-level registry for accessing processed
-            DataFrames by their identifier within the pipeline execution context.
+        _data_registry: Shared registry for accessing processed DataFrames by
+            their identifier within the pipeline execution context.
 
     Note:
-        The data_registry is a class-level attribute shared across all instances
-        within a pipeline execution, enabling cross-reference between DataFrames
-        created by different transformation steps. This is essential for operations
-        that operate on multiple DataFrames such as joins and unions.
+        The _data_registry is a private attribute initialized per-instance via
+        Pydantic's PrivateAttr. Since DataFrameRegistry is a singleton, all
+        instances share the same underlying registry, enabling cross-reference
+        between DataFrames created by different transformation steps.
     """
 
-    data_registry: ClassVar[DataFrameRegistry] = DataFrameRegistry()
+    _data_registry: DataFrameRegistry = PrivateAttr(default_factory=DataFrameRegistry)

{samara-0.2 → samara-0.3}/src/samara/workflow/jobs/spark/transforms/filter.py

@@ -121,15 +121,9 @@ class FilterFunction(FilterFunctionModel, FunctionSpark):
         """
 
         def __f(df: DataFrame) -> DataFrame:
-            logger.debug("Applying filter transform with condition: %s", self.arguments.condition)
-            original_count = df.count()
-            logger.debug("Input DataFrame has %d rows", original_count)
-
             result_df = df.filter(self.arguments.condition)
-            filtered_count = result_df.count()
-            filtered_out = original_count - filtered_count
 
-            logger.info("Filter transform completed - kept %d rows, filtered out %d rows", filtered_count, filtered_out)
+            logger.info("Filter transform completed - condition: %s", self.arguments.condition)
             return result_df
 
         return __f

{samara-0.2 → samara-0.3}/src/samara/workflow/jobs/spark/transforms/join.py

@@ -90,36 +90,17 @@ class JoinFunction(JoinFunctionModel, FunctionSpark):
             may require appropriate cluster resources and shuffle operations.
         """
         logger.debug(
-            "Creating join transform - other: %s, on: %s, how: %s",
+            "Configuring join - other: %s, on: %s, how: %s",
             self.arguments.other_upstream_id,
             self.arguments.on,
             self.arguments.how,
         )
 
         def __f(df: DataFrame) -> DataFrame:
-            logger.debug("Applying join transform")
+            right_df = self._data_registry[self.arguments.other_upstream_id]
+            result_df = df.join(right_df, on=self.arguments.on, how=self.arguments.how)
 
-            # Get the right DataFrame from the registry
-            right_df = self.data_registry[self.arguments.other_upstream_id]
-            logger.debug(
-                "Retrieved right DataFrame: %s (columns: %s)",
-                self.arguments.other_upstream_id,
-                right_df.columns,
-            )
-
-            # Get the join type
-            join_type = self.arguments.how
-            # Get the join columns
-            join_on = self.arguments.on
-
-            logger.debug("Performing join - left: %d rows, right: %d rows", df.count(), right_df.count())
-            logger.debug("Join parameters - on: %s, how: %s", join_on, join_type)
-
-            # Perform the join operation
-            result_df = df.join(right_df, on=join_on, how=join_type)
-            result_count = result_df.count()
-
-            logger.info("Join transform completed - result: %d rows, join type: %s", result_count, join_type)
+            logger.info("Join transform completed - type: %s", self.arguments.how)
 
             return result_df
 

{samara-0.2 → samara-0.3}/src/samara/workflow/jobs/spark/transforms/select.py

@@ -102,17 +102,12 @@ class SelectFunction(SelectFunctionModel, FunctionSpark):
 
             The output contains only the projected columns in the order specified.
         """
-        logger.debug("Creating select transform for columns: %s", self.arguments.columns)
 
         def __f(df: DataFrame) -> DataFrame:
-            logger.debug("Applying select transform - input columns: %s", df.columns)
-            logger.debug("Selecting columns: %s", self.arguments.columns)
-
             result_df = df.select(*self.arguments.columns)
             logger.info(
                 "Select transform completed - selected %d columns from %d", len(result_df.columns), len(df.columns)
             )
-            logger.debug("Selected columns: %s", result_df.columns)
             return result_df
 
         return __f