data-prep-toolkit 0.2.2.dev0__py3-none-any.whl → 0.2.2.dev1__py3-none-any.whl

data_processing_ray/runtime/ray/transform_invoker.py

@@ -0,0 +1,103 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from typing import Any
+
+from data_processing.runtime.pure_python import invoke_transform
+from data_processing.utils import (
+    PipInstaller,
+    TransformRuntime,
+    TransformsConfiguration,
+    get_logger,
+)
+from data_processing_ray.runtime.ray import RayTransformLauncher
+
+
+project = "https://github.com/IBM/data-prep-kit.git"
+logger = get_logger(__name__)
+
+
+def execute_ray_transform(
+    configuration: TransformsConfiguration,
+    name: str,
+    params: dict[str, Any],
+    input_folder: str,
+    output_folder: str,
+    s3_config: dict[str, Any] = None,
+) -> bool:
+    """
+    Execute Ray transform
+    :param configuration: transforms configuration
+    :param name: transform name
+    :param params: transform params
+    :param input_folder: input folder (local or S3)
+    :param output_folder: output folder (local or S3)
+    :param s3_config: S3 configuration - None local data
+    :return: True/False - execution result
+    """
+    # get transform configuration
+    r_subdirectory, r_l_name, extra_libraries, t_class = configuration.get_configuration(
+        transform=name, runtime=TransformRuntime.RAY
+    )
+    if r_subdirectory is None:
+        return False
+    p_subdirectory, p_l_name, _, _ = configuration.get_configuration(transform=name, runtime=TransformRuntime.PYTHON)
+
+    installer = PipInstaller()
+    # Ray installer can depend on Python installer, if this is the case install Python one first
+    p_installed = False
+    if p_subdirectory is not None and not installer.validate(name=r_l_name):
+        # install corresponding python transform library, if required
+        if not installer.install(project=project, subdirectory=p_subdirectory, name=p_l_name):
+            logger.warning(f"failed to install transform {name}")
+            return False
+        p_installed = True
+
+    # Check if transformer already installed
+    r_installed = False
+    if not installer.validate(name=r_l_name):
+        # transformer is not installed, install it
+        if not installer.install(project=project, subdirectory=r_subdirectory, name=r_l_name):
+            logger.warning(f"failed to install transform {name}")
+            if p_installed:
+                installer.uninstall(name=p_l_name)
+                for library in extra_libraries:
+                    installer.uninstall(name=library)
+            return False
+        r_installed = True
+    # invoke transform
+    res = invoke_transform(
+        name=name,
+        t_class=t_class,
+        launcher=RayTransformLauncher,
+        input_folder=input_folder,
+        output_folder=output_folder,
+        s3_config=s3_config,
+        params=params | {"run_locally": True},
+    )
+    # clean up
+    if p_installed:
+        # we installed transformer, uninstall it
+        if not installer.uninstall(name=p_l_name):
+            logger.warning(f"failed uninstall transform {r_l_name}")
+    if r_installed:
+        # we installed transformer, uninstall it
+        if not installer.uninstall(name=r_l_name):
+            logger.warning(f"failed uninstall transform {r_l_name}")
+        # uninstall support libraries
+        for library in extra_libraries:
+            if not installer.uninstall(name=library):
+                logger.warning(f"failed uninstall transform {library}")
+    if res == 0:
+        return True
+    logger.warning(f"failed execution of transform {name}")
+    return False

data_processing_ray/runtime/ray/transform_launcher.py

@@ -0,0 +1,117 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import argparse
+import sys
+import time
+
+import ray
+from data_processing.data_access import DataAccessFactory, DataAccessFactoryBase
+from data_processing.runtime.transform_launcher import AbstractTransformLauncher
+from data_processing.utils import get_logger, str2bool
+from data_processing_ray.runtime.ray import (
+    RayTransformExecutionConfiguration,
+    RayTransformRuntimeConfiguration,
+    orchestrate,
+)
+
+
+logger = get_logger(__name__)
+
+
+class RayTransformLauncher(AbstractTransformLauncher):
+    """
+    Driver class starting Filter execution
+    """
+
+    def __init__(
+        self,
+        runtime_config: RayTransformRuntimeConfiguration,
+        data_access_factory: DataAccessFactoryBase = DataAccessFactory(),
+    ):
+        """
+        Creates driver
+        :param runtime_config: transform runtime factory
+        :param data_access_factory: the factory to create DataAccess instances.
+        """
+        super().__init__(runtime_config, data_access_factory)
+        self.execution_config = RayTransformExecutionConfiguration(name=self.name)
+
+    def _get_arguments(self, parser: argparse.ArgumentParser) -> argparse.Namespace:
+        """
+        Parse input parameters
+        :param parser: parser
+        :return: list of arguments
+        """
+        parser.add_argument(
+            "--run_locally", type=lambda x: bool(str2bool(x)), default=False, help="running ray local flag"
+        )
+        return super()._get_arguments(parser)
+
+    def _get_parameters(self, args: argparse.Namespace) -> bool:
+        """
+        This method creates arg parser, fill it with the parameters
+        and does parameters validation
+        :return: True id validation passe or False, if not
+        """
+        result = super()._get_parameters(args)
+        self.run_locally = args.run_locally
+        if self.run_locally:
+            logger.info("Running locally")
+        else:
+            logger.info("connecting to existing cluster")
+        return result
+
+    def _submit_for_execution(self) -> int:
+        """
+        Submit for Ray execution
+        :return:
+        """
+        res = 1
+        start = time.time()
+        try:
+            if self.run_locally:
+                # Will create a local Ray cluster
+                logger.debug("running locally creating Ray cluster")
+                # enable metrics for local Ray
+                ray.init(_metrics_export_port=8088)
+            else:
+                # connect to the existing cluster
+                logger.info("Connecting to the existing Ray cluster")
+                ray.init(f"ray://localhost:10001", ignore_reinit_error=True)
+            logger.debug("Starting orchestrator")
+            res = ray.get(
+                orchestrate.remote(
+                    preprocessing_params=self.execution_config,
+                    data_access_factory=self.data_access_factory,
+                    runtime_config=self.runtime_config,
+                )
+            )
+            logger.debug("Completed orchestrator")
+            time.sleep(10)
+        except Exception as e:
+            logger.info(f"Exception running ray remote orchestration\n{e}")
+        finally:
+            logger.info(f"Completed execution in {round((time.time() - start)/60., 3)} min, execution result {res}")
+            ray.shutdown()
+            return res
+
+    def launch(self) -> int:
+        """
+        Execute method orchestrates driver invocation
+        :return: launch result
+        """
+        res = super().launch()
+        if not self.run_locally and res > 0:
+            # if we are running in kfp exit to signal kfp that we failed
+            sys.exit(1)
+        return res

data_processing_ray/runtime/ray/transform_orchestrator.py

@@ -0,0 +1,159 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import time
+import traceback
+from datetime import datetime
+
+import ray
+from data_processing.data_access import DataAccessFactoryBase
+from data_processing_ray.runtime.ray import (
+    RayTransformExecutionConfiguration,
+    RayTransformFileProcessor,
+    RayTransformRuntimeConfiguration,
+    RayUtils,
+    TransformStatisticsRay,
+)
+from ray.util import ActorPool
+
+
+@ray.remote(num_cpus=1, scheduling_strategy="SPREAD")
+def orchestrate(
+    preprocessing_params: RayTransformExecutionConfiguration,
+    data_access_factory: DataAccessFactoryBase,
+    runtime_config: RayTransformRuntimeConfiguration,
+) -> int:
+    """
+    orchestrator for transformer execution
+    :param preprocessing_params: orchestrator configuration
+    :param data_access_factory: data access factory
+    :param runtime_config: transformer runtime configuration
+    :return: 0 - success or 1 - failure
+    """
+
+    from data_processing.utils import get_logger
+    from ray.util.metrics import Gauge
+
+    logger = get_logger(__name__)
+    start_ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    start_time = time.time()
+    logger.info(f"orchestrator started at {start_ts}")
+    # create data access
+    data_access = data_access_factory.create_data_access()
+    if data_access is None:
+        logger.error("No DataAccess instance provided - exiting")
+        return 1
+    statistics = TransformStatisticsRay.remote({})
+    # create transformer runtime
+    runtime = runtime_config.create_transform_runtime()
+    resources = RayUtils.get_cluster_resources()
+    try:
+        # Get files to process
+        files, profile, retries = data_access.get_files_to_process()
+        if len(files) == 0:
+            logger.error("No input files to process - exiting")
+            return 0
+        logger.info(f"Number of files is {len(files)}, source profile {profile}")
+        # Print interval
+        print_interval = int(len(files) / 100)
+        if print_interval == 0:
+            print_interval = 1
+        # Get Resources for execution
+        logger.info(f"Cluster resources: {resources}")
+        # print execution params
+        logger.info(
+            f"Number of workers - {preprocessing_params.n_workers} " f"with {preprocessing_params.worker_options} each"
+        )
+        # log retries
+        if retries > 0:
+            statistics.add_stats.remote({"data access retries": retries})
+        # create executors
+        processor_params = {
+            "data_access_factory": data_access_factory,
+            "transform_class": runtime_config.get_transform_class(),
+            "transform_params": runtime.get_transform_config(
+                data_access_factory=data_access_factory, statistics=statistics, files=files
+            ),
+            "statistics": statistics,
+        }
+        logger.debug("Creating actors")
+        processors = RayUtils.create_actors(
+            clazz=RayTransformFileProcessor,
+            params=processor_params,
+            actor_options=preprocessing_params.worker_options,
+            n_actors=preprocessing_params.n_workers,
+            creation_delay=preprocessing_params.creation_delay,
+        )
+        processors_pool = ActorPool(processors)
+        # create gauges
+        files_in_progress_gauge = Gauge("files_in_progress", "Number of files in progress")
+        files_completed_gauge = Gauge("files_processed_total", "Number of files completed")
+        available_cpus_gauge = Gauge("available_cpus", "Number of available CPUs")
+        available_gpus_gauge = Gauge("available_gpus", "Number of available GPUs")
+        available_memory_gauge = Gauge("available_memory", "Available memory")
+        available_object_memory_gauge = Gauge("available_object_store", "Available object store")
+        # process data
+        logger.debug("Begin processing files")
+        failures = RayUtils.process_files(
+            executors=processors_pool,
+            files=files,
+            print_interval=print_interval,
+            files_in_progress_gauge=files_in_progress_gauge,
+            files_completed_gauge=files_completed_gauge,
+            available_cpus_gauge=available_cpus_gauge,
+            available_gpus_gauge=available_gpus_gauge,
+            available_memory_gauge=available_memory_gauge,
+            object_memory_gauge=available_object_memory_gauge,
+            logger=logger,
+        )
+        if failures > 0:
+            statistics.add_stats.remote({"actor failures": failures})
+        logger.debug("Done processing files, waiting for flush() completion.")
+        # invoke flush to ensure that all results are returned
+        start = time.time()
+        replies = [processor.flush.remote() for processor in processors]
+        failures = RayUtils.wait_for_execution_completion(logger=logger, replies=replies)
+        if failures > 0:
+            statistics.add_stats.remote({"actor failures": failures})
+        logger.info(f"done flushing in {round(time.time() - start, 3)} sec")
+        status = "success"
+        return_code = 0
+    except Exception as e:
+        logger.error(f"Exception during execution {e}: {traceback.print_exc()}")
+        status = "failure"
+        return_code = 1
+    try:
+        # Compute execution statistics
+        logger.debug("Computing execution stats")
+        stats = runtime.compute_execution_stats(ray.get(statistics.get_execution_stats.remote()))
+        stats["processing_time"] = round(stats["processing_time"], 3)
+
+        # build and save metadata
+        logger.debug("Building job metadata")
+        metadata = {
+            "pipeline": preprocessing_params.pipeline_id,
+            "job details": preprocessing_params.job_details
+            | {"start_time": start_ts, "end_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "status": status},
+            "code": preprocessing_params.code_location,
+            "job_input_params": runtime_config.get_transform_metadata()
+            | data_access_factory.get_input_params()
+            | preprocessing_params.get_input_params(),
+            "execution_stats": resources | {"execution time, min": round((time.time() - start_time) / 60.0, 3)},
+            "job_output_stats": stats,
+        }
+        logger.debug(f"Saving job metadata: {metadata}.")
+        data_access.save_job_metadata(metadata)
+        logger.debug("Saved job metadata.")
+        return return_code
+    except Exception as e:
+        logger.error(f"Exception during execution {e}: {traceback.print_exc()}")
+        return 1

data_processing_ray/runtime/ray/transform_runtime.py

@@ -0,0 +1,53 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from typing import Any
+
+from data_processing.data_access import DataAccessFactoryBase
+from ray.actor import ActorHandle
+
+
+class DefaultRayTransformRuntime:
+    """
+    Transformer runtime used by processor to to create Transform specific environment
+    """
+
+    def __init__(self, params: dict[str, Any]):
+        """
+        Create/config this runtime.
+        :param params: parameters, often provided by the CLI arguments as defined by a TableTansformConfiguration.
+        """
+        self.params = params
+
+    def get_transform_config(
+        self, data_access_factory: DataAccessFactoryBase, statistics: ActorHandle, files: list[str]
+    ) -> dict[str, Any]:
+        """
+        Get the dictionary of configuration that will be provided to the transform's initializer.
+        This is the opportunity for this runtime to create a new set of configuration based on the
+        config/params provided to this instance's initializer.  This may include the addition
+        of new configuration data such as ray shared memory, new actors, etc, that might be needed and
+        expected by the transform in its initializer and/or transform() methods.
+        :param data_access_factory - data access factory class being used by the RayOrchestrator.
+        :param statistics - reference to statistics actor
+        :param files - list of files to process
+        :return: dictionary of transform init params
+        """
+        return self.params
+
+    def compute_execution_stats(self, stats: dict[str, Any]) -> dict[str, Any]:
+        """
+        Update/augment the given stats object with runtime-specific additions/modifications.
+        :param stats: output of statistics as aggregated across all calls to all transforms.
+        :return: job execution statistics.  These are generally reported as metadata by the Ray Orchestrator.
+        """
+        return stats

data_processing_ray/runtime/ray/transform_statistics.py

@@ -0,0 +1,76 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from typing import Any
+
+import ray
+from data_processing.transform import TransformStatistics
+
+
+@ray.remote(num_cpus=0.25, scheduling_strategy="SPREAD")
+class TransformStatisticsRay(TransformStatistics):
+    """
+    Basic statistics class collecting basic execution statistics.
+    It can be extended for specific processors
+    """
+
+    def __init__(self, params: dict[str, Any]):
+        from ray.util.metrics import Counter
+
+        super().__init__()
+        self.data_write_counter = Counter(name="data_written", description="Total data written bytes")
+        self.data_read_counter = Counter(name="data_read", description="Total data read bytes")
+        self.source_files_counter = Counter(name="source_files_processed", description="Total source files processed")
+        self.result_files_counter = Counter(name="result_files_written", description="Total result files written")
+        self.source_documents_counter = Counter(
+            name="source_documents_processed", description="Total source document processed"
+        )
+        self.result_documents_counter = Counter(
+            name="result_documents_written", description="Total result documents written"
+        )
+        self.empty_table_counter = Counter(name="empty_tables", description="Total empty tables read")
+        self.failed_read_counter = Counter(name="failed_read_files", description="Total read failed files")
+        self.failed_write_counter = Counter(name="failed_write_files", description="Total write failed files")
+        self.transform_exceptions_counter = Counter(
+            name="transform_exceptions", description="Transform exception occurred"
+        )
+        self.data_retries_counter = Counter(name="data_access_retries", description="Data access retries")
+
+    def add_stats(self, stats=dict[str, Any]) -> None:
+        """
+        Add statistics
+        :param stats - dictionary creating new statistics
+        :return: None
+        """
+        for key, val in stats.items():
+            self.stats[key] = self.stats.get(key, 0) + val
+            if val > 0:
+                if key == "source_files":
+                    self.source_files_counter.inc(val)
+                if key == "source_size":
+                    self.data_read_counter.inc(val)
+                if key == "result_files":
+                    self.result_files_counter.inc(val)
+                if key == "source_doc_count":
+                    self.source_documents_counter.inc(val)
+                if key == "result_doc_count":
+                    self.result_documents_counter.inc(val)
+                if key == "skipped empty tables":
+                    self.empty_table_counter.inc(val)
+                if key == "failed_reads":
+                    self.failed_read_counter.inc(val)
+                if key == "failed_writes":
+                    self.failed_write_counter.inc(val)
+                if key == "transform execution exception":
+                    self.transform_exceptions_counter.inc(val)
+                if key == "data access retries":
+                    self.data_retries_counter.inc(val)

data_processing_ray/test_support/transform/__init__.py

@@ -0,0 +1 @@
+from data_processing_ray.test_support.transform.noop_transform import NOOPRayTransformConfiguration

data_processing_ray/test_support/transform/noop_transform.py

@@ -0,0 +1,45 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+
+from data_processing.test_support.transform.noop_transform import (
+    NOOPTransformConfiguration,
+)
+from data_processing.utils import get_logger
+from data_processing_ray.runtime.ray import (
+    RayTransformLauncher,
+    RayTransformRuntimeConfiguration,
+)
+
+
+logger = get_logger(__name__)
+
+
+class NOOPRayTransformConfiguration(RayTransformRuntimeConfiguration):
+    """
+    Implements the RayTransformConfiguration for NOOP as required by the RayTransformLauncher.
+    NOOP does not use a RayRuntime class so the superclass only needs the base
+    python-only configuration.
+    """
+
+    def __init__(self):
+        """
+        Initialization
+        """
+        super().__init__(transform_config=NOOPTransformConfiguration())
+
+
+if __name__ == "__main__":
+    # launcher = NOOPRayLauncher()
+    launcher = RayTransformLauncher(NOOPRayTransformConfiguration())
+    logger.info("Launching noop transform")
+    launcher.launch()

data_processing_spark/runtime/spark/__init__.py

@@ -0,0 +1,17 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from data_processing_spark.runtime.spark.transform_runtime import DefaultSparkTransformRuntime
+from data_processing_spark.runtime.spark.execution_configuration import SparkTransformExecutionConfiguration
+from data_processing_spark.runtime.spark.runtime_configuration import SparkTransformRuntimeConfiguration
+from data_processing_spark.runtime.spark.transform_file_processor import SparkTransformFileProcessor
+from data_processing_spark.runtime.spark.transform_orchestrator import orchestrate
+from data_processing_spark.runtime.spark.transform_launcher import SparkTransformLauncher

data_processing_spark/runtime/spark/execution_configuration.py

@@ -0,0 +1,85 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import argparse
+from typing import Any
+
+from data_processing.runtime import TransformExecutionConfiguration, runtime_cli_prefix
+from data_processing.utils import CLIArgumentProvider, get_logger
+
+
+logger = get_logger(__name__)
+
+
+class SparkTransformExecutionConfiguration(TransformExecutionConfiguration):
+    """
+    A class specifying and validating Spark orchestrator configuration
+    """
+
+    def __init__(self, name: str):
+        """
+        Initialization
+        """
+        super().__init__(name=name, print_params=False)
+        self.parallelization = -1
+
+    def add_input_params(self, parser: argparse.ArgumentParser) -> None:
+        """
+        This method adds transformer specific parameter to parser
+        :param parser: parser
+        :return: None
+        """
+        """
+        This determines how many partitions the RDD should be divided into. See 
+        https://sparktpoint.com/how-to-create-rdd-using-parallelize/ for the explanation
+        of this parameter
+        If you specify a positive value of the parameter, Spark will attempt to evenly 
+        distribute the data from seq into that many partitions. For example, if you have 
+        a collection of 100 elements and you specify numSlices as 4, Spark will try 
+        to create 4 partitions with approximately 25 elements in each partition.
+        If you don’t specify this parameter, Spark will use a default value, which is 
+        typically determined based on the cluster configuration or the available resources
+        (number of workers).    
+        """
+        parser.add_argument(f"--{runtime_cli_prefix}parallelization", type=int, default=-1, help="parallelization.")
+        return TransformExecutionConfiguration.add_input_params(self, parser=parser)
+
+    def apply_input_params(self, args: argparse.Namespace) -> bool:
+        """
+        Validate transformer specific parameters
+        :param args: user defined arguments
+        :return: True, if validate pass or False otherwise
+        """
+        if not TransformExecutionConfiguration.apply_input_params(self, args=args):
+            return False
+        captured = CLIArgumentProvider.capture_parameters(args, runtime_cli_prefix, False)
+        # store parameters locally
+        self.job_details = {
+            "job category": "preprocessing",
+            "job name": self.name,
+            "job type": "spark",
+            "job id": captured["job_id"],
+        }
+        self.parallelization = captured["parallelization"]
+        # if the user did not define actor max_restarts set it up for fault tolerance
+        logger.info(f"job details {self.job_details}")
+        logger.info(f"RDD parallelization {self.parallelization}")
+        return True
+
+    def get_input_params(self) -> dict[str, Any]:
+        """
+        get input parameters for job_input_params in metadata
+        :return: dictionary of parameters
+        """
+        return {
+            "RDD parallelization": self.parallelization,
+        }