data-prep-toolkit 0.0.1.dev12__tar.gz → 0.1.1__tar.gz

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.1}/Makefile

@@ -1,7 +1,6 @@
 # Use make help, to see the available rules
-REPOROOT=../
-include ../.make.defaults
-include ../.make.versions
+REPOROOT=../..
+include $(REPOROOT)/.make.defaults
 
 TAG := "v${DPK_LIB_VERSION}"
 
@@ -53,10 +52,13 @@ venv::	pyproject.toml
 # pytest-forked was tried, but then we get SIGABRT in pytest when running the s3 tests, some of which are skipped.. 
 test::  
 	@# Help: Use the already-built virtual environment to run pytest on the test directory. 
-	source venv/bin/activate; export PYTHONPATH=../src; cd test; $(PYTEST)  data_processing_tests/data_access;	
-	source venv/bin/activate; export PYTHONPATH=../src;  cd test; $(PYTEST)  data_processing_tests/transform;	
-	source venv/bin/activate; export PYTHONPATH=../src;  cd test; $(PYTEST)  data_processing_tests/launch/pure_python;
+	source venv/bin/activate; export PYTHONPATH=../src; cd test; $(PYTEST)  data_processing_tests/data_access;
+	source venv/bin/activate; export PYTHONPATH=../src;  cd test; $(PYTEST)  data_processing_tests/transform;
+	source venv/bin/activate; export PYTHONPATH=../src;  cd test; $(PYTEST)  data_processing_tests/launch/pure_python/launcher_test.py;
+	source venv/bin/activate; export PYTHONPATH=../src;  cd test; $(PYTEST)  data_processing_tests/launch/pure_python/multi_launcher_test.py;
+	source venv/bin/activate; export PYTHONPATH=../src;  cd test; $(PYTEST)  data_processing_tests/launch/pure_python/test_noop_launch.py;
 	source venv/bin/activate; export PYTHONPATH=../src; cd test; $(PYTEST)  data_processing_tests/launch/ray/ray_util_test.py;
-	source venv/bin/activate; export PYTHONPATH=../src; cd test; $(PYTEST)  data_processing_tests/launch/ray/launcher_test.py;	
-	source venv/bin/activate; export PYTHONPATH=../src; cd test; $(PYTEST)  data_processing_tests/launch/ray/test_noop_launch.py;	
+	source venv/bin/activate; export PYTHONPATH=../src; cd test; $(PYTEST)  data_processing_tests/launch/ray/multi_launcher_test.py;
+	source venv/bin/activate; export PYTHONPATH=../src; cd test; $(PYTEST)  data_processing_tests/launch/ray/launcher_test.py;
+	source venv/bin/activate; export PYTHONPATH=../src; cd test; $(PYTEST)  data_processing_tests/launch/ray/test_noop_launch.py;
 

{data_prep_toolkit-0.0.1.dev12/src/data_prep_toolkit.egg-info → data_prep_toolkit-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data_prep_toolkit
-Version: 0.0.1.dev12
+Version: 0.1.1
 Summary: Data Preparation Toolkit Library
 Author-email: David Wood <dawood@us.ibm.com>, Boris Lublinsky <blublinsky@ibm.com>
 License: Apache-2.0
@@ -27,9 +27,9 @@ Requires-Dist: markupsafe==2.0.1; extra == "dev"
 # Data Processing Library
 This provides a python framework for developing _transforms_
 on data stored in files - currently parquet files are supported -
-and running them in a [ray](https://ray.com) cluster.
+and running them in a [ray](https://www.ray.io/) cluster.
 Data files may be stored in the local file system or  COS/S3.
-For more details see the [documentation](doc/overview.md).
+For more details see the [documentation](../doc/overview.md).
 
 ### Virtual Environment
 The project uses `pyproject.toml` and a Makefile for operations.

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.1}/README.md

@@ -1,9 +1,9 @@
 # Data Processing Library
 This provides a python framework for developing _transforms_
 on data stored in files - currently parquet files are supported -
-and running them in a [ray](https://ray.com) cluster.
+and running them in a [ray](https://www.ray.io/) cluster.
 Data files may be stored in the local file system or  COS/S3.
-For more details see the [documentation](doc/overview.md).
+For more details see the [documentation](../doc/overview.md).
 
 ### Virtual Environment
 The project uses `pyproject.toml` and a Makefile for operations.

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit"
-version = "0.0.1-dev12"
+version = "0.1.1"
 requires-python = ">=3.10"
 description = "Data Preparation Toolkit Library"
 license = {text = "Apache-2.0"}

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.1/src/data_prep_toolkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data_prep_toolkit
-Version: 0.0.1.dev12
+Version: 0.1.1
 Summary: Data Preparation Toolkit Library
 Author-email: David Wood <dawood@us.ibm.com>, Boris Lublinsky <blublinsky@ibm.com>
 License: Apache-2.0
@@ -27,9 +27,9 @@ Requires-Dist: markupsafe==2.0.1; extra == "dev"
 # Data Processing Library
 This provides a python framework for developing _transforms_
 on data stored in files - currently parquet files are supported -
-and running them in a [ray](https://ray.com) cluster.
+and running them in a [ray](https://www.ray.io/) cluster.
 Data files may be stored in the local file system or  COS/S3.
-For more details see the [documentation](doc/overview.md).
+For more details see the [documentation](../doc/overview.md).
 
 ### Virtual Environment
 The project uses `pyproject.toml` and a Makefile for operations.

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.1}/src/data_prep_toolkit.egg-info/SOURCES.txt

@@ -2,24 +2,6 @@
 Makefile
 README.md
 pyproject.toml
-doc/advanced-transform-tutorial.md
-doc/architecture.md
-doc/overview.md
-doc/processing-architecture.jpg
-doc/python-launcher-options.md
-doc/python-runtime.md
-doc/ray-launcher-options.md
-doc/ray-runtime.md
-doc/simplest-transform-tutorial.md
-doc/testing-e2e-transform.md
-doc/transform-external-resources.md
-doc/transform-runtimes.md
-doc/transform-s3-testing.md
-doc/transform-standalone-testing.md
-doc/transform-testing.md
-doc/transform-tutorial-examples.md
-doc/transform-tutorials.md
-doc/transformer-utilities.md
 src/data_prep_toolkit.egg-info/PKG-INFO
 src/data_prep_toolkit.egg-info/SOURCES.txt
 src/data_prep_toolkit.egg-info/dependency_links.txt
@@ -35,22 +17,23 @@ src/data_processing/data_access/data_access_local.py
 src/data_processing/data_access/data_access_s3.py
 src/data_processing/runtime/__init__.py
 src/data_processing/runtime/execution_configuration.py
+src/data_processing/runtime/runtime_configuration.py
+src/data_processing/runtime/transform_file_processor.py
 src/data_processing/runtime/transform_launcher.py
 src/data_processing/runtime/pure_python/__init__.py
-src/data_processing/runtime/pure_python/python_launcher_configuration.py
+src/data_processing/runtime/pure_python/runtime_configuration.py
+src/data_processing/runtime/pure_python/transform_file_processor.py
 src/data_processing/runtime/pure_python/transform_launcher.py
 src/data_processing/runtime/pure_python/transform_orchestrator.py
-src/data_processing/runtime/pure_python/transform_table_processor.py
 src/data_processing/runtime/ray/__init__.py
+src/data_processing/runtime/ray/execution_configuration.py
 src/data_processing/runtime/ray/ray_utils.py
-src/data_processing/runtime/ray/transform_configuration.py
-src/data_processing/runtime/ray/transform_launch_configuration.py
+src/data_processing/runtime/ray/runtime_configuration.py
+src/data_processing/runtime/ray/transform_file_processor.py
 src/data_processing/runtime/ray/transform_launcher.py
 src/data_processing/runtime/ray/transform_orchestrator.py
-src/data_processing/runtime/ray/transform_orchestrator_configuration.py
 src/data_processing/runtime/ray/transform_runtime.py
 src/data_processing/runtime/ray/transform_statistics.py
-src/data_processing/runtime/ray/transform_table_processor.py
 src/data_processing/test_support/__init__.py
 src/data_processing/test_support/abstract_test.py
 src/data_processing/test_support/data_access/__init__.py
@@ -61,6 +44,7 @@ src/data_processing/test_support/transform/__init__.py
 src/data_processing/test_support/transform/noop_transform.py
 src/data_processing/test_support/transform/transform_test.py
 src/data_processing/transform/__init__.py
+src/data_processing/transform/binary_transform.py
 src/data_processing/transform/table_transform.py
 src/data_processing/transform/transform_configuration.py
 src/data_processing/transform/transform_statistics.py
@@ -88,8 +72,10 @@ test/data_processing_tests/data_access/data_access_local_test.py
 test/data_processing_tests/data_access/data_access_s3_test.py
 test/data_processing_tests/data_access/sample_input_data_test.py
 test/data_processing_tests/launch/pure_python/launcher_test.py
+test/data_processing_tests/launch/pure_python/multi_launcher_test.py
 test/data_processing_tests/launch/pure_python/test_noop_launch.py
 test/data_processing_tests/launch/ray/launcher_test.py
+test/data_processing_tests/launch/ray/multi_launcher_test.py
 test/data_processing_tests/launch/ray/ray_util_test.py
 test/data_processing_tests/launch/ray/test_noop_launch.py
 test/data_processing_tests/transform/test_noop.py

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.1}/src/data_processing/data_access/data_access_factory.py

@@ -142,14 +142,14 @@ class DataAccessFactory(DataAccessFactoryBase):
             arg_dict = args
         else:
             raise ValueError("args must be Namespace or dictionary")
-        s3_cred = arg_dict.get(f"{self.cli_arg_prefix}s3_cred")
-        s3_config = arg_dict.get(f"{self.cli_arg_prefix}s3_config")
-        local_config = arg_dict.get(f"{self.cli_arg_prefix}local_config")
-        checkpointing = arg_dict.get(f"{self.cli_arg_prefix}checkpointing")
+        s3_cred = arg_dict.get(f"{self.cli_arg_prefix}s3_cred", None)
+        s3_config = arg_dict.get(f"{self.cli_arg_prefix}s3_config", None)
+        local_config = arg_dict.get(f"{self.cli_arg_prefix}local_config", None)
+        checkpointing = arg_dict.get(f"{self.cli_arg_prefix}checkpointing", False)
         max_files = arg_dict.get(f"{self.cli_arg_prefix}max_files", -1)
-        data_sets = arg_dict.get(f"{self.cli_arg_prefix}data_sets")
+        data_sets = arg_dict.get(f"{self.cli_arg_prefix}data_sets", None)
         n_samples = arg_dict.get(f"{self.cli_arg_prefix}num_samples", -1)
-        files_to_use = arg_dict.get(f"{self.cli_arg_prefix}files_to_use")
+        files_to_use = arg_dict.get(f"{self.cli_arg_prefix}files_to_use", [".parquet"])
         # check which configuration (S3, LakeHouse, or Local) is specified
         s3_config_specified = 1 if s3_config is not None else 0
         local_config_specified = 1 if local_config is not None else 0

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.1}/src/data_processing/data_access/data_access_local.py

@@ -318,32 +318,32 @@ class DataAccessLocal(DataAccess):
         metadata["source"] = {"name": self.input_folder, "type": "path"}
         metadata["target"] = {"name": self.output_folder, "type": "path"}
         return self.save_file(
-            file_path=os.path.join(self.output_folder, "metadata.json"),
-            bytes_data=json.dumps(metadata, indent=2).encode(),
+            path=os.path.join(self.output_folder, "metadata.json"),
+            data=json.dumps(metadata, indent=2).encode(),
         )
 
-    def get_file(self, file_path: str) -> bytes:
+    def get_file(self, path: str) -> bytes:
         """
         Gets the contents of a file as a byte array, decompressing gz files if needed.
 
         Args:
-            file_path (str): The path to the file.
+            path (str): The path to the file.
 
         Returns:
             bytes: The contents of the file as a byte array, or None if an error occurs.
         """
 
         try:
-            if file_path.endswith(".gz"):
-                with gzip.open(file_path, "rb") as f:
+            if path.endswith(".gz"):
+                with gzip.open(path, "rb") as f:
                     data = f.read()
             else:
-                with open(file_path, "rb") as f:
+                with open(path, "rb") as f:
                     data = f.read()
             return data
 
         except (FileNotFoundError, gzip.BadGzipFile) as e:
-            logger.error(f"Error reading file {file_path}: {e}")
+            logger.error(f"Error reading file {path}: {e}")
             raise e
 
     def get_folder_files(self, path: str, extensions: list[str] = None, return_data: bool = True) -> dict[str, bytes]:
@@ -374,13 +374,13 @@ class DataAccessLocal(DataAccess):
             matching_files[filename] = _get_file_content(filename, return_data)
         return matching_files
 
-    def save_file(self, file_path: str, bytes_data: bytes) -> dict[str, Any]:
+    def save_file(self, path: str, data: bytes) -> dict[str, Any]:
         """
         Saves bytes to a file and returns a dictionary with file information.
 
         Args:
-            bytes_data (bytes): The bytes data to save.
-            file_path (str): The full name of the file to save.
+            data (bytes): The bytes data to save.
+            path (str): The full name of the file to save.
 
         Returns:
             dict or None: A dictionary with "name" and "size" keys if successful,
@@ -388,12 +388,12 @@ class DataAccessLocal(DataAccess):
         """
 
         try:
-            os.makedirs(os.path.dirname(file_path), exist_ok=True)
-            with open(file_path, "wb") as f:
-                f.write(bytes_data)
-            file_info = {"name": file_path, "size": os.path.getsize(file_path)}
+            os.makedirs(os.path.dirname(path), exist_ok=True)
+            with open(path, "wb") as f:
+                f.write(data)
+            file_info = {"name": path, "size": os.path.getsize(path)}
             return file_info
 
         except Exception as e:
-            logger.error(f"Error saving bytes to file {file_path}: {e}")
+            logger.error(f"Error saving bytes to file {path}: {e}")
             return None

data_prep_toolkit-0.1.1/src/data_processing/runtime/__init__.py

@@ -0,0 +1,4 @@
+from data_processing.runtime.execution_configuration import TransformExecutionConfiguration
+from data_processing.runtime.runtime_configuration import TransformRuntimeConfiguration
+from data_processing.runtime.transform_launcher import AbstractTransformLauncher, multi_launcher
+from data_processing.runtime.transform_file_processor import AbstractTransformFileProcessor

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.1}/src/data_processing/runtime/execution_configuration.py

@@ -27,15 +27,17 @@ class TransformExecutionConfiguration(CLIArgumentProvider):
     A class specifying and validating transform execution configuration
     """
 
-    def __init__(self, name: str, pp: bool = True):
+    def __init__(self, name: str, print_params: bool = True):
         """
         Initialization
+        :param name: job name
+        :param print_params: flag to print parameters
         """
         self.pipeline_id = ""
         self.job_details = {}
         self.code_location = {}
         self.name = name
-        self.pp = pp
+        self.print_params = print_params
 
     def add_input_params(self, parser: argparse.ArgumentParser) -> None:
         """
@@ -74,10 +76,9 @@ class TransformExecutionConfiguration(CLIArgumentProvider):
             "job id": captured["job_id"],
         }
         self.code_location = captured["code_location"]
-
-        if self.pp:
-            # print parameters
-            logger.info(f"pipeline id {self.pipeline_id}")
+        # print parameters
+        logger.info(f"pipeline id {self.pipeline_id}")
+        if self.print_params:
             logger.info(f"job details {self.job_details}")
-            logger.info(f"code location {self.code_location}")
+        logger.info(f"code location {self.code_location}")
         return True

data_prep_toolkit-0.1.1/src/data_processing/runtime/pure_python/__init__.py

@@ -0,0 +1,4 @@
+from data_processing.runtime.pure_python.runtime_configuration import PythonTransformRuntimeConfiguration
+from data_processing.runtime.pure_python.transform_file_processor import PythonTransformFileProcessor
+from data_processing.runtime.pure_python.transform_orchestrator import orchestrate
+from data_processing.runtime.pure_python.transform_launcher import PythonTransformLauncher

data_prep_toolkit-0.1.1/src/data_processing/runtime/pure_python/runtime_configuration.py

@@ -0,0 +1,24 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from data_processing.runtime import TransformRuntimeConfiguration
+from data_processing.transform import TransformConfiguration
+
+
+class PythonTransformRuntimeConfiguration(TransformRuntimeConfiguration):
+    def __init__(self, transform_config: TransformConfiguration):
+        """
+        Initialization
+        :param transform_config - base configuration class
+        """
+        self.transform_config = transform_config
+        super().__init__(transform_config=transform_config)

data_prep_toolkit-0.1.1/src/data_processing/runtime/pure_python/transform_file_processor.py

@@ -0,0 +1,53 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from typing import Any
+
+from data_processing.data_access import DataAccessFactoryBase
+from data_processing.runtime import (
+    AbstractTransformFileProcessor,
+)
+from data_processing.transform import TransformStatistics
+from data_processing.runtime.pure_python import PythonTransformRuntimeConfiguration
+
+
+class PythonTransformFileProcessor(AbstractTransformFileProcessor):
+    """
+    This is the class implementing the worker class processing of a single file
+    """
+
+    def __init__(
+        self,
+        data_access_factory: DataAccessFactoryBase,
+        statistics: TransformStatistics,
+        runtime_configuration: PythonTransformRuntimeConfiguration,
+    ):
+        """
+        Init method
+        :param data_access_factory - data access factory
+        :param statistics - reference to statistics class
+        :param runtime_configuration: transform configuration class
+        """
+        # Create data access
+        super().__init__()
+        self.data_access = data_access_factory.create_data_access()
+        # Add data access and statistics to the processor parameters
+        transform_params = dict(runtime_configuration.get_transform_params())
+        transform_params["data_access"] = self.data_access
+        transform_params["statistics"] = statistics
+        # Create local processor
+        self.transform = runtime_configuration.get_transform_class()(transform_params)
+        # Create statistics
+        self.stats = statistics
+
+    def _publish_stats(self, stats: dict[str, Any]) -> None:
+        self.stats.add_stats(stats)

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.1}/src/data_processing/runtime/pure_python/transform_launcher.py

@@ -15,9 +15,11 @@ import time
 
 from data_processing.data_access import DataAccessFactory, DataAccessFactoryBase
 from data_processing.runtime import TransformExecutionConfiguration
-from data_processing.runtime.pure_python import PythonLauncherConfiguration, orchestrate
+from data_processing.runtime.pure_python import (
+    PythonTransformRuntimeConfiguration,
+    orchestrate,
+)
 from data_processing.runtime.transform_launcher import AbstractTransformLauncher
-from data_processing.transform import TransformConfiguration
 from data_processing.utils import get_logger
 
 
@@ -31,18 +33,16 @@ class PythonTransformLauncher(AbstractTransformLauncher):
 
     def __init__(
         self,
-        # transform_runtime_config: PythonLauncherConfiguration,
-        transform_config: TransformConfiguration,
+        runtime_config: PythonTransformRuntimeConfiguration,
         data_access_factory: DataAccessFactoryBase = DataAccessFactory(),
     ):
         """
         Creates driver
-        :param transform_runtime_config: transform runtime factory
+        :param runtime_config: transform runtime factory
         :param data_access_factory: the factory to create DataAccess instances.
         """
-        super().__init__(transform_config, data_access_factory)
-        self.transform_runtime_config = PythonLauncherConfiguration(transform_config)
-        self.execution_config = TransformExecutionConfiguration(name=self.transform_runtime_config.get_name())
+        super().__init__(runtime_config, data_access_factory)
+        self.execution_config = TransformExecutionConfiguration(name=runtime_config.get_name())
 
     def __get_parameters(self) -> bool:
         """
@@ -57,12 +57,12 @@ class PythonTransformLauncher(AbstractTransformLauncher):
             formatter_class=argparse.RawTextHelpFormatter,
         )
         # add additional arguments
-        self.transform_runtime_config.add_input_params(parser=parser)
+        self.runtime_config.add_input_params(parser=parser)
         self.data_access_factory.add_input_params(parser=parser)
         self.execution_config.add_input_params(parser=parser)
         args = parser.parse_args()
         return (
-            self.transform_runtime_config.apply_input_params(args=args)
+            self.runtime_config.apply_input_params(args=args)
             and self.execution_config.apply_input_params(args=args)
             and self.data_access_factory.apply_input_params(args=args)
         )
@@ -78,7 +78,7 @@ class PythonTransformLauncher(AbstractTransformLauncher):
             logger.debug("Starting orchestrator")
             res = orchestrate(
                 data_access_factory=self.data_access_factory,
-                transform_config=self.transform_runtime_config,
+                runtime_config=self.runtime_config,
                 execution_config=self.execution_config,
             )
             logger.debug("Completed orchestrator")

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.1}/src/data_processing/runtime/pure_python/transform_orchestrator.py

@@ -15,11 +15,11 @@ import traceback
 from datetime import datetime
 
 from data_processing.data_access import DataAccessFactoryBase
-from data_processing.runtime import TransformExecutionConfiguration
-from data_processing.runtime.pure_python import (
-    PythonLauncherConfiguration,
-    TransformTableProcessor,
+from data_processing.runtime import (
+    TransformExecutionConfiguration,
+    TransformRuntimeConfiguration,
 )
+from data_processing.runtime.pure_python import PythonTransformFileProcessor
 from data_processing.transform import TransformStatistics
 from data_processing.utils import get_logger
 
@@ -29,17 +29,18 @@ logger = get_logger(__name__)
 
 def orchestrate(
     data_access_factory: DataAccessFactoryBase,
-    transform_config: PythonLauncherConfiguration,
+    runtime_config: TransformRuntimeConfiguration,
     execution_config: TransformExecutionConfiguration,
 ) -> int:
     """
     orchestrator for transformer execution
     :param data_access_factory: data access factory
-    :param transform_config: transformer configuration
+    :param runtime_config: transformer configuration
+    :param execution_config: execution configuration
     :return: 0 - success or 1 - failure
     """
     start_ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    logger.info(f"orchestrator {transform_config.get_name()} started at {start_ts}")
+    logger.info(f"orchestrator {runtime_config.get_name()} started at {start_ts}")
     try:
         # create data access
         data_access = data_access_factory.create_data_access()
@@ -59,15 +60,15 @@ def orchestrate(
         # create statistics
         statistics = TransformStatistics()
         # create executor
-        executor = TransformTableProcessor(
-            data_access_factory=data_access_factory, statistics=statistics, params=transform_config
+        executor = PythonTransformFileProcessor(
+            data_access_factory=data_access_factory, statistics=statistics, runtime_configuration=runtime_config
         )
         # process data
-        logger.debug(f"{transform_config.get_name()} Begin processing files")
+        logger.debug(f"{runtime_config.get_name()} Begin processing files")
         t_start = time.time()
         completed = 0
         for path in files:
-            executor.process_data(path)
+            executor.process_file(path)
             completed += 1
             if completed % print_interval == 0:
                 logger.info(f"Completed {completed} files in {(time.time() - t_start)/60} min")
@@ -81,7 +82,7 @@ def orchestrate(
         stats = statistics.get_execution_stats()
         # build and save metadata
         logger.debug("Building job metadata")
-        input_params = transform_config.get_transform_metadata()
+        input_params = runtime_config.get_transform_metadata()
         metadata = {
             "pipeline": execution_config.pipeline_id,
             "job details": execution_config.job_details

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.1}/src/data_processing/runtime/ray/__init__.py

@@ -1,8 +1,8 @@
 from data_processing.runtime.ray.ray_utils import RayUtils
 from data_processing.runtime.ray.transform_statistics import TransformStatisticsRay
-from data_processing.runtime.ray.transform_table_processor import TransformTableProcessorRay
-from data_processing.runtime.ray.transform_runtime import DefaultTableTransformRuntimeRay
-from data_processing.runtime.ray.transform_launch_configuration import RayLauncherConfiguration
-from data_processing.runtime.ray.transform_orchestrator_configuration import TransformOrchestratorConfiguration
+from data_processing.runtime.ray.transform_runtime import DefaultRayTransformRuntime
+from data_processing.runtime.ray.runtime_configuration import RayTransformRuntimeConfiguration
+from data_processing.runtime.ray.transform_file_processor import RayTransformFileProcessor
+from data_processing.runtime.ray.execution_configuration import RayTransformExecutionConfiguration
 from data_processing.runtime.ray.transform_orchestrator import orchestrate
 from data_processing.runtime.ray.transform_launcher import RayTransformLauncher

data_prep_toolkit-0.0.1.dev12/src/data_processing/runtime/ray/transform_orchestrator_configuration.py → data_prep_toolkit-0.1.1/src/data_processing/runtime/ray/execution_configuration.py

@@ -24,7 +24,7 @@ logger = get_logger(__name__)
 cli_prefix = "runtime_"
 
 
-class TransformOrchestratorConfiguration(TransformExecutionConfiguration):
+class RayTransformExecutionConfiguration(TransformExecutionConfiguration):
     """
     A class specifying and validating Ray orchestrator configuration
     """
@@ -33,7 +33,7 @@ class TransformOrchestratorConfiguration(TransformExecutionConfiguration):
         """
         Initialization
         """
-        super().__init__(name=name, pp=False)
+        super().__init__(name=name, print_params=False)
         self.worker_options = {}
         self.n_workers = 1
         self.creation_delay = 0
@@ -91,10 +91,8 @@ class TransformOrchestratorConfiguration(TransformExecutionConfiguration):
 
         # print them
         logger.info(f"number of workers {self.n_workers} worker options {self.worker_options}")
-        logger.info(f"pipeline id {self.pipeline_id}; number workers {self.n_workers}")
-        logger.info(f"job details {self.job_details}")
-        logger.info(f"code location {self.code_location}")
         logger.info(f"actor creation delay {self.creation_delay}")
+        logger.info(f"job details {self.job_details}")
         return True
 
     def get_input_params(self) -> dict[str, Any]:

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.1}/src/data_processing/runtime/ray/ray_utils.py

@@ -132,12 +132,12 @@ class RayUtils:
         completed = 0
         for path in files:
             if executors.has_free():  # still have room
-                executors.submit(lambda a, v: a.process_data.remote(v), path)
+                executors.submit(lambda a, v: a.process_file.remote(v), path)
                 running = running + 1
                 files_in_progress_gauge.set(running)
             else:  # need to wait for some actors
                 executors.get_next_unordered()
-                executors.submit(lambda a, v: a.process_data.remote(v), path)
+                executors.submit(lambda a, v: a.process_file.remote(v), path)
                 completed = completed + 1
                 files_completed_gauge.set(completed)
                 RayUtils.get_available_resources(

data_prep_toolkit-0.1.1/src/data_processing/runtime/ray/runtime_configuration.py

@@ -0,0 +1,38 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from data_processing.runtime import TransformRuntimeConfiguration
+from data_processing.runtime.ray import DefaultRayTransformRuntime
+from data_processing.transform import TransformConfiguration
+
+
+class RayTransformRuntimeConfiguration(TransformRuntimeConfiguration):
+    def __init__(
+        self,
+        transform_config: TransformConfiguration,
+        runtime_class: type[DefaultRayTransformRuntime] = DefaultRayTransformRuntime,
+    ):
+        """
+        Initialization
+        :param transform_config - base configuration class
+        :param runtime_class: implementation of the transform runtime
+        :param remove_from_metadata - list of parameters to remove from metadata
+        """
+        super().__init__(transform_config=transform_config)
+        self.runtime_class = runtime_class
+
+    def create_transform_runtime(self) -> DefaultRayTransformRuntime:
+        """
+        Create transform runtime with the parameters captured during apply_input_params()
+        :return: transform runtime object
+        """
+        return self.runtime_class(self.transform_config.get_transform_params())