data-prep-toolkit 0.2.1.dev1__tar.gz → 0.2.1.dev2__tar.gz

{data_prep_toolkit-0.2.1.dev1/src/data_prep_toolkit.egg-info → data_prep_toolkit-0.2.1.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data_prep_toolkit
-Version: 0.2.1.dev1
+Version: 0.2.1.dev2
 Summary: Data Preparation Toolkit Library
 Author-email: David Wood <dawood@us.ibm.com>, Boris Lublinsky <blublinsky@ibm.com>
 License: Apache-2.0

{data_prep_toolkit-0.2.1.dev1 → data_prep_toolkit-0.2.1.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit"
-version = "0.2.1.dev1"
+version = "0.2.1.dev2"
 requires-python = ">=3.10"
 keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 description = "Data Preparation Toolkit Library"

{data_prep_toolkit-0.2.1.dev1 → data_prep_toolkit-0.2.1.dev2/src/data_prep_toolkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data_prep_toolkit
-Version: 0.2.1.dev1
+Version: 0.2.1.dev2
 Summary: Data Preparation Toolkit Library
 Author-email: David Wood <dawood@us.ibm.com>, Boris Lublinsky <blublinsky@ibm.com>
 License: Apache-2.0

{data_prep_toolkit-0.2.1.dev1 → data_prep_toolkit-0.2.1.dev2}/src/data_prep_toolkit.egg-info/SOURCES.txt

@@ -20,11 +20,13 @@ src/data_processing/runtime/runtime_configuration.py
 src/data_processing/runtime/transform_file_processor.py
 src/data_processing/runtime/transform_launcher.py
 src/data_processing/runtime/pure_python/__init__.py
+src/data_processing/runtime/pure_python/execution_configuration.py
 src/data_processing/runtime/pure_python/runtime_configuration.py
 src/data_processing/runtime/pure_python/transform_file_processor.py
 src/data_processing/runtime/pure_python/transform_invoker.py
 src/data_processing/runtime/pure_python/transform_launcher.py
 src/data_processing/runtime/pure_python/transform_orchestrator.py
+src/data_processing/runtime/pure_python/transform_runtime.py
 src/data_processing/test_support/__init__.py
 src/data_processing/test_support/abstract_test.py
 src/data_processing/test_support/data_access/__init__.py
@@ -74,5 +76,6 @@ test/data_processing_tests/invoker/python_invoker_test.py
 test/data_processing_tests/launch/pure_python/launcher_test.py
 test/data_processing_tests/launch/pure_python/multi_launcher_test.py
 test/data_processing_tests/launch/pure_python/test_noop_launch.py
+test/data_processing_tests/launch/pure_python/test_noop_python_multiprocessor.py
 test/data_processing_tests/transform/test_noop.py
 test/data_processing_tests/util/transform_utils_test.py

{data_prep_toolkit-0.2.1.dev1 → data_prep_toolkit-0.2.1.dev2}/src/data_processing/runtime/pure_python/__init__.py

@@ -1,5 +1,8 @@
+from data_processing.runtime.pure_python.transform_runtime import DefaultPythonTransformRuntime
 from data_processing.runtime.pure_python.runtime_configuration import PythonTransformRuntimeConfiguration
-from data_processing.runtime.pure_python.transform_file_processor import PythonTransformFileProcessor
+from data_processing.runtime.pure_python.execution_configuration import PythonTransformExecutionConfiguration
+from data_processing.runtime.pure_python.transform_file_processor import (PythonTransformFileProcessor,
+                                                                          PythonPoolTransformFileProcessor)
 from data_processing.runtime.pure_python.transform_orchestrator import orchestrate
 from data_processing.runtime.pure_python.transform_launcher import PythonTransformLauncher
 from data_processing.runtime.pure_python.transform_invoker import invoke_transform, execute_python_transform

data_prep_toolkit-0.2.1.dev2/src/data_processing/runtime/pure_python/execution_configuration.py

@@ -0,0 +1,72 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import argparse
+from typing import Any
+
+from data_processing.runtime import TransformExecutionConfiguration
+from data_processing.utils import CLIArgumentProvider, get_logger
+
+
+logger = get_logger(__name__)
+
+
+cli_prefix = "runtime_"
+
+
+class PythonTransformExecutionConfiguration(TransformExecutionConfiguration):
+    """
+    A class specifying and validating Python orchestrator configuration
+    """
+
+    def __init__(self, name: str):
+        """
+        Initialization
+        """
+        super().__init__(name=name, print_params=False)
+        self.num_processors = 0
+
+    def add_input_params(self, parser: argparse.ArgumentParser) -> None:
+        """
+        This method adds transformer specific parameter to parser
+        :param parser: parser
+        :return:
+        """
+        parser.add_argument(f"--{cli_prefix}num_processors", type=int, default=0, help="size of multiprocessing pool")
+
+        return TransformExecutionConfiguration.add_input_params(self, parser=parser)
+
+    def apply_input_params(self, args: argparse.Namespace) -> bool:
+        """
+        Validate transformer specific parameters
+        :param args: user defined arguments
+        :return: True, if validate pass or False otherwise
+        """
+        if not TransformExecutionConfiguration.apply_input_params(self, args=args):
+            return False
+        captured = CLIArgumentProvider.capture_parameters(args, cli_prefix, False)
+        # store parameters locally
+        self.num_processors = captured["num_processors"]
+        # print them
+        if self.num_processors > 0:
+            # we are using multiprocessing
+            logger.info(f"using multiprocessing, num processors {self.num_processors}")
+        return True
+
+    def get_input_params(self) -> dict[str, Any]:
+        """
+        get input parameters for job_input_params in metadata
+        :return: dictionary of parameters
+        """
+        return {
+            "num_processors": self.num_processors
+        }

{data_prep_toolkit-0.2.1.dev1 → data_prep_toolkit-0.2.1.dev2}/src/data_processing/runtime/pure_python/runtime_configuration.py

@@ -12,13 +12,25 @@
 
 from data_processing.runtime import TransformRuntimeConfiguration
 from data_processing.transform import TransformConfiguration
+from data_processing.runtime.pure_python import DefaultPythonTransformRuntime
 
 
 class PythonTransformRuntimeConfiguration(TransformRuntimeConfiguration):
-    def __init__(self, transform_config: TransformConfiguration):
+    def __init__(self,
+                 transform_config: TransformConfiguration,
+                 runtime_class: type[DefaultPythonTransformRuntime] = DefaultPythonTransformRuntime,
+                 ):
         """
         Initialization
         :param transform_config - base configuration class
+        :param runtime_class: implementation of the transform runtime
         """
-        self.transform_config = transform_config
+        self.runtime_class = runtime_class
         super().__init__(transform_config=transform_config)
+
+    def create_transform_runtime(self) -> DefaultPythonTransformRuntime:
+        """
+        Create transform runtime with the parameters captured during apply_input_params()
+        :return: transform runtime object
+        """
+        return self.runtime_class(self.transform_config.get_transform_params())

data_prep_toolkit-0.2.1.dev2/src/data_processing/runtime/pure_python/transform_file_processor.py

@@ -0,0 +1,109 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from typing import Any
+
+from data_processing.data_access import DataAccessFactoryBase
+from data_processing.runtime import AbstractTransformFileProcessor
+from data_processing.transform import AbstractBinaryTransform, TransformStatistics
+
+
+class PythonTransformFileProcessor(AbstractTransformFileProcessor):
+    """
+    This is the class implementing the worker class processing of a single file
+    """
+
+    def __init__(
+        self,
+        data_access_factory: DataAccessFactoryBase,
+        statistics: TransformStatistics,
+        transform_params: dict[str, Any],
+        transform_class: type[AbstractBinaryTransform],
+    ):
+        """
+        Init method
+        :param data_access_factory - data access factory
+        :param statistics - reference to statistics class
+        :param transform_params - transform parameters
+        :param transform_class: transform class
+        """
+        # invoke superclass
+        super().__init__(
+            data_access_factory=data_access_factory,
+            transform_parameters=dict(transform_params),
+        )
+        self.transform_params["statistics"] = statistics
+        # Create local processor
+        self.transform = transform_class(self.transform_params)
+        # Create statistics
+        self.stats = statistics
+
+    def _publish_stats(self, stats: dict[str, Any]) -> None:
+        self.stats.add_stats(stats)
+
+
+class PythonPoolTransformFileProcessor(AbstractTransformFileProcessor):
+    """
+    This is the class implementing the worker class processing of a single file
+    """
+
+    def __init__(
+            self,
+            data_access_factory: DataAccessFactoryBase,
+            transform_params: dict[str, Any],
+            transform_class: type[AbstractBinaryTransform],
+    ):
+        """
+        Init method
+        :param data_access_factory - data access factory
+        :param transform_params - transform parameters
+        :param transform_class: transform class
+        """
+        super().__init__(
+            data_access_factory=data_access_factory,
+            transform_parameters=dict(transform_params),
+        )
+        # Add data access and statistics to the processor parameters
+        self.transform_params["data_access"] = self.data_access
+        self.transform_class = transform_class
+        self.transform = None
+
+    def process_file(self, f_name: str) -> dict[str, Any]:
+        # re initialize statistics
+        self.stats = {}
+        if self.transform is None:
+            # create transform. Make sure to do this locally
+            self.transform = self.transform_class(self.transform_params)
+        # Invoke superclass method
+        super().process_file(f_name=f_name)
+        # return collected statistics
+        return self.stats
+
+    def flush(self) -> dict[str, Any]:
+        # re initialize statistics
+        self.stats = {}
+        # Invoke superclass method
+        super().flush()
+        # return collected statistics
+        return self.stats
+
+    def _publish_stats(self, stats: dict[str, Any]) -> None:
+        """
+        Publish statistics (to the local dictionary)
+        :param stats: statistics dictionary
+        :return: None
+        """
+        for key, val in stats.items():
+            # for all key/values
+            if val > 0:
+                # for values greater then 0
+                self.stats[key] = self.stats.get(key, 0) + val

{data_prep_toolkit-0.2.1.dev1 → data_prep_toolkit-0.2.1.dev2}/src/data_processing/runtime/pure_python/transform_launcher.py

@@ -14,9 +14,9 @@ import argparse
 import time
 
 from data_processing.data_access import DataAccessFactory, DataAccessFactoryBase
-from data_processing.runtime import TransformExecutionConfiguration
 from data_processing.runtime.pure_python import (
     PythonTransformRuntimeConfiguration,
+    PythonTransformExecutionConfiguration,
     orchestrate,
 )
 from data_processing.runtime.transform_launcher import AbstractTransformLauncher
@@ -42,7 +42,7 @@ class PythonTransformLauncher(AbstractTransformLauncher):
         :param data_access_factory: the factory to create DataAccess instances.
         """
         super().__init__(runtime_config, data_access_factory)
-        self.execution_config = TransformExecutionConfiguration(name=runtime_config.get_name())
+        self.execution_config = PythonTransformExecutionConfiguration(name=runtime_config.get_name())
 
     def __get_parameters(self) -> bool:
         """
@@ -85,7 +85,7 @@ class PythonTransformLauncher(AbstractTransformLauncher):
         except Exception as e:
             logger.info(f"Exception running orchestration\n{e}")
         finally:
-            logger.info(f"Completed execution in {(time.time() - start)/60.} min, execution result {res}")
+            logger.info(f"Completed execution in {round((time.time() - start)/60., 3)} min, execution result {res}")
             return res
 
     def launch(self) -> int:

data_prep_toolkit-0.2.1.dev2/src/data_processing/runtime/pure_python/transform_orchestrator.py

@@ -0,0 +1,201 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import time
+from typing import Any
+from multiprocessing import Pool
+import traceback
+from datetime import datetime
+
+from data_processing.data_access import DataAccessFactoryBase
+from data_processing.runtime.pure_python import (
+    PythonTransformExecutionConfiguration,
+    PythonTransformRuntimeConfiguration,
+    PythonTransformFileProcessor,
+    PythonPoolTransformFileProcessor,
+)
+from data_processing.transform import TransformStatistics, AbstractBinaryTransform
+from data_processing.utils import get_logger
+
+
+logger = get_logger(__name__)
+
+
+def orchestrate(
+    data_access_factory: DataAccessFactoryBase,
+    runtime_config: PythonTransformRuntimeConfiguration,
+    execution_config: PythonTransformExecutionConfiguration,
+) -> int:
+    """
+    orchestrator for transformer execution
+    :param data_access_factory: data access factory
+    :param runtime_config: transformer configuration
+    :param execution_config: execution configuration
+    :return: 0 - success or 1 - failure
+    """
+    start_ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    logger.info(f"orchestrator {runtime_config.get_name()} started at {start_ts}")
+    # create statistics
+    statistics = TransformStatistics()
+    # create data access
+    data_access = data_access_factory.create_data_access()
+    if data_access is None:
+        logger.error("No DataAccess instance provided - exiting")
+        return 1
+    # create additional execution parameters
+    runtime = runtime_config.create_transform_runtime()
+    try:
+        # Get files to process
+        files, profile, retries = data_access.get_files_to_process()
+        if len(files) == 0:
+            logger.error("No input files to process - exiting")
+            return 0
+        if retries > 0:
+            statistics.add_stats({"data access retries": retries})
+        logger.info(f"Number of files is {len(files)}, source profile {profile}")
+        # Print interval
+        print_interval = int(len(files) / 100)
+        if print_interval == 0:
+            print_interval = 1
+        logger.debug(f"{runtime_config.get_name()} Begin processing files")
+        if execution_config.num_processors > 0:
+            # using multiprocessor pool for execution
+            statistics = _process_transforms_multiprocessor(files=files, size=execution_config.num_processors,
+                                                            data_access_factory=data_access_factory,
+                                                            print_interval=print_interval,
+                                                            transform_params=runtime.get_transform_config(
+                                                                data_access_factory=data_access_factory,
+                                                                statistics=statistics, files=files),
+                                                            transform_class=runtime_config.get_transform_class())
+        else:
+            # using sequential execution
+            _process_transforms(files=files, data_access_factory=data_access_factory,
+                                print_interval=print_interval, statistics=statistics,
+                                transform_params=runtime.get_transform_config(
+                                    data_access_factory=data_access_factory,
+                                    statistics=statistics, files=files),
+                                transform_class=runtime_config.get_transform_class())
+        status = "success"
+        return_code = 0
+    except Exception as e:
+        logger.error(f"Exception during execution {e}: {traceback.print_exc()}")
+        return_code = 1
+        status = "failure"
+    try:
+        # Compute execution statistics
+        logger.debug("Computing execution stats")
+        stats = statistics.get_execution_stats()
+        stats["processing_time"] = round(stats["processing_time"], 3)
+        # build and save metadata
+        logger.debug("Building job metadata")
+        input_params = runtime_config.get_transform_metadata()
+        runtime.compute_execution_stats(stats=statistics)
+        metadata = {
+            "pipeline": execution_config.pipeline_id,
+            "job details": execution_config.job_details
+            | {
+                "start_time": start_ts,
+                "end_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                "status": status,
+            },
+            "code": execution_config.code_location,
+            "job_input_params":
+                input_params | data_access_factory.get_input_params() | execution_config.get_input_params(),
+            "job_output_stats": stats,
+        }
+        logger.debug(f"Saving job metadata: {metadata}.")
+        data_access.save_job_metadata(metadata)
+        logger.debug("Saved job metadata.")
+        return return_code
+    except Exception as e:
+        logger.error(f"Exception during execution {e}: {traceback.print_exc()}")
+        return 1
+
+
+def _process_transforms(files: list[str], print_interval: int, data_access_factory: DataAccessFactoryBase,
+                        statistics: TransformStatistics, transform_params: dict[str, Any],
+                        transform_class: type[AbstractBinaryTransform]) -> None:
+    """
+    Process transforms sequentially
+    :param files: list of files to process
+    :param statistics: statistics class
+    :param print_interval: print interval
+    :param data_access_factory: data access factory
+    :param transform_params - transform parameters
+    :param transform_class: transform class
+    :return: metadata for the execution
+
+    :return: None
+    """
+    # create executor
+    executor = PythonTransformFileProcessor(data_access_factory=data_access_factory, statistics=statistics,
+                                            transform_params=transform_params, transform_class=transform_class)
+    # process data
+    t_start = time.time()
+    completed = 0
+    for path in files:
+        executor.process_file(path)
+        completed += 1
+        if completed % print_interval == 0:
+            logger.info(
+                f"Completed {completed} files ({round(100 * completed / len(files), 2)}%) "
+                f"in {round((time.time() - t_start)/60., 3)} min"
+            )
+    logger.info(f"Done processing {completed} files, waiting for flush() completion.")
+    # invoke flush to ensure that all results are returned
+    start = time.time()
+    executor.flush()
+    logger.info(f"done flushing in {round(time.time() - start, 3)} sec")
+
+
+def _process_transforms_multiprocessor(files: list[str], size: int, print_interval: int,
+                                       data_access_factory: DataAccessFactoryBase, transform_params: dict[str, Any],
+                                       transform_class: type[AbstractBinaryTransform]) -> TransformStatistics:
+    """
+    Process transforms using multiprocessing pool
+    :param files: list of files to process
+    :param size: pool size
+    :param print_interval: print interval
+    :param data_access_factory: data access factory
+    :param transform_params - transform parameters
+    :param transform_class: transform class
+    :return: metadata for the execution
+    """
+    # result statistics
+    statistics = TransformStatistics()
+    # create processor
+    processor = PythonPoolTransformFileProcessor(data_access_factory=data_access_factory,
+                                                 transform_params=transform_params, transform_class=transform_class)
+    completed = 0
+    t_start = time.time()
+    # create multiprocessing pool
+    with Pool(processes=size) as pool:
+        # execute for every input file
+        for result in pool.imap_unordered(processor.process_file, files):
+            completed += 1
+            # accumulate statistics
+            statistics.add_stats(result)
+            if completed % print_interval == 0:
+                # print intermediate statistics
+                logger.info(
+                    f"Completed {completed} files ({round(100 * completed / len(files), 2)}%) "
+                    f"in {round((time.time() - t_start)/60., 3)} min"
+                )
+        logger.info(f"Done processing {completed} files, waiting for flush() completion.")
+        results = [{}] * size
+        # flush
+        for i in range(size):
+            results[i] = pool.apply_async(processor.flush)
+        for s in results:
+            statistics.add_stats(s.get())
+    logger.info(f"done flushing in {time.time() - t_start} sec")
+    return statistics

data_prep_toolkit-0.2.1.dev2/src/data_processing/runtime/pure_python/transform_runtime.py

@@ -0,0 +1,53 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from typing import Any
+
+from data_processing.data_access import DataAccessFactoryBase
+from data_processing.transform import TransformStatistics
+
+
+class DefaultPythonTransformRuntime:
+    """
+    Transformer runtime used by processor to to create Transform specific environment
+    """
+
+    def __init__(self, params: dict[str, Any]):
+        """
+        Create/config this runtime.
+        :param params: parameters, often provided by the CLI arguments as defined by a TableTansformConfiguration.
+        """
+        self.params = params
+
+    def get_transform_config(
+        self, data_access_factory: DataAccessFactoryBase, statistics: TransformStatistics, files: list[str]
+    ) -> dict[str, Any]:
+        """
+        Get the dictionary of configuration that will be provided to the transform's initializer.
+        This is the opportunity for this runtime to create a new set of configuration based on the
+        config/params provided to this instance's initializer.  This may include the addition
+        of new configuration data such as ray shared memory, new actors, etc, that might be needed and
+        expected by the transform in its initializer and/or transform() methods.
+        :param data_access_factory - data access factory class being used by the RayOrchestrator.
+        :param statistics - reference to statistics actor
+        :param files - list of files to process
+        :return: dictionary of transform init params
+        """
+        return self.params
+
+    def compute_execution_stats(self, stats: TransformStatistics) -> None:
+        """
+        Update/augment the given statistics object with runtime-specific additions/modifications.
+        :param stats: output of statistics as aggregated across all calls to all transforms.
+        :return: job execution statistics.  These are generally reported as metadata by the Ray Orchestrator.
+        """
+        return stats

{data_prep_toolkit-0.2.1.dev1 → data_prep_toolkit-0.2.1.dev2}/src/data_processing/runtime/transform_file_processor.py

@@ -13,6 +13,7 @@ import time
 import traceback
 from typing import Any
 
+from data_processing.data_access import DataAccessFactoryBase
 from data_processing.utils import TransformUtils, UnrecoverableException, get_logger
 
 
@@ -21,17 +22,30 @@ class AbstractTransformFileProcessor:
     This is the the base class implementing processing of a single binary file
     """
 
-    def __init__(self):
+    def __init__(
+            self,
+            data_access_factory: DataAccessFactoryBase,
+            transform_parameters: dict[str, Any],
+    ):
         """
         Init method
+        :param data_access_factory: Data Access Factory
+        :param transform_parameters: Transform parameters
         """
-        self.data_access = None
+        self.logger = get_logger(__name__)
+        # validate parameters
+        if data_access_factory is None:
+            self.logger.error("Transform file processor: data access factory is not specified")
+            raise UnrecoverableException("data access factory is None")
         self.transform = None
         self.stats = None
         self.last_file_name = None
         self.last_extension = None
         self.last_file_name_next_index = None
-        self.logger = get_logger(__name__)
+        self.data_access = data_access_factory.create_data_access()
+        # Add data access and statistics to the processor parameters
+        self.transform_params = transform_parameters
+        self.transform_params["data_access"] = self.data_access
 
     def process_file(self, f_name: str) -> None:
         """
@@ -83,14 +97,15 @@ class AbstractTransformFileProcessor:
         """
         if self.last_file_name is None:
             # for some reason a given worker never processed anything. Happens in testing
-            # when the amount of workers is greater then the amount of files
+            # when the amount of workers is greater than the amount of files
             self.logger.debug("skipping flush, no name for file is defined")
             return
         try:
             t_start = time.time()
             # get flush results
             self.logger.debug(
-                f"Begin flushing transform, last file name {self.last_file_name}, last index {self.last_file_name_next_index}"
+                f"Begin flushing transform, last file name {self.last_file_name}, "
+                f"last index {self.last_file_name_next_index}"
             )
             out_files, stats = self.transform.flush_binary()
             self.logger.debug(f"Done flushing transform, got {len(out_files)} files")
@@ -148,7 +163,7 @@ class AbstractTransformFileProcessor:
                 else:
                     self.last_file_name_next_index += 1
             case _:
-                # we have more then 1 file
+                # we have more than 1 file
                 file_sizes = 0
                 output_file_name = self.data_access.get_output_location(path=self.last_file_name)
                 start_index = self.last_file_name_next_index

{data_prep_toolkit-0.2.1.dev1 → data_prep_toolkit-0.2.1.dev2}/src/data_processing/utils/log.py

@@ -26,7 +26,13 @@ def get_log_level(name: str = None) -> str:
     return level_name
 
 
+__logger_cache = {}
+
+
 def get_logger(name: str, level=None, file=None) -> logging.Logger:
+    logger = __logger_cache.get(name, None)
+    if logger is not None:
+        return logger
     logger = logging.getLogger(name)
     if level is None:
         level = get_log_level(name)
@@ -50,6 +56,7 @@ def get_logger(name: str, level=None, file=None) -> logging.Logger:
         logger.addHandler(f_handler)
 
     # Add handlers to the logger
+    __logger_cache[name] = logger
     return logger
 
 

{data_prep_toolkit-0.2.1.dev1 → data_prep_toolkit-0.2.1.dev2}/test/data_processing_tests/launch/pure_python/test_noop_launch.py

@@ -20,11 +20,6 @@ from data_processing.test_support.launch.transform_test import (
 from data_processing.test_support.transform import NOOPPythonTransformConfiguration
 
 
-table = pa.Table.from_pydict({"name": pa.array(["Tom"]), "age": pa.array([23])})
-expected_table = table  # We're a noop after all.
-expected_metadata_list = [{"nfiles": 1, "nrows": 1}, {}]  # transform() result  # flush() result
-
-
 class TestRayNOOPTransform(AbstractTransformLauncherTest):
     """
     Extends the super-class to define the test data for the tests defined there.

data_prep_toolkit-0.2.1.dev2/test/data_processing_tests/launch/pure_python/test_noop_python_multiprocessor.py

@@ -0,0 +1,37 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import os
+
+from data_processing.runtime.pure_python import PythonTransformLauncher
+from data_processing.test_support.launch.transform_test import (
+    AbstractTransformLauncherTest,
+)
+from data_processing.test_support.transform import NOOPPythonTransformConfiguration
+
+
+class TestPythonNOOPTransform(AbstractTransformLauncherTest):
+    """
+    Extends the super-class to define the test data for the tests defined there.
+    The name of this class MUST begin with the word Test so that pytest recognizes it as a test class.
+    """
+
+    def get_test_transform_fixtures(self) -> list[tuple]:
+        basedir = "../../../../test-data/data_processing/python/noop/"
+        basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), basedir))
+        fixtures = []
+        launcher = PythonTransformLauncher(NOOPPythonTransformConfiguration())
+        fixtures.append((
+            launcher,
+            {"noop_sleep_sec": 0, "runtime_num_processors": 2},
+            basedir + "/input", basedir + "/expected"))
+        return fixtures

data_prep_toolkit-0.2.1.dev1/src/data_processing/runtime/pure_python/transform_file_processor.py

@@ -1,51 +0,0 @@
-# (C) Copyright IBM Corp. 2024.
-# Licensed under the Apache License, Version 2.0 (the “License”);
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an “AS IS” BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-from typing import Any
-
-from data_processing.data_access import DataAccessFactoryBase
-from data_processing.runtime import AbstractTransformFileProcessor
-from data_processing.runtime.pure_python import PythonTransformRuntimeConfiguration
-from data_processing.transform import TransformStatistics
-
-
-class PythonTransformFileProcessor(AbstractTransformFileProcessor):
-    """
-    This is the class implementing the worker class processing of a single file
-    """
-
-    def __init__(
-        self,
-        data_access_factory: DataAccessFactoryBase,
-        statistics: TransformStatistics,
-        runtime_configuration: PythonTransformRuntimeConfiguration,
-    ):
-        """
-        Init method
-        :param data_access_factory - data access factory
-        :param statistics - reference to statistics class
-        :param runtime_configuration: transform configuration class
-        """
-        # Create data access
-        super().__init__()
-        self.data_access = data_access_factory.create_data_access()
-        # Add data access and statistics to the processor parameters
-        transform_params = dict(runtime_configuration.get_transform_params())
-        transform_params["data_access"] = self.data_access
-        transform_params["statistics"] = statistics
-        # Create local processor
-        self.transform = runtime_configuration.get_transform_class()(transform_params)
-        # Create statistics
-        self.stats = statistics
-
-    def _publish_stats(self, stats: dict[str, Any]) -> None:
-        self.stats.add_stats(stats)

data_prep_toolkit-0.2.1.dev1/src/data_processing/runtime/pure_python/transform_orchestrator.py

@@ -1,116 +0,0 @@
-# (C) Copyright IBM Corp. 2024.
-# Licensed under the Apache License, Version 2.0 (the “License”);
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an “AS IS” BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-import time
-import traceback
-from datetime import datetime
-
-from data_processing.data_access import DataAccessFactoryBase
-from data_processing.runtime import (
-    TransformExecutionConfiguration,
-    TransformRuntimeConfiguration,
-)
-from data_processing.runtime.pure_python import PythonTransformFileProcessor
-from data_processing.transform import TransformStatistics
-from data_processing.utils import get_logger
-
-
-logger = get_logger(__name__)
-
-
-def orchestrate(
-    data_access_factory: DataAccessFactoryBase,
-    runtime_config: TransformRuntimeConfiguration,
-    execution_config: TransformExecutionConfiguration,
-) -> int:
-    """
-    orchestrator for transformer execution
-    :param data_access_factory: data access factory
-    :param runtime_config: transformer configuration
-    :param execution_config: execution configuration
-    :return: 0 - success or 1 - failure
-    """
-    start_ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    logger.info(f"orchestrator {runtime_config.get_name()} started at {start_ts}")
-    # create statistics
-    statistics = TransformStatistics()
-    # create data access
-    data_access = data_access_factory.create_data_access()
-    if data_access is None:
-        logger.error("No DataAccess instance provided - exiting")
-        return 1
-    try:
-        # Get files to process
-        files, profile, retries = data_access.get_files_to_process()
-        if len(files) == 0:
-            logger.error("No input files to process - exiting")
-            return 0
-        logger.info(f"Number of files is {len(files)}, source profile {profile}")
-        # Print interval
-        print_interval = int(len(files) / 100)
-        if print_interval == 0:
-            print_interval = 1
-        if retries > 0:
-            statistics.add_stats({"data access retries": retries})
-        # create executor
-        executor = PythonTransformFileProcessor(
-            data_access_factory=data_access_factory, statistics=statistics, runtime_configuration=runtime_config
-        )
-        # process data
-        logger.debug(f"{runtime_config.get_name()} Begin processing files")
-        t_start = time.time()
-        completed = 0
-        for path in files:
-            executor.process_file(path)
-            completed += 1
-            if completed % print_interval == 0:
-                logger.info(
-                    f"Completed {completed} files ({100 * completed / len(files)}%) "
-                    f"in {(time.time() - t_start)/60} min"
-                )
-        logger.debug(f"Done processing {completed} files, waiting for flush() completion.")
-        # invoke flush to ensure that all results are returned
-        start = time.time()
-        executor.flush()
-        logger.info(f"done flushing in {time.time() - start} sec")
-        status = "success"
-        return_code = 0
-    except Exception as e:
-        logger.error(f"Exception during execution {e}: {traceback.print_exc()}")
-        return_code = 1
-        status = "failure"
-    try:
-        # Compute execution statistics
-        logger.debug("Computing execution stats")
-        stats = statistics.get_execution_stats()
-        # build and save metadata
-        logger.debug("Building job metadata")
-        input_params = runtime_config.get_transform_metadata()
-        metadata = {
-            "pipeline": execution_config.pipeline_id,
-            "job details": execution_config.job_details
-            | {
-                "start_time": start_ts,
-                "end_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
-                "status": status,
-            },
-            "code": execution_config.code_location,
-            "job_input_params": input_params | data_access_factory.get_input_params(),
-            "job_output_stats": stats,
-        }
-        logger.debug(f"Saving job metadata: {metadata}.")
-        data_access.save_job_metadata(metadata)
-        logger.debug("Saved job metadata.")
-        return return_code
-    except Exception as e:
-        logger.error(f"Exception during execution {e}: {traceback.print_exc()}")
-        return 1