data-prep-toolkit 0.2.1.dev2__tar.gz → 0.2.1.dev3__tar.gz

{data_prep_toolkit-0.2.1.dev2/src/data_prep_toolkit.egg-info → data_prep_toolkit-0.2.1.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data_prep_toolkit
-Version: 0.2.1.dev2
+Version: 0.2.1.dev3
 Summary: Data Preparation Toolkit Library
 Author-email: David Wood <dawood@us.ibm.com>, Boris Lublinsky <blublinsky@ibm.com>
 License: Apache-2.0

{data_prep_toolkit-0.2.1.dev2 → data_prep_toolkit-0.2.1.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit"
-version = "0.2.1.dev2"
+version = "0.2.1.dev3"
 requires-python = ">=3.10"
 keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 description = "Data Preparation Toolkit Library"
@@ -41,7 +41,7 @@ dev = [
 ]
 
 [options]
-package_dir = ["src"]
+package_dir = ["src","test"]
 
 [options.packages.find]
 where = ["src/data_processing"]

{data_prep_toolkit-0.2.1.dev2 → data_prep_toolkit-0.2.1.dev3/src/data_prep_toolkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data_prep_toolkit
-Version: 0.2.1.dev2
+Version: 0.2.1.dev3
 Summary: Data Preparation Toolkit Library
 Author-email: David Wood <dawood@us.ibm.com>, Boris Lublinsky <blublinsky@ibm.com>
 License: Apache-2.0

{data_prep_toolkit-0.2.1.dev2 → data_prep_toolkit-0.2.1.dev3}/src/data_prep_toolkit.egg-info/SOURCES.txt

@@ -14,6 +14,7 @@ src/data_processing/data_access/data_access_factory.py
 src/data_processing/data_access/data_access_factory_base.py
 src/data_processing/data_access/data_access_local.py
 src/data_processing/data_access/data_access_s3.py
+src/data_processing/data_access/snapshot_utils.py
 src/data_processing/runtime/__init__.py
 src/data_processing/runtime/execution_configuration.py
 src/data_processing/runtime/runtime_configuration.py

{data_prep_toolkit-0.2.1.dev2 → data_prep_toolkit-0.2.1.dev3}/src/data_processing/data_access/__init__.py

@@ -4,3 +4,4 @@ from data_processing.data_access.data_access_local import DataAccessLocal
 from data_processing.data_access.data_access_s3 import DataAccessS3
 from data_processing.data_access.data_access_factory_base import DataAccessFactoryBase
 from data_processing.data_access.data_access_factory import DataAccessFactory
+from data_processing.data_access.snapshot_utils import SnapshotUtils

data_prep_toolkit-0.2.1.dev3/src/data_processing/data_access/snapshot_utils.py

@@ -0,0 +1,31 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from data_processing.data_access import DataAccess
+
+
+class SnapshotUtils:
+    """
+    Class implementing support methods for snapshotting
+    """
+
+    @staticmethod
+    def get_snapshot_folder(data_access: DataAccess) -> str:
+        """
+        Get snapshot folder from data access
+        :param data_access: data access class
+        :return: output folder
+        """
+        output_folder = data_access.get_output_folder()
+        if not output_folder.endswith("/"):
+            output_folder += "/"
+        return f"{output_folder}snapshot/"

{data_prep_toolkit-0.2.1.dev2 → data_prep_toolkit-0.2.1.dev3}/src/data_processing/runtime/pure_python/__init__.py

@@ -1,8 +1,10 @@
 from data_processing.runtime.pure_python.transform_runtime import DefaultPythonTransformRuntime
 from data_processing.runtime.pure_python.runtime_configuration import PythonTransformRuntimeConfiguration
 from data_processing.runtime.pure_python.execution_configuration import PythonTransformExecutionConfiguration
-from data_processing.runtime.pure_python.transform_file_processor import (PythonTransformFileProcessor,
-                                                                          PythonPoolTransformFileProcessor)
+from data_processing.runtime.pure_python.transform_file_processor import (
+    PythonTransformFileProcessor,
+    PythonPoolTransformFileProcessor,
+)
 from data_processing.runtime.pure_python.transform_orchestrator import orchestrate
 from data_processing.runtime.pure_python.transform_launcher import PythonTransformLauncher
 from data_processing.runtime.pure_python.transform_invoker import invoke_transform, execute_python_transform

{data_prep_toolkit-0.2.1.dev2 → data_prep_toolkit-0.2.1.dev3}/src/data_processing/runtime/pure_python/execution_configuration.py

@@ -67,6 +67,4 @@ class PythonTransformExecutionConfiguration(TransformExecutionConfiguration):
         get input parameters for job_input_params in metadata
         :return: dictionary of parameters
         """
-        return {
-            "num_processors": self.num_processors
-        }
+        return {"num_processors": self.num_processors}

{data_prep_toolkit-0.2.1.dev2 → data_prep_toolkit-0.2.1.dev3}/src/data_processing/runtime/pure_python/runtime_configuration.py

@@ -11,15 +11,16 @@
 ################################################################################
 
 from data_processing.runtime import TransformRuntimeConfiguration
-from data_processing.transform import TransformConfiguration
 from data_processing.runtime.pure_python import DefaultPythonTransformRuntime
+from data_processing.transform import TransformConfiguration
 
 
 class PythonTransformRuntimeConfiguration(TransformRuntimeConfiguration):
-    def __init__(self,
-                 transform_config: TransformConfiguration,
-                 runtime_class: type[DefaultPythonTransformRuntime] = DefaultPythonTransformRuntime,
-                 ):
+    def __init__(
+        self,
+        transform_config: TransformConfiguration,
+        runtime_class: type[DefaultPythonTransformRuntime] = DefaultPythonTransformRuntime,
+    ):
         """
         Initialization
         :param transform_config - base configuration class

{data_prep_toolkit-0.2.1.dev2 → data_prep_toolkit-0.2.1.dev3}/src/data_processing/runtime/pure_python/transform_file_processor.py

@@ -57,10 +57,10 @@ class PythonPoolTransformFileProcessor(AbstractTransformFileProcessor):
     """
 
     def __init__(
-            self,
-            data_access_factory: DataAccessFactoryBase,
-            transform_params: dict[str, Any],
-            transform_class: type[AbstractBinaryTransform],
+        self,
+        data_access_factory: DataAccessFactoryBase,
+        transform_params: dict[str, Any],
+        transform_class: type[AbstractBinaryTransform],
     ):
         """
         Init method
@@ -104,6 +104,4 @@ class PythonPoolTransformFileProcessor(AbstractTransformFileProcessor):
         """
         for key, val in stats.items():
             # for all key/values
-            if val > 0:
-                # for values greater then 0
-                self.stats[key] = self.stats.get(key, 0) + val
+            self.stats[key] = self.stats.get(key, 0) + val

{data_prep_toolkit-0.2.1.dev2 → data_prep_toolkit-0.2.1.dev3}/src/data_processing/runtime/pure_python/transform_orchestrator.py

@@ -11,19 +11,19 @@
 ################################################################################
 
 import time
-from typing import Any
-from multiprocessing import Pool
 import traceback
 from datetime import datetime
+from multiprocessing import Pool
+from typing import Any
 
 from data_processing.data_access import DataAccessFactoryBase
 from data_processing.runtime.pure_python import (
+    PythonPoolTransformFileProcessor,
     PythonTransformExecutionConfiguration,
-    PythonTransformRuntimeConfiguration,
     PythonTransformFileProcessor,
-    PythonPoolTransformFileProcessor,
+    PythonTransformRuntimeConfiguration,
 )
-from data_processing.transform import TransformStatistics, AbstractBinaryTransform
+from data_processing.transform import AbstractBinaryTransform, TransformStatistics
 from data_processing.utils import get_logger
 
 
@@ -69,21 +69,28 @@ def orchestrate(
         logger.debug(f"{runtime_config.get_name()} Begin processing files")
         if execution_config.num_processors > 0:
             # using multiprocessor pool for execution
-            statistics = _process_transforms_multiprocessor(files=files, size=execution_config.num_processors,
-                                                            data_access_factory=data_access_factory,
-                                                            print_interval=print_interval,
-                                                            transform_params=runtime.get_transform_config(
-                                                                data_access_factory=data_access_factory,
-                                                                statistics=statistics, files=files),
-                                                            transform_class=runtime_config.get_transform_class())
+            statistics = _process_transforms_multiprocessor(
+                files=files,
+                size=execution_config.num_processors,
+                data_access_factory=data_access_factory,
+                print_interval=print_interval,
+                transform_params=runtime.get_transform_config(
+                    data_access_factory=data_access_factory, statistics=statistics, files=files
+                ),
+                transform_class=runtime_config.get_transform_class(),
+            )
         else:
             # using sequential execution
-            _process_transforms(files=files, data_access_factory=data_access_factory,
-                                print_interval=print_interval, statistics=statistics,
-                                transform_params=runtime.get_transform_config(
-                                    data_access_factory=data_access_factory,
-                                    statistics=statistics, files=files),
-                                transform_class=runtime_config.get_transform_class())
+            _process_transforms(
+                files=files,
+                data_access_factory=data_access_factory,
+                print_interval=print_interval,
+                statistics=statistics,
+                transform_params=runtime.get_transform_config(
+                    data_access_factory=data_access_factory, statistics=statistics, files=files
+                ),
+                transform_class=runtime_config.get_transform_class(),
+            )
         status = "success"
         return_code = 0
     except Exception as e:
@@ -108,8 +115,9 @@ def orchestrate(
                 "status": status,
             },
             "code": execution_config.code_location,
-            "job_input_params":
-                input_params | data_access_factory.get_input_params() | execution_config.get_input_params(),
+            "job_input_params": input_params
+            | data_access_factory.get_input_params()
+            | execution_config.get_input_params(),
             "job_output_stats": stats,
         }
         logger.debug(f"Saving job metadata: {metadata}.")
@@ -121,9 +129,14 @@ def orchestrate(
         return 1
 
 
-def _process_transforms(files: list[str], print_interval: int, data_access_factory: DataAccessFactoryBase,
-                        statistics: TransformStatistics, transform_params: dict[str, Any],
-                        transform_class: type[AbstractBinaryTransform]) -> None:
+def _process_transforms(
+    files: list[str],
+    print_interval: int,
+    data_access_factory: DataAccessFactoryBase,
+    statistics: TransformStatistics,
+    transform_params: dict[str, Any],
+    transform_class: type[AbstractBinaryTransform],
+) -> None:
     """
     Process transforms sequentially
     :param files: list of files to process
@@ -137,8 +150,12 @@ def _process_transforms(files: list[str], print_interval: int, data_access_facto
     :return: None
     """
     # create executor
-    executor = PythonTransformFileProcessor(data_access_factory=data_access_factory, statistics=statistics,
-                                            transform_params=transform_params, transform_class=transform_class)
+    executor = PythonTransformFileProcessor(
+        data_access_factory=data_access_factory,
+        statistics=statistics,
+        transform_params=transform_params,
+        transform_class=transform_class,
+    )
     # process data
     t_start = time.time()
     completed = 0
@@ -157,9 +174,14 @@ def _process_transforms(files: list[str], print_interval: int, data_access_facto
     logger.info(f"done flushing in {round(time.time() - start, 3)} sec")
 
 
-def _process_transforms_multiprocessor(files: list[str], size: int, print_interval: int,
-                                       data_access_factory: DataAccessFactoryBase, transform_params: dict[str, Any],
-                                       transform_class: type[AbstractBinaryTransform]) -> TransformStatistics:
+def _process_transforms_multiprocessor(
+    files: list[str],
+    size: int,
+    print_interval: int,
+    data_access_factory: DataAccessFactoryBase,
+    transform_params: dict[str, Any],
+    transform_class: type[AbstractBinaryTransform],
+) -> TransformStatistics:
     """
     Process transforms using multiprocessing pool
     :param files: list of files to process
@@ -173,8 +195,9 @@ def _process_transforms_multiprocessor(files: list[str], size: int, print_interv
     # result statistics
     statistics = TransformStatistics()
     # create processor
-    processor = PythonPoolTransformFileProcessor(data_access_factory=data_access_factory,
-                                                 transform_params=transform_params, transform_class=transform_class)
+    processor = PythonPoolTransformFileProcessor(
+        data_access_factory=data_access_factory, transform_params=transform_params, transform_class=transform_class
+    )
     completed = 0
     t_start = time.time()
     # create multiprocessing pool

{data_prep_toolkit-0.2.1.dev2 → data_prep_toolkit-0.2.1.dev3}/src/data_processing/runtime/transform_file_processor.py

@@ -23,9 +23,9 @@ class AbstractTransformFileProcessor:
     """
 
     def __init__(
-            self,
-            data_access_factory: DataAccessFactoryBase,
-            transform_parameters: dict[str, Any],
+        self,
+        data_access_factory: DataAccessFactoryBase,
+        transform_parameters: dict[str, Any],
     ):
         """
         Init method
@@ -85,7 +85,7 @@ class AbstractTransformFileProcessor:
             raise UnrecoverableException
         # Process other exceptions
         except Exception as e:
-            self.logger.warning(f"Exception {e} processing file {f_name}: {traceback.format_exc()}")
+            self.logger.warning(f"Exception processing file {f_name}: {traceback.format_exc()}")
             self._publish_stats({"transform execution exception": 1})
 
     def flush(self) -> None:
@@ -133,6 +133,12 @@ class AbstractTransformFileProcessor:
                 self.logger.debug(
                     f"Transform did not produce a transformed file for " f"file {self.last_file_name}.parquet"
                 )
+                self._publish_stats(
+                    {
+                        "result_files": len(out_files),
+                        "processing_time": time.time() - t_start,
+                    }
+                )
             case 1:
                 # we have exactly 1 output file
                 file_ext = out_files[0]

{data_prep_toolkit-0.2.1.dev2 → data_prep_toolkit-0.2.1.dev3}/src/data_processing/test_support/abstract_test.py

@@ -75,8 +75,8 @@ class AbstractTest:
     def _install_test_fixtures(self, metafunc):
         raise NotImplemented("Sub-class must implemented this to install the fixtures for its tests.")
 
-    @staticmethod
-    def validate_expected_tables(table_list: list[pa.Table], expected_table_list: list[pa.Table]):
+    @classmethod
+    def validate_expected_tables(cls, table_list: list[pa.Table], expected_table_list: list[pa.Table]):
         """
         Verify with assertion messages that the two lists of Tables are equivalent.
         :param table_list:
@@ -100,10 +100,10 @@ class AbstractTest:
                 r1 = t1.take([j])
                 r2 = t2.take([j])
                 # assert r1 == r2, f"Row {j} of table {i} are not equal\n\tTransformed: {r1}\n\tExpected   : {r2}"
-                AbstractTest.validate_expected_row(i, j, r1, r2)
+                cls.validate_expected_row(i, j, r1, r2)
 
-    @staticmethod
-    def validate_expected_row(table_index: int, row_index: int, test_row: pa.Table, expected_row: pa.Table):
+    @classmethod
+    def validate_expected_row(cls, table_index: int, row_index: int, test_row: pa.Table, expected_row: pa.Table):
         """
         Compare the two rows for equality, allowing float values to be within a percentage
         of each other as defined by global _allowed_float_percent_diff.
@@ -139,8 +139,8 @@ class AbstractTest:
                         diff = abs(test_value - expected_value)
                         assert diff <= allowed_diff, msg
 
-    @staticmethod
-    def validate_expected_files(files_list: list[tuple[bytes, str]], expected_files_list: list[tuple[bytes, str]]):
+    @classmethod
+    def validate_expected_files(cls, files_list: list[tuple[bytes, str]], expected_files_list: list[tuple[bytes, str]]):
         """
         Verify with assertion messages that the two lists of Tables are equivalent.
         :param files_list:
@@ -171,15 +171,15 @@ class AbstractTest:
                 diff <= diff_allowed
             ), f"produced file length {lenf1} vs expected {lenf2}, exceeds allowance of {diff_allowed}"
 
-    @staticmethod
-    def validate_expected_metadata_lists(metadata: list[dict[str, float]], expected_metadata: list[dict[str, float]]):
+    @classmethod
+    def validate_expected_metadata_lists(cls, metadata: list[dict[str, float]], expected_metadata: list[dict[str, float]]):
         elen = len(expected_metadata)
         assert len(metadata) == elen, f"Number of metadata dictionaries not the expected of {elen}"
         for index in range(elen):
-            AbstractTest.validate_expected_metadata(metadata[index], expected_metadata[index])
+            cls.validate_expected_metadata(metadata[index], expected_metadata[index])
 
-    @staticmethod
-    def validate_expected_metadata(metadata: dict[str, float], expected_metadata: dict[str, float]):
+    @classmethod
+    def validate_expected_metadata(cls, metadata: dict[str, float], expected_metadata: dict[str, float]):
         """
         Verify with assertion messages that the two dictionaries are as expected.
         :param metadata:
@@ -194,8 +194,8 @@ class AbstractTest:
             f"Metadata not equal\n" "\tTransformed: {metadata}  Expected   : {expected_metadata}"
         )
 
-    @staticmethod
-    def validate_directory_contents(directory: str, expected_dir: str, drop_columns: list[str] = []):
+    @classmethod
+    def validate_directory_contents(cls, directory: str, expected_dir: str, drop_columns: list[str] = []):
         """
         Make sure the directory contents are the same.
         :param directory:
@@ -217,28 +217,28 @@ class AbstractTest:
             expected_diffs = 0
         failed = len(dir_cmp.diff_files) != expected_diffs
         if failed:
-            AbstractTest.__confirm_diffs(directory, expected_dir, dir_cmp.diff_files, "/tmp", drop_columns)
+            cls.__confirm_diffs(directory, expected_dir, dir_cmp.diff_files, "/tmp", drop_columns)
 
         # Traverse into the subdirs since dircmp doesn't seem to do that.
         subdirs = [f.name for f in os.scandir(expected_dir) if f.is_dir()]
         for subdir in subdirs:
             d1 = os.path.join(directory, subdir)
             d2 = os.path.join(expected_dir, subdir)
-            AbstractTest.validate_directory_contents(d1, d2, drop_columns)
+            cls.validate_directory_contents(d1, d2, drop_columns)
 
-    @staticmethod
-    def _validate_table_files(parquet1: str, parquet2: str, drop_columns: list[str] = []):
+    @classmethod
+    def _validate_table_files(cls, parquet1: str, parquet2: str, drop_columns: list[str] = []):
         da = DataAccessLocal()
         t1, _ = da.get_table(parquet1)
         t2, _ = da.get_table(parquet2)
         if len(drop_columns) > 0:
             t1 = t1.drop_columns(drop_columns)
             t2 = t2.drop_columns(drop_columns)
-        AbstractTest.validate_expected_tables([t1], [t2])
+        cls.validate_expected_tables([t1], [t2])
 
-    @staticmethod
+    @classmethod
     def __confirm_diffs(
-        src_dir: str, expected_dir: str, diff_files: list, dest_dir: str, drop_columns: list[str] = []
+        cls, src_dir: str, expected_dir: str, diff_files: list, dest_dir: str, drop_columns: list[str] = []
     ):
         """
         Copy all files from the source dir to the dest dir.
@@ -256,7 +256,7 @@ class AbstractTest:
                 # It seems file can be different on disk, but contain the same column/row values.
                 # so for these, do the inmemory comparison.
                 try:
-                    AbstractTest._validate_table_files(expected, src, drop_columns)
+                    cls._validate_table_files(expected, src, drop_columns)
                 except AssertionError as e:
                     logger.info(f"Copying file with difference: {src} to {dest}")
                     shutil.copyfile(src, dest)

{data_prep_toolkit-0.2.1.dev2 → data_prep_toolkit-0.2.1.dev3}/src/data_processing/test_support/launch/transform_test.py

@@ -65,7 +65,7 @@ class AbstractTransformLauncherTest(AbstractTest):
         Confirm that the two directories contains the same files.
         Stubbed out like this to allow spark tests to override this since spark tends to rename the files.
         """
-        AbstractTest.validate_directory_contents(dir, expected, ignore_columns)
+        self.validate_directory_contents(dir, expected, ignore_columns)
 
     def _install_test_fixtures(self, metafunc):
         # Apply the fixtures for the method with these input names (i.e. test_transform()).

{data_prep_toolkit-0.2.1.dev2 → data_prep_toolkit-0.2.1.dev3}/src/data_processing/transform/transform_statistics.py

@@ -32,8 +32,7 @@ class TransformStatistics:
         :return: None
         """
         for key, val in stats.items():
-            if val > 0:
-                self.stats[key] = self.stats.get(key, 0) + val
+            self.stats[key] = self.stats.get(key, 0) + val
 
     def get_execution_stats(self) -> dict[str, Any]:
         """

{data_prep_toolkit-0.2.1.dev2 → data_prep_toolkit-0.2.1.dev3}/src/data_processing/runtime/pure_python/transform_launcher.py

@@ -15,8 +15,8 @@ import time
 
 from data_processing.data_access import DataAccessFactory, DataAccessFactoryBase
 from data_processing.runtime.pure_python import (
-    PythonTransformRuntimeConfiguration,
     PythonTransformExecutionConfiguration,
+    PythonTransformRuntimeConfiguration,
     orchestrate,
 )
 from data_processing.runtime.transform_launcher import AbstractTransformLauncher