data-prep-toolkit 0.2.2.dev1__py3-none-any.whl → 0.2.2.dev2__py3-none-any.whl

data_processing/test_support/transform/noop_folder_transform.py

@@ -0,0 +1,105 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import time
+from typing import Any
+
+from data_processing.data_access import DataAccess
+from data_processing.runtime.pure_python import (
+    PythonTransformLauncher,
+    PythonTransformRuntimeConfiguration,
+    DefaultPythonTransformRuntime)
+from data_processing.transform import AbstractFolderTransform
+from data_processing.utils import get_logger
+from data_processing.test_support.transform import NOOPTransformConfiguration
+
+
+logger = get_logger(__name__)
+
+
+class NOOPFolderTransform(AbstractFolderTransform):
+    """
+    Implements a simple copy of a pyarrow Table.
+    """
+
+    def __init__(self, config: dict[str, Any]):
+        """
+        Initialize based on the dictionary of configuration information.
+        This is generally called with configuration parsed from the CLI arguments defined
+        by the companion runtime, NOOPTransformRuntime.  If running inside the RayMutatingDriver,
+        these will be provided by that class with help from the RayMutatingDriver.
+        """
+        # Make sure that the param name corresponds to the name used in apply_input_params method
+        # of NOOPTransformConfiguration class
+        super().__init__(config)
+        self.sleep = config.get("sleep_sec", 1)
+        self.data_access = config.get("data_access")
+
+    def transform(self, folder_name: str) -> tuple[list[tuple[bytes, str]], dict[str, Any]]:
+        """
+        Converts input folder into o or more output files.
+        If there is an error, an exception must be raised - exit()ing is not generally allowed.
+        :param folder_name: the name of the folder containing arbitrary amount of files.
+        :return: a tuple of a list of 0 or more tuples and a dictionary of statistics that will be propagated
+                to metadata.  Each element of the return list, is a tuple of the transformed bytes and a string
+                holding the file name to use.
+        """
+        logger.debug(f"Transforming one folder {folder_name}")
+        metadata = {}
+        # get folder files
+        files, retries = self.data_access.get_folder_files(path=folder_name)
+        if retries > 0:
+            metadata |= {"data access retries": retries}
+        result = [()] * len(files)
+        index = 0
+        for name, file in files.items():
+            result[index] = (file, self.data_access.get_output_location(name))
+            if self.sleep is not None:
+                logger.info(f"Sleep for {self.sleep} seconds")
+                time.sleep(self.sleep)
+                logger.info("Sleep completed - continue")
+            index += 1
+        # Add some sample metadata.
+        metadata |= {"nfiles": len(files)}
+        return result, metadata
+
+
+class NOOPFolderPythonRuntime(DefaultPythonTransformRuntime):
+    def get_folders(self, data_access: DataAccess) -> list[str]:
+        """
+        Get folders to process
+        :param data_access: data access
+        :return: list of folders to process
+        """
+        return [data_access.get_input_folder()]
+
+
+class NOOPFolderPythonTransformConfiguration(PythonTransformRuntimeConfiguration):
+    """
+    Implements the PythonTransformConfiguration for NOOP as required by the PythonTransformLauncher.
+    NOOP does not use a RayRuntime class so the superclass only needs the base
+    python-only configuration.
+    """
+
+    def __init__(self):
+        """
+        Initialization
+        """
+        super().__init__(transform_config=NOOPTransformConfiguration(clazz=NOOPFolderTransform),
+                         runtime_class=NOOPFolderPythonRuntime)
+
+
+if __name__ == "__main__":
+    # launcher = NOOPRayLauncher()
+    launcher = PythonTransformLauncher(NOOPFolderPythonTransformConfiguration())
+    logger.info("Launching noop transform")
+    launcher.launch()

data_processing/test_support/transform/noop_transform.py

@@ -19,7 +19,7 @@ from data_processing.runtime.pure_python import PythonTransformLauncher
 from data_processing.runtime.pure_python.runtime_configuration import (
     PythonTransformRuntimeConfiguration,
 )
-from data_processing.transform import AbstractTableTransform, TransformConfiguration
+from data_processing.transform import AbstractTableTransform, TransformConfiguration, AbstractTransform
 from data_processing.utils import CLIArgumentProvider, get_logger
 
 
@@ -75,10 +75,10 @@ class NOOPTransformConfiguration(TransformConfiguration):
     configuration with CLI args.
     """
 
-    def __init__(self):
+    def __init__(self, clazz: type[AbstractTransform] = NOOPTransform):
         super().__init__(
             name=short_name,
-            transform_class=NOOPTransform,
+            transform_class=clazz,
             remove_from_metadata=[pwd_key],
         )
 

data_processing/transform/__init__.py

@@ -1,3 +1,5 @@
+from data_processing.transform.abstract_transform import AbstractTransform
+from data_processing.transform.folder_transform import AbstractFolderTransform
 from data_processing.transform.binary_transform import AbstractBinaryTransform
 from data_processing.transform.table_transform import AbstractTableTransform
 from data_processing.transform.transform_statistics import TransformStatistics

data_processing/transform/abstract_transform.py

@@ -0,0 +1,16 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+class AbstractTransform:
+    """
+    Base class for all transform types
+    """

data_processing/transform/binary_transform.py

@@ -10,10 +10,11 @@
 # limitations under the License.
 ################################################################################
 
-from typing import Any, TypeVar
+from typing import Any
+from data_processing.transform import AbstractTransform
 
 
-class AbstractBinaryTransform:
+class AbstractBinaryTransform(AbstractTransform):
     """
     Converts input binary file to output file(s) (binary)
     Sub-classes must provide the transform() method to provide the conversion of one binary files to 0 or

data_processing/transform/folder_transform.py

@@ -0,0 +1,40 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from typing import Any
+from data_processing.transform import AbstractTransform
+
+
+class AbstractFolderTransform(AbstractTransform):
+    """
+    Converts input folder to output file(s) (binary)
+    Sub-classes must provide the transform() method to provide the conversion of a folder to 0 or
+    more new binary files and metadata.
+    """
+
+    def __init__(self, config: dict[str, Any]):
+        """
+        Initialize based on the dictionary of configuration information.
+        This simply stores the given instance in this instance for later use.
+        """
+        self.config = config
+
+    def transform(self, folder_name: str) -> tuple[list[tuple[bytes, str]], dict[str, Any]]:
+        """
+        Converts input folder into o or more output files.
+        If there is an error, an exception must be raised - exit()ing is not generally allowed.
+        :param folder_name: the name of the folder containing arbitrary amount of files.
+        :return: a tuple of a list of 0 or more tuples and a dictionary of statistics that will be propagated
+                to metadata.  Each element of the return list, is a tuple of the transformed bytes and a string
+                holding the file name to use.
+        """
+        raise NotImplemented()

data_processing/transform/transform_configuration.py

@@ -13,7 +13,7 @@
 from argparse import ArgumentParser
 from typing import Any
 
-from data_processing.transform import AbstractBinaryTransform
+from data_processing.transform import AbstractTransform
 from data_processing.utils import CLIArgumentProvider
 
 
@@ -23,7 +23,7 @@ class TransformConfiguration(CLIArgumentProvider):
     """
 
     def __init__(
-        self, name: str, transform_class: type[AbstractBinaryTransform], remove_from_metadata: list[str] = []
+        self, name: str, transform_class: type[AbstractTransform], remove_from_metadata: list[str] = []
     ):
         """
         Initialization
@@ -36,7 +36,7 @@ class TransformConfiguration(CLIArgumentProvider):
         self.remove_from_metadata = remove_from_metadata
         self.params = {}
 
-    def get_transform_class(self) -> type[AbstractBinaryTransform]:
+    def get_transform_class(self) -> type[AbstractTransform]:
         """
         Get the class extending AbstractBinaryTransform which implements a specific transformation.
         The class will generally be instantiated with a dictionary of configuration produced by

data_processing/utils/multilock.py

@@ -0,0 +1,160 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import abc
+import datetime
+import fcntl
+import os
+import tempfile
+import threading
+import time
+
+
+_tempdir = tempfile.gettempdir()
+
+
+class MultiLock(abc.ABC):
+    """
+    Provides a process- and thread-locked lock.
+    To use
+        lock = MultiLock("mylock")
+        ...
+        lock.acquire(block=false, timeout=30)
+        # do something critical
+        ...
+        lock.release()
+    """
+
+    def __init__(self, name):
+        """
+        Create the lock with the given name.
+
+        :param name: the global name associated with this lock. All processes using the same
+            name will be part of the same locking cohort.  It is up to the caller to define
+            and coordinate lock names.
+        """
+        if name is None or len(name) == 0:
+            raise ValueError("lock name must not be None or the empty string")
+        self.lock_filename = os.path.join(_tempdir, name + ".multilock")
+        # print(f"lock file name is {self.lock_filename}")
+        self.fd = None
+        self.thread_lock = threading.Lock()
+
+    def acquire(self, block=True, timeout=None):
+        """
+        With the block argument set to True (the default), the method call will block until the
+        lock is in an unlocked state, then set it to locked and return True.
+
+        With the block argument set to False, the method call does not block. If the lock
+        is currently in a locked state, return False; otherwise set the lock to a locked state and return True.
+
+        When invoked with a positive, floating-point value for timeout, wait for at most the number
+        of seconds specified by timeout as long as the lock can not be acquired. Invocations with a
+        negative value for timeout are equivalent to a timeout of zero. Invocations with a timeout
+        value of None (the default) set the timeout period to infinite. The timeout argument has no practical
+        implications if the block argument is set to False and is thus ignored.
+
+        Returns True if the lock has been acquired or False if the timeout period has elapsed.
+
+        """
+        if self.fd is not None:  # Already locked.
+            return True
+
+        start = time.time()
+        if block:
+            thread_timeout = timeout if timeout is not None and timeout >= 0 else -1
+            locked = self.thread_lock.acquire(blocking=True, timeout=thread_timeout)
+        else:
+            locked = self.thread_lock.acquire(blocking=False)
+        if not locked:
+            return False
+        end = time.time()
+        if not block and timeout > 0:
+            timeout -= end - start
+            if timeout <= 0:
+                self.thread_lock.release()
+                return False
+
+        # open a file and create a file descriptor
+        self.fd = os.open(self.lock_filename, os.O_RDWR | os.O_CREAT)
+
+        msg = f"MultiLock last held by process with pid={os.getpid()}\n"
+        os.write(self.fd, str.encode(msg))
+
+        # put a lock on an open file
+        locked = False
+        waited = 0
+        sleep_seconds = 1
+        if timeout is not None:
+            timeout = max(0, timeout)
+        while not locked and (timeout is None or waited <= timeout):
+            try:
+                fcntl.lockf(self.fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                locked = True
+            except Exception as exc:
+                # Only get here if lock could not be acquired
+                # print(f"sleeping {exc=}")
+                time.sleep(sleep_seconds)
+                if not block:
+                    break
+                waited += sleep_seconds
+        if not locked:
+            # If we didn't get the lock, then release the file.
+            os.close(self.fd)
+            self.fd = None
+
+        self.thread_lock.release()
+        return locked
+
+    def release(self):
+        """
+        Release an acquired lock.  Do nothing if the lock is not acquired.
+        :return:
+        """
+        if self.fd is not None:
+            self.thread_lock.acquire()
+            if self.fd is not None: # Retest now that we have the thread lock.
+                os.close(self.fd)
+                self.fd = None
+            self.thread_lock.release()
+
+    def is_locked(self):
+        return self.fd is not None
+
+
+def main(block, timeout, sleep):
+    lock = MultiLock("foo")
+    if block:
+        print(f"going to acquire the blocking lock with timeout={timeout}")
+    else:
+        print(f"going to acquire the non-blocking lock with timemout={timeout}")
+    locked = lock.acquire(block=block, timeout=timeout)
+    start = datetime.datetime.now()
+    start = start.strftime("%Y-%m-%d %H:%M:%S")
+    if not locked:
+        print(f"Could not get lock at {start}")
+        return
+    print(f"{start}: I got the lock")
+    time.sleep(sleep)
+    lock.release()
+    end = datetime.datetime.now()
+    end = end.strftime("%Y-%m-%d %H:%M:%S")
+    print(f"{end}: lock released")
+    print(f"lock held from {start} to {end}")
+
+
+if __name__ == "__main__":
+    sleep = 10
+    timeout = 10
+    main(True, timeout, sleep)
+    time.sleep(1)
+    main(False, timeout, sleep)

data_processing/utils/unrecoverable.py

@@ -1,3 +1,16 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+
 class UnrecoverableException(Exception):
     """
     Raised when a transform wants to cancel overall execution

data_processing_ray/runtime/ray/transform_file_processor.py

@@ -35,6 +35,7 @@ class RayTransformFileProcessor(AbstractTransformFileProcessor):
         super().__init__(
             data_access_factory=params.get("data_access_factory", None),
             transform_parameters=dict(params.get("transform_params", {})),
+            is_folder=params.get("is_folder", False)
         )
         # Create statistics
         self.stats = params.get("statistics", None)

data_processing_ray/runtime/ray/transform_orchestrator.py

@@ -16,6 +16,7 @@ from datetime import datetime
 
 import ray
 from data_processing.data_access import DataAccessFactoryBase
+from data_processing.transform import AbstractFolderTransform
 from data_processing_ray.runtime.ray import (
     RayTransformExecutionConfiguration,
     RayTransformFileProcessor,
@@ -56,13 +57,21 @@ def orchestrate(
     # create transformer runtime
     runtime = runtime_config.create_transform_runtime()
     resources = RayUtils.get_cluster_resources()
+    is_folder = issubclass(runtime_config.get_transform_class(), AbstractFolderTransform)
     try:
-        # Get files to process
-        files, profile, retries = data_access.get_files_to_process()
-        if len(files) == 0:
-            logger.error("No input files to process - exiting")
-            return 0
-        logger.info(f"Number of files is {len(files)}, source profile {profile}")
+        if is_folder:
+            # folder transform
+            files = runtime.get_folders(data_access=data_access)
+            logger.info(f"Number of folders is {len(files)}")  # Get files to process
+        else:
+            files, profile, retries = data_access.get_files_to_process()
+            if len(files) == 0:
+                logger.error("No input files to process - exiting")
+                return 0
+            # log retries
+            if retries > 0:
+                statistics.add_stats.remote({"data access retries": retries})
+            logger.info(f"Number of files is {len(files)}, source profile {profile}")
         # Print interval
         print_interval = int(len(files) / 100)
         if print_interval == 0:
@@ -73,9 +82,6 @@ def orchestrate(
         logger.info(
             f"Number of workers - {preprocessing_params.n_workers} " f"with {preprocessing_params.worker_options} each"
         )
-        # log retries
-        if retries > 0:
-            statistics.add_stats.remote({"data access retries": retries})
         # create executors
         processor_params = {
             "data_access_factory": data_access_factory,
@@ -84,6 +90,7 @@ def orchestrate(
                 data_access_factory=data_access_factory, statistics=statistics, files=files
             ),
             "statistics": statistics,
+            "is_folder": is_folder,
         }
         logger.debug("Creating actors")
         processors = RayUtils.create_actors(
@@ -135,7 +142,8 @@ def orchestrate(
         # Compute execution statistics
         logger.debug("Computing execution stats")
         stats = runtime.compute_execution_stats(ray.get(statistics.get_execution_stats.remote()))
-        stats["processing_time"] = round(stats["processing_time"], 3)
+        if "processing_time" in stats:
+            stats["processing_time"] = round(stats["processing_time"], 3)
 
         # build and save metadata
         logger.debug("Building job metadata")

data_processing_ray/runtime/ray/transform_runtime.py

@@ -12,7 +12,7 @@
 
 from typing import Any
 
-from data_processing.data_access import DataAccessFactoryBase
+from data_processing.data_access import DataAccessFactoryBase, DataAccess
 from ray.actor import ActorHandle
 
 
@@ -28,6 +28,14 @@ class DefaultRayTransformRuntime:
         """
         self.params = params
 
+    def get_folders(self, data_access: DataAccess) -> list[str]:
+        """
+        Get folders to process
+        :param data_access: data access
+        :return: list of folders to process
+        """
+        raise NotImplemented()
+
     def get_transform_config(
         self, data_access_factory: DataAccessFactoryBase, statistics: ActorHandle, files: list[str]
     ) -> dict[str, Any]:

data_processing_ray/test_support/transform/__init__.py

@@ -1 +1,2 @@
 from data_processing_ray.test_support.transform.noop_transform import NOOPRayTransformConfiguration
+from data_processing_ray.test_support.transform.noop_folder_transform import NOOPFolderRayTransformConfiguration

data_processing_ray/test_support/transform/noop_folder_transform.py

@@ -0,0 +1,56 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+
+from data_processing.test_support.transform import NOOPFolderTransform, NOOPTransformConfiguration
+from data_processing.utils import get_logger
+from data_processing_ray.runtime.ray import (
+    RayTransformLauncher,
+    RayTransformRuntimeConfiguration,
+    DefaultRayTransformRuntime
+)
+from data_processing.data_access import DataAccess
+
+
+logger = get_logger(__name__)
+
+
+class NOOPFolderRayRuntime(DefaultRayTransformRuntime):
+    def get_folders(self, data_access: DataAccess) -> list[str]:
+        """
+        Get folders to process
+        :param data_access: data access
+        :return: list of folders to process
+        """
+        return [data_access.get_input_folder()]
+
+
+class NOOPFolderRayTransformConfiguration(RayTransformRuntimeConfiguration):
+    """
+    Implements the RayTransformConfiguration for NOOP as required by the RayTransformLauncher.
+    NOOP does not use a RayRuntime class so the superclass only needs the base
+    python-only configuration.
+    """
+
+    def __init__(self):
+        """
+        Initialization
+        """
+        super().__init__(transform_config=NOOPTransformConfiguration(clazz=NOOPFolderTransform),
+                         runtime_class=NOOPFolderRayRuntime)
+
+
+if __name__ == "__main__":
+    # launcher = NOOPRayLauncher()
+    launcher = RayTransformLauncher(NOOPFolderRayTransformConfiguration())
+    logger.info("Launching noop transform")
+    launcher.launch()

data_processing_ray/test_support/transform/noop_transform.py

@@ -11,9 +11,7 @@
 ################################################################################
 
 
-from data_processing.test_support.transform.noop_transform import (
-    NOOPTransformConfiguration,
-)
+from data_processing.test_support.transform import NOOPTransformConfiguration
 from data_processing.utils import get_logger
 from data_processing_ray.runtime.ray import (
     RayTransformLauncher,

data_processing_spark/runtime/spark/runtime_configuration.py

@@ -10,6 +10,9 @@
 # limitations under the License.
 ################################################################################
 
+from typing import Any
+
+from data_processing.data_access import DataAccessFactoryBase
 from data_processing.runtime import TransformRuntimeConfiguration
 from data_processing.transform import TransformConfiguration
 from data_processing_spark.runtime.spark import DefaultSparkTransformRuntime
@@ -29,6 +32,16 @@ class SparkTransformRuntimeConfiguration(TransformRuntimeConfiguration):
         super().__init__(transform_config=transform_config)
         self.runtime_class = runtime_class
 
+    def get_bcast_params(self, data_access_factory: DataAccessFactoryBase) -> dict[str, Any]:
+        """Allows retrieving and broadcasting to all the workers very large
+        configuration parameters, like the list of document IDs to remove for
+        fuzzy dedup, or the list of blocked web domains for block listing.  This
+        function is called by the spark runtime after spark initialization, and
+        before spark_context.parallelize()
+        :param data_access_factory - creates data_access object to download the large config parameter
+        """
+        return {}
+
     def create_transform_runtime(self) -> DefaultSparkTransformRuntime:
         """
         Create transform runtime with the parameters captured during apply_input_params()

data_processing_spark/runtime/spark/transform_file_processor.py

@@ -29,12 +29,15 @@ class SparkTransformFileProcessor(AbstractTransformFileProcessor):
         data_access_factory: DataAccessFactoryBase,
         runtime_configuration: SparkTransformRuntimeConfiguration,
         statistics: TransformStatistics,
+        is_folder: bool,
     ):
         """
         Init method
         """
         super().__init__(
-            data_access_factory=data_access_factory, transform_parameters=runtime_configuration.get_transform_params()
+            data_access_factory=data_access_factory,
+            transform_parameters=runtime_configuration.get_transform_params(),
+            is_folder=is_folder,
         )
         # Add data access ant statistics to the processor parameters
         self.runtime_configuration = runtime_configuration