data-prep-toolkit 0.0.1.dev12__tar.gz → 0.1.0__tar.gz

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.0}/Makefile

@@ -53,10 +53,11 @@ venv::	pyproject.toml
 # pytest-forked was tried, but then we get SIGABRT in pytest when running the s3 tests, some of which are skipped.. 
 test::  
 	@# Help: Use the already-built virtual environment to run pytest on the test directory. 
-	source venv/bin/activate; export PYTHONPATH=../src; cd test; $(PYTEST)  data_processing_tests/data_access;	
-	source venv/bin/activate; export PYTHONPATH=../src;  cd test; $(PYTEST)  data_processing_tests/transform;	
-	source venv/bin/activate; export PYTHONPATH=../src;  cd test; $(PYTEST)  data_processing_tests/launch/pure_python;
+	source venv/bin/activate; export PYTHONPATH=../src; cd test; $(PYTEST)  data_processing_tests/data_access;
+	source venv/bin/activate; export PYTHONPATH=../src;  cd test; $(PYTEST)  data_processing_tests/transform;
+	source venv/bin/activate; export PYTHONPATH=../src;  cd test; $(PYTEST)  data_processing_tests/launch/pure_python/launcher_test.py;
+	source venv/bin/activate; export PYTHONPATH=../src;  cd test; $(PYTEST)  data_processing_tests/launch/pure_python/test_noop_launch.py;
 	source venv/bin/activate; export PYTHONPATH=../src; cd test; $(PYTEST)  data_processing_tests/launch/ray/ray_util_test.py;
-	source venv/bin/activate; export PYTHONPATH=../src; cd test; $(PYTEST)  data_processing_tests/launch/ray/launcher_test.py;	
-	source venv/bin/activate; export PYTHONPATH=../src; cd test; $(PYTEST)  data_processing_tests/launch/ray/test_noop_launch.py;	
+	source venv/bin/activate; export PYTHONPATH=../src; cd test; $(PYTEST)  data_processing_tests/launch/ray/launcher_test.py;
+	source venv/bin/activate; export PYTHONPATH=../src; cd test; $(PYTEST)  data_processing_tests/launch/ray/test_noop_launch.py;
 

{data_prep_toolkit-0.0.1.dev12/src/data_prep_toolkit.egg-info → data_prep_toolkit-0.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data_prep_toolkit
-Version: 0.0.1.dev12
+Version: 0.1.0
 Summary: Data Preparation Toolkit Library
 Author-email: David Wood <dawood@us.ibm.com>, Boris Lublinsky <blublinsky@ibm.com>
 License: Apache-2.0

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.0}/doc/advanced-transform-tutorial.md

@@ -13,6 +13,7 @@ removes duplicate documents across all files. In this tutorial, we will show the
   the operation of our _noop_ transform.
 
 The complete task involves the following:
+
 * EdedupTransform - class that implements the specific transformation
 * EdedupRuntime - class that implements custom TransformRuntime to create supporting Ray objects and enhance job output
   statistics
@@ -39,6 +40,7 @@ First, let's define the transform class.  To do this we extend
 the base abstract/interface class
 [AbstractTableTransform](../src/data_processing/transform/table_transform.py),
 which requires definition of the following:
+
 * an initializer (i.e. `init()`) that accepts a dictionary of configuration
   data.  For this example, the configuration data will only be defined by
   command line arguments (defined below).
@@ -56,15 +58,17 @@ from typing import Any
 
 import pyarrow as pa
 import ray
-from data_processing.data_access import DataAccessFactory
+from data_processing.data_access import DataAccessFactoryBase
 from data_processing.runtime.ray import (
-  RayLauncherConfiguration,
   DefaultTableTransformRuntimeRay,
-  RayUtils,
   RayTransformLauncher,
+  RayUtils,
+)
+from data_processing.runtime.ray.runtime_configuration import (
+  RayTransformRuntimeConfiguration,
 )
-from data_processing.transform import AbstractTableTransform
-from data_processing.utils import GB, TransformUtils
+from data_processing.transform import AbstractTableTransform, TransformConfiguration
+from data_processing.utils import GB, CLIArgumentProvider, TransformUtils, get_logger
 from ray.actor import ActorHandle
 
 
@@ -138,6 +142,7 @@ First, let's define the transform runtime class.  To do this we extend
 the base abstract/interface class
 [DefaultTableTransformRuntime](../src/data_processing/runtime/ray/transform_runtime.py),
 which requires definition of the following:
+
 * an initializer (i.e. `init()`) that accepts a dictionary of configuration
   data.  For this example, the configuration data will only be defined by
   command line arguments (defined below).
@@ -202,8 +207,10 @@ collected by hash actors and custom computations based on statistics data.
 
 ## EdedupTableTransformConfiguration
 
-The final class we need to implement is `EdedupTableTransformConfiguration` class and its initializer that
-define the following:
+The final class we need to implement is `EdedupRayTransformConfiguration` class that provides configuration for 
+running our transform. Although we provide only Ray-based implementation, Ray-based configuration relies on Python-based
+configuration that we need to define first. So we first need to define `EdedupTableTransformConfiguration` class, 
+defining the following:
 
 * The short name for the transform
 * The class implementing the transform - in our case EdedupTransform
@@ -216,10 +223,12 @@ First we define the class and its initializer,
 short_name = "ededup"
 cli_prefix = f"{short_name}_"
 
-class EdedupTableTransformConfiguration(DefaultTableTransformConfiguration):
-    def __init__(self):
-        super().__init__(name=short_name, runtime_class=EdedupRuntime, transform_class=EdedupTransform)
-        self.params = {}
+class EdedupTableTransformConfiguration(TransformConfiguration):
+  def __init__(self):
+    super().__init__(
+      name=short_name,
+      transform_class=EdedupTransform,
+    )
 ```
 
 The initializer extends the DefaultTableTransformConfiguration which provides simple
@@ -253,6 +262,13 @@ and which allows us to capture the `EdedupTransform`-specific arguments and opti
       logger.info(f"exact dedup params are {self.params}")
       return True
 ```
+Now we can implement `EdedupRayTransformConfiguration`with the following code
+```python
+class EdedupRayTransformConfiguration(RayTransformConfiguration):
+    def __init__(self):
+        super().__init__(transform_config=EdedupTableTransformConfiguration(), runtime_class=EdedupRuntime)
+
+```
 
 ## main()
 
@@ -261,11 +277,12 @@ framework's `TransformLauncher` class.
 
 ```python
 if __name__ == "__main__":
-    launcher = TransformLauncher(transform_runtime_config=EdedupTransformConfiguration())
-    launcher.launch()
+  launcher = RayTransformLauncher(EdedupRayTransformConfiguration())
+  launcher.launch()
 ```
+
 The launcher requires only an instance of DefaultTableTransformConfiguration
-(our `EdedupTransformConfiguration` class).
+(our `EdedupRayTransformConfiguration` class).
 A single method `launch()` is then invoked to run the transform in a Ray cluster.
 
 ## Running

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.0}/doc/architecture.md

@@ -16,7 +16,7 @@ The architecture includes the following core components:
 * [RayLauncher](../src/data_processing/runtime/ray/transform_launcher.py) accepts and validates 
  CLI parameters to establish the Ray Orchestrator with the proper configuration. 
 It uses the following components, all of which can/do define CLI configuration parameters.:
-    * [Transform Orchestrator Configuration](../src/data_processing/runtime/ray/transform_orchestrator_configuration.py) is responsible 
+    * [Transform Orchestrator Configuration](../src/data_processing/runtime/ray/execution_configuration.py) is responsible 
      for defining and validating infrastructure parameters 
      (e.g., number of workers, memory and cpu, local or remote cluster, etc.). This class has very simple state
      (several dictionaries) and is fully pickleable. As a result framework uses its instance as a
@@ -32,20 +32,22 @@ It uses the following components, all of which can/do define CLI configuration p
     After all parameters are validated, the ray cluster is started and the DataAccessFactory, TransformOrchestratorConfiguraiton
     and TransformConfiguration are given to the Ray Orchestrator, via Ray remote() method invocation.
     The Launcher waits for the Ray Orchestrator to complete.
-* [Ray Orchestrator](../src/data_processing/runtime/ray/transform_orchestrator.py) is responsible for overall management of
+
+* documents with [Ray Orchestrator](../src/data_processing/runtime/ray/transform_orchestrator.py) is responsible for overall management of
   the data processing job. It creates the actors, determines the set of input data and distributes the 
   references to the data files to be processed by the workers. More specifically, it performs the following:
+
   1. Uses the DataAccess instance created by the DataAccessFactory to determine the set of the files 
   to be processed.  
   2. uses the TransformConfiguration to create the TransformRuntime instance 
   3. Uses the TransformRuntime to optionally apply additional configuration (ray object storage, etc) for the configuration
   and operation of the Transform.
-  3. uses the TransformOrchestratorConfiguration to determine the set of RayWorkers to create
+  4. uses the TransformOrchestratorConfiguration to determine the set of RayWorkers to create
   to execute transformers in parallel, providing the following to each worker:
       * Ray worker configuration
       * DataAccessFactory 
       * Transform class and its TransformConfiguration containing the CLI parameters and any TransformRuntime additions.
-  4. in a load-balanced, round-robin fashion, distributes the names of the input files to the workers for them to transform/process.
+  5. in a load-balanced, round-robin fashion, distributes the names of the input files to the workers for them to transform/process.
    
   Additionally, to provide monitoring of long-running transforms, the orchestrator is instrumented with 
   [custom metrics](https://docs.ray.io/en/latest/ray-observability/user-guides/add-app-metrics.html), that are exported to localhost:8080 (this is the endpoint that 
@@ -53,11 +55,13 @@ It uses the following components, all of which can/do define CLI configuration p
   Once all data is processed, the orchestrator will collect execution statistics (from the statistics actor) 
   and build and save it in the form of execution metadata (`metadata.json`). Finally, it will return the execution 
   result to the Launcher.
+
 * [Ray worker](../src/data_processing/runtime/ray/transform_table_processor.py) is responsible for 
 reading files (as [PyArrow Tables](https://levelup.gitconnected.com/deep-dive-into-pyarrow-understanding-its-features-and-benefits-2cce8b1466c8))
 assigned by the orchestrator, applying the transform to the input table and writing out the 
 resulting table(s).  Metadata produced by each table transformation is aggregated into
 Transform Statistics (below).
+
 * [Transform Statistics](../src/data_processing/runtime/ray/transform_statistics.py) is a general 
 purpose data collector actor aggregating the numeric metadata from different places of 
 the framework (especially metadata produced by the transform).

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.0}/doc/python-launcher-options.md

@@ -1,5 +1,6 @@
-# Ray Launcher Command Line Options
-A number of command line options are available when launching a transform.  
+# Pure Python Launcher Command Line Options
+
+A number of command line options are available when launching a transform as a Python class.  
 
 The following is a current --help output (a work in progress) for 
 the `NOOPTransform` (note the --noop_sleep_sec option):

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.0}/doc/ray-runtime.md

@@ -5,7 +5,7 @@ The Ray runtime includes the following set of components:
 class generally used to implement `main()` that makes use of a `TransformConfiguration` to 
 start the Ray runtime and execute the transform over the specified set of input files.
 The RayTransformLauncher is created using a `RayTransformConfiguration` instance.
-* [RayTransformConfiguration](../src/data_processing/runtime/ray/transform_configuration.py) - this 
+* [RayTransformConfiguration](../src/data_processing/runtime/ray/runtime_configuration.py) - this 
 class extends transform's base TransformConfiguration implementation to add an optional 
 `TranformRuntime` (see next) class to be used by the transform implementation.
 * [TransformRuntime](../src/data_processing/runtime/ray/transform_runtime.py) - 
@@ -40,7 +40,7 @@ launcher = RayTransformLauncher(YourTransformConfiguration())
 launcher.launch()
 ```
 Note that the launcher defines some additional CLI parameters that are used to control the operation of the 
-[orchestrator and workers](../src/data_processing/runtime/ray/transform_orchestrator_configuration.py) and 
+[orchestrator and workers](../src/data_processing/runtime/ray/execution_configuration.py) and 
 [data access](../src/data_processing/data_access/data_access_factory.py).  Things such as data access configuration,
 number of workers, worker resources, etc.
 Discussion of these options is beyond the scope of this document 
@@ -51,7 +51,7 @@ In general, a transform should be able to run in both the python and Ray runtime
 As such we first define the python-only transform configuration, which will then
 be used by the Ray-runtime-specific transform configuration. 
 The python transform configuration implements  
-[TransformConfiguration](../src/data_processing/transform/transform_configuration.py)
+[TransformConfiguration](../src/data_processing/runtime/runtime_configuration.py)
 and deifnes with transform-specific name, and implementation 
 and class. In addition, it is responsible for providing transform-specific
 methods to define and capture optional command line arguments.

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.0}/doc/simplest-transform-tutorial.md

@@ -15,33 +15,38 @@ one table to another.  That said, we will show the following:
   the operation of our _noop_ transform.
 
 We will **not** be showing the following:
-* The creation of a custom TransformRuntime that would enable more global
-  state and/or coordination among the transforms running in other ray actors.
+* The creation of a custom `TransformRuntime` that would enable more global
+  state and/or coordination among the transforms running in other Ray actors.
   This will be covered in an advanced tutorial.
 
 The complete task involves the following:
-* NOOPTransform - class that implements the specific transformation
-* NOOPTableTransformConfiguration - class that provides configuration for the
-  NOOPTransform, specifically the command line arguments used to configure it.
-* main() - simple creation and use of the TransformLauncher.
+
+* `NOOPTransform` - class that implements the specific transformation
+* `NOOPTableTransformConfiguration` - class that provides configuration for the
+  `NOOPTransform`, specifically the command line arguments used to configure it.
+* `main()` - simple creation and use of the `TransformLauncher`.
 
 (Currently, the complete code for the noop transform used for this
 tutorial can be found in the
 [noop transform](../../transforms/universal/noop) directory.
 
-Finally, we show to use the command line to run the transform in a local ray cluster
+Finally, we show how to use the command line to run the transform in a local ray cluster.
+
+> **Note:** You will need to run the setup commands in the [`../README`](..) before running the following examples.
+
 
-## NOOPTransform
+## `NOOPTransform`
 
 First, let's define the transform class.  To do this we extend
 the base abstract/interface class
-[AbstractTableTransform](../src/data_processing_ibm/transform/table_transform.py),
+[`AbstractTableTransform`](../src/data_processing_ibm/transform/table_transform.py),
 which requires definition of the following:
+
 * an initializer (i.e. `init()`) that accepts a dictionary of configuration
   data.  For this example, the configuration data will only be defined by
   command line arguments (defined below).
-* the `transform()` method itself that takes an input table produces an output
-  table and any associated metadata for that table transformation.
+* the `transform()` method itself that takes an input table and produces an output
+  table with any associated metadata for that table transformation.
 
 Other methods such as `flush()` need not be overridden/redefined for this simple example.
 
@@ -54,18 +59,18 @@ from argparse import ArgumentParser, Namespace
 from typing import Any
 
 import pyarrow as pa
-from data_processing.runtime.ray import (
-    RayLauncherConfiguration,
-    DefaultTableTransformRuntimeRay,
-    RayTransformLauncher,
+from data_processing.runtime.ray import RayTransformLauncher
+from data_processing.runtime.ray.runtime_configuration import (
+  RayTransformRuntimeConfiguration,
 )
-from data_processing.transform import AbstractTableTransform
+from data_processing.transform import AbstractTableTransform, TransformConfiguration
+from data_processing.utils import CLIArgumentProvider, get_logger
 
 
 class NOOPTransform(AbstractTableTransform):
 
-    def __init__(self, config: dict[str, Any]):
-        self.sleep = config.get("sleep", 1)
+  def __init__(self, config: dict[str, Any]):
+    self.sleep = config.get("sleep", 1)
 ```
 The `NOOPTransform` class extends the `AbstractTableTransform`, which defines the required methods.
 
@@ -75,7 +80,7 @@ with an amount of seconds to sleep/delay during the call to `transform()`.
 Configuration is provided by the framework in a dictionary provided to the initializer.
 Below we will cover how this `sleep` argument is made available to the initializer.
 
-Note that in more complex transforms that might, for example, load a hugging face or other model,
+Note that in more complex transforms that might, for example, load a Hugging Face or other model,
 or perform other deep initializations, these can be done in the initializer.
 
 Next we define the `transform()` method itself, which includes the addition of some
@@ -90,19 +95,18 @@ almost trivial metadata.
         return [table], metadata
 ```
 The single input to this method is the in-memory pyarrow table to be transformed.
-The return of this function is a list of tables and optional metadata.  In this
-case of simple 1:1 table conversion the list will contain a single table, the input.
+The return value of this method is a list of tables and optional metadata.  In this
+case, we are doing a simple 1:1 table conversion, so the list will contain a single table, the input table.
 The metadata is a free-form dictionary of keys with numeric values that will be aggregated
 by the framework and reported as aggregated job statistics metadata.
 If there is no metadata then simply return an empty dictionary.
 
-## NOOPTransformConfiguration
+## `NOOPTransformConfiguration`
 
-Next we define the `NOOPTransformConfiguration` and 
-classes and there initializer that define the following:
+Next we define the `NOOPTransformConfiguration` class and its initializer that defines the following:
 
 * The short name for the transform
-* The class implementing the transform - in our case NOOPTransform
+* The class implementing the transform - in our case `NOOPTransform`
 * Command line argument support.
  
 We also define the `NOOPRayTransformationConfiguration` so we can run the transform
@@ -125,18 +129,20 @@ class NOOPTransformConfiguration(TransformConfiguration):
             remove_from_metadata=[pwd_key],
         )
 ```
-The initializer extends the TransformConfiguration which provides simple
-capture of our configuration data and enables picklability through the network.
+
+The initializer extends the `TransformConfiguration` that provides simple
+capture of our configuration data and enables the ability to pickle through the network.
 It also adds a `params` field that will be used below to hold the transform's
 configuration data (used in `NOOPTransform.init()` above).
 
 Next, we provide two methods that define and capture the command line configuration that
-is specific to the `NOOPTransform`, in this case the number of seconds to sleep during transformation
-and an example command line, `pwd`, option holding sensitive data that we don't want reported
-in the job metadata produced by the ray orchestrator.
-First we define the method establishes the command line arguments.
-This method is given a global argument parser to which the `NOOPTransform` arguments are added.
-It is good practice to include a common prefix to all transform-specific options (i.e. pii, lang, etc).
+is specific to the `NOOPTransform`, in this case the parameters are the number of seconds to sleep during transformation
+and an example command line parameter, `pwd` ("password"), option holding sensitive data that we don't want reported
+in the job metadata produced by the Ray orchestrator.
+
+The first method establishes the command line arguments.
+It is given a global argument parser to which the `NOOPTransform` arguments are added.
+It is a good practice to include a common prefix to all transform-specific options (i.e. pii, lang, etc).
 In our case we will use `noop_`.
 
 ```python
@@ -159,6 +165,7 @@ In our case we will use `noop_`.
 ```
 Next we implement a method that is called after the CLI args are parsed (usually by one
 of the runtimes) and which allows us to capture the `NOOPTransform`-specific arguments. 
+ 
 
 ```python
 
@@ -176,13 +183,16 @@ To run the transform on a set of input data, we use one of the runtimes, each de
 ### Python Runtime
 To run in the python runtime, we need to create the instance of `PythonTransformLauncher`
 using the `NOOPTransformConfiguration`, and launch it as follows:
+
 ```python
 if __name__ == "__main__":
     launcher = PythonTransformLauncher(transform_config=NOOPTransformConfiguration())
     launcher.launch()
 ```
 
-To run this on some test data, we'll use data in the repo for the noop transform
+## Running
+
+Assuming the above `main` code is placed in `noop_main.py` we can run the transform on some test data. We'll use data in the repo for the noop transform
 and create a temporary directory to hold the output:
 ```shell
 export DPK_REPOROOT=...
@@ -191,9 +201,9 @@ export NOOP_INPUT=$DPK_REPOROOT/transforms/universal/noop/test-data/input
 To run
 ```shell
 python noop_main.py --noop_sleep_msec 2 \
---data_local_config "{'input_folder': '"$NOOP_INPUT"', 'output_folder': '/tmp/noop-output'}"
+  --data_local_config "{'input_folder': '"$NOOP_INPUT"', 'output_folder': '/tmp/noop-output'}"
 ```
-See the [python launcher options](python-launcher-options) for a complete list of
+See the [python launcher options](python-launcher-options.md) for a complete list of
 transform-independent command line options.
 
 ### Ray Runtime
@@ -207,5 +217,5 @@ if __name__ == "__main__":
 ```
 We can run this with the same command as for the python runtime but to run in local Ray
 add the `--run_locally True` option.
-See the [ray launcher options](ray-launcher-options) for a complete list of
+See the [ray launcher options](ray-launcher-options.md) for a complete list of
 transform-independent command line options.

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.0}/doc/testing-e2e-transform.md

@@ -1,4 +1,5 @@
 # Testing End-to-End Transform operation
 WIP - Points to discuss
+
 1. Reading input files and writing output files. 
-2. Testing of the transform runtime and use of ray components in the transform
+2. Testing of the transform runtime and use of ray components in the transform

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.0}/doc/transform-external-resources.md

@@ -8,6 +8,7 @@ In addition to actually loading the resource(s), the transform needs to define t
 defines the location of the domain list. 
 
 In the next sections we cover the following:
+
    1. How to define the transform-specific resource location(s) as command line arguments
    2. How to load the transform-specific resources, either or both of:
       1. During transform initialization - this is useful for testing outside of ray, and optionally

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.0}/doc/transform-standalone-testing.md

@@ -15,6 +15,7 @@ transform implementation tests will easily leverage.
 
 The first (currently only test) is a the `test_transform()` method that takes the
 following inputs:
+
 * the transform implementation being tested, properly configured with the configuration
 dictionary for the associated test data.
 * a list of N (1 or more) input tables to be processed with the transform's `transform(Table)` method.

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.0}/doc/transform-tutorials.md

@@ -41,19 +41,22 @@ The return values are handled the same waa as the return values for `transform()
 not need this feature, a default implementation is provided to return an empty list and empty dictionary.
  
 #### TransformConfiguration class
-The [TransformConfiguration](../src/data_processing/transform/transform_configuration.py)
+The [TransformConfiguration](../src/data_processing/runtime/runtime_configuration.py)
 serves as an interface and must be implemented by the any `AbstractTableTransform`
 implementation to provide the following configuration:
+
 * the transform class to be used,
 * command line arguments used to initialize the Transform Runtime and generally, the Transform.
 * Transform Runtime class to use
 * transform short name 
+
 It is expected that transforms are initialized with a fixed name, the class of its corresponding
 `AbstractTableTransform` implementation and optionally the configuration keys that should not
 be exposed as metadata for a run.
 To support command line configuration, the `TransformConfiguration` extends the
 [CLIArgumentProvider](../src/data_processing/utils/cli_utils.py) class.
 The set of methods of interest are
+
 * ```__init__(self, name:str, transform_class:type[AbstractTableTransform], list[str]:remove_from_metadata )``` - sets the required fields
 * ```add_input_params(self, parser:ArgumentParser)``` - adds transform-specific command line options that will
 be made available in the dictionary provided to the transform's initializer.

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.0}/doc/transformer-utilities.md

@@ -2,6 +2,7 @@
 
 A class [TransformUtils](../src/data_processing/utils/transform_utils.py) provides several methods that simplify 
 transformer's implementation. Currently it includes the following methods:
+
 * `deep_get_size` is the method to get the complete size of the Python object based on
   https://www.askpython.com/python/built-in-methods/variables-memory-size-in-python
   It supports Python structures: list, tuple and set
@@ -17,8 +18,9 @@ be removed before it is added
   removes URL encoding
 
 It also contain two variables:
+
 * `RANDOM_SEED` number that is used for methods that require seed
 * `LOCAL_TO_DISK` rough local size to size on disk/S3
 
 This class should be extended with additional methods, generally useful across multiple transformers and documentation 
-should be added here 
+should be added here 

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit"
-version = "0.0.1-dev12"
+version = "0.1.0"
 requires-python = ">=3.10"
 description = "Data Preparation Toolkit Library"
 license = {text = "Apache-2.0"}

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.0/src/data_prep_toolkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data_prep_toolkit
-Version: 0.0.1.dev12
+Version: 0.1.0
 Summary: Data Preparation Toolkit Library
 Author-email: David Wood <dawood@us.ibm.com>, Boris Lublinsky <blublinsky@ibm.com>
 License: Apache-2.0

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.0}/src/data_prep_toolkit.egg-info/SOURCES.txt

@@ -35,19 +35,20 @@ src/data_processing/data_access/data_access_local.py
 src/data_processing/data_access/data_access_s3.py
 src/data_processing/runtime/__init__.py
 src/data_processing/runtime/execution_configuration.py
+src/data_processing/runtime/runtime_configuration.py
 src/data_processing/runtime/transform_launcher.py
+src/data_processing/runtime/transform_table_processor.py
 src/data_processing/runtime/pure_python/__init__.py
-src/data_processing/runtime/pure_python/python_launcher_configuration.py
+src/data_processing/runtime/pure_python/runtime_configuration.py
 src/data_processing/runtime/pure_python/transform_launcher.py
 src/data_processing/runtime/pure_python/transform_orchestrator.py
 src/data_processing/runtime/pure_python/transform_table_processor.py
 src/data_processing/runtime/ray/__init__.py
+src/data_processing/runtime/ray/execution_configuration.py
 src/data_processing/runtime/ray/ray_utils.py
-src/data_processing/runtime/ray/transform_configuration.py
-src/data_processing/runtime/ray/transform_launch_configuration.py
+src/data_processing/runtime/ray/runtime_configuration.py
 src/data_processing/runtime/ray/transform_launcher.py
 src/data_processing/runtime/ray/transform_orchestrator.py
-src/data_processing/runtime/ray/transform_orchestrator_configuration.py
 src/data_processing/runtime/ray/transform_runtime.py
 src/data_processing/runtime/ray/transform_statistics.py
 src/data_processing/runtime/ray/transform_table_processor.py
@@ -88,8 +89,10 @@ test/data_processing_tests/data_access/data_access_local_test.py
 test/data_processing_tests/data_access/data_access_s3_test.py
 test/data_processing_tests/data_access/sample_input_data_test.py
 test/data_processing_tests/launch/pure_python/launcher_test.py
+test/data_processing_tests/launch/pure_python/multi_launcher_test.py
 test/data_processing_tests/launch/pure_python/test_noop_launch.py
 test/data_processing_tests/launch/ray/launcher_test.py
+test/data_processing_tests/launch/ray/multi_launcher_test.py
 test/data_processing_tests/launch/ray/ray_util_test.py
 test/data_processing_tests/launch/ray/test_noop_launch.py
 test/data_processing_tests/transform/test_noop.py

data_prep_toolkit-0.1.0/src/data_processing/runtime/__init__.py

@@ -0,0 +1,4 @@
+from data_processing.runtime.execution_configuration import TransformExecutionConfiguration
+from data_processing.runtime.runtime_configuration import TransformRuntimeConfiguration
+from data_processing.runtime.transform_launcher import AbstractTransformLauncher, multi_luncher
+from data_processing.runtime.transform_table_processor import AbstractTransformTableProcessor

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.0}/src/data_processing/runtime/pure_python/__init__.py

@@ -1,4 +1,4 @@
-from data_processing.runtime.pure_python.python_launcher_configuration import PythonLauncherConfiguration
+from data_processing.runtime.pure_python.runtime_configuration import PythonTransformRuntimeConfiguration
 from data_processing.runtime.pure_python.transform_table_processor import TransformTableProcessor
 from data_processing.runtime.pure_python.transform_orchestrator import orchestrate
 from data_processing.runtime.pure_python.transform_launcher import PythonTransformLauncher

data_prep_toolkit-0.1.0/src/data_processing/runtime/pure_python/runtime_configuration.py

@@ -0,0 +1,24 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from data_processing.runtime import TransformRuntimeConfiguration
+from data_processing.transform import TransformConfiguration
+
+
+class PythonTransformRuntimeConfiguration(TransformRuntimeConfiguration):
+    def __init__(self, transform_config: TransformConfiguration):
+        """
+        Initialization
+        :param transform_config - base configuration class
+        """
+        self.transform_config = transform_config
+        super().__init__(transform_config=transform_config)

{data_prep_toolkit-0.0.1.dev12 → data_prep_toolkit-0.1.0}/src/data_processing/runtime/pure_python/transform_launcher.py

@@ -15,9 +15,11 @@ import time
 
 from data_processing.data_access import DataAccessFactory, DataAccessFactoryBase
 from data_processing.runtime import TransformExecutionConfiguration
-from data_processing.runtime.pure_python import PythonLauncherConfiguration, orchestrate
+from data_processing.runtime.pure_python import (
+    PythonTransformRuntimeConfiguration,
+    orchestrate,
+)
 from data_processing.runtime.transform_launcher import AbstractTransformLauncher
-from data_processing.transform import TransformConfiguration
 from data_processing.utils import get_logger
 
 
@@ -31,18 +33,16 @@ class PythonTransformLauncher(AbstractTransformLauncher):
 
     def __init__(
         self,
-        # transform_runtime_config: PythonLauncherConfiguration,
-        transform_config: TransformConfiguration,
+        runtime_config: PythonTransformRuntimeConfiguration,
         data_access_factory: DataAccessFactoryBase = DataAccessFactory(),
     ):
         """
         Creates driver
-        :param transform_runtime_config: transform runtime factory
+        :param runtime_config: transform runtime factory
         :param data_access_factory: the factory to create DataAccess instances.
         """
-        super().__init__(transform_config, data_access_factory)
-        self.transform_runtime_config = PythonLauncherConfiguration(transform_config)
-        self.execution_config = TransformExecutionConfiguration(name=self.transform_runtime_config.get_name())
+        super().__init__(runtime_config, data_access_factory)
+        self.execution_config = TransformExecutionConfiguration(name=runtime_config.get_name())
 
     def __get_parameters(self) -> bool:
         """
@@ -57,12 +57,12 @@ class PythonTransformLauncher(AbstractTransformLauncher):
             formatter_class=argparse.RawTextHelpFormatter,
         )
         # add additional arguments
-        self.transform_runtime_config.add_input_params(parser=parser)
+        self.runtime_config.add_input_params(parser=parser)
         self.data_access_factory.add_input_params(parser=parser)
         self.execution_config.add_input_params(parser=parser)
         args = parser.parse_args()
         return (
-            self.transform_runtime_config.apply_input_params(args=args)
+            self.runtime_config.apply_input_params(args=args)
             and self.execution_config.apply_input_params(args=args)
             and self.data_access_factory.apply_input_params(args=args)
         )
@@ -78,7 +78,7 @@ class PythonTransformLauncher(AbstractTransformLauncher):
             logger.debug("Starting orchestrator")
             res = orchestrate(
                 data_access_factory=self.data_access_factory,
-                transform_config=self.transform_runtime_config,
+                runtime_config=self.runtime_config,
                 execution_config=self.execution_config,
             )
             logger.debug("Completed orchestrator")