data-prep-toolkit 0.2.2.dev1__py3-none-any.whl → 0.2.2.dev2__py3-none-any.whl

{data_prep_toolkit-0.2.2.dev1.dist-info → data_prep_toolkit-0.2.2.dev2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data_prep_toolkit
-Version: 0.2.2.dev1
+Version: 0.2.2.dev2
 Summary: Data Preparation Toolkit Library for Ray and Python
 Author-email: Maroun Touma <touma@us.ibm.com>
 License: Apache-2.0
@@ -30,4 +30,36 @@ Requires-Dist: pillow>=10.3.0; extra == "ray"
 Provides-Extra: spark
 Requires-Dist: pyspark>=3.5.2; extra == "spark"
 Requires-Dist: psutil>=6.0.0; extra == "spark"
+Requires-Dist: PyYAML>=6.0.2; extra == "spark"
+
+# Data Processing Library
+This provides a python framework for developing _transforms_
+on data stored in files - currently parquet files are supported -
+and running them in a [ray](https://www.ray.io/) cluster.
+Data files may be stored in the local file system or  COS/S3.
+For more details see the [documentation](../doc/overview.md).
+
+### Virtual Environment
+The project uses `pyproject.toml` and a Makefile for operations.
+To do development you should establish the virtual environment
+```shell
+make venv
+```
+and then either activate
+```shell
+source venv/bin/activate
+```
+or set up your IDE to use the venv directory when developing in this project
+
+## Library Artifact Build and Publish
+To test, build and publish the library 
+```shell
+make test build publish
+```
+
+To up the version number, edit the Makefile to change VERSION and rerun
+the above.  This will require committing both the `Makefile` and the
+autotmatically updated `pyproject.toml` file.
+
+
 

{data_prep_toolkit-0.2.2.dev1.dist-info → data_prep_toolkit-0.2.2.dev2.dist-info}/RECORD

@@ -1,72 +1,78 @@
 data_processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 data_processing/data_access/__init__.py,sha256=996nZjaFiXZ-Zu648oC3eGlvZPTu-DTgsqvlKY4y4Ck,496
 data_processing/data_access/arrow_s3.py,sha256=0wxYFrw1MSsntO0IbGUGaBuiB366cIAmpsWvycenU_4,9724
-data_processing/data_access/data_access.py,sha256=_Xo5FjcJXQvkRlxJp0pfGoCgt50CNO6hmCUbAflLwE0,18632
+data_processing/data_access/data_access.py,sha256=hfERzfigBLgYoq5hsODcgsOMgh6lKISAGy_LP76Y2go,18813
 data_processing/data_access/data_access_factory.py,sha256=Hdm06HBs_AIjmU4Aqyd_8klZEty57MbjueGlF-8basw,12339
 data_processing/data_access/data_access_factory_base.py,sha256=dY_69ic4hKCQWSbRPNqxTRs8gcBhbW9rbvcAeeq0iFk,5685
-data_processing/data_access/data_access_local.py,sha256=swwLGNaLDFjXr0ViZdsagB3JM5ainOP0stf-hQOvSo8,9229
-data_processing/data_access/data_access_s3.py,sha256=QbDWalTiScP8nko7Txa8oVWK0neW6adcLGxjenz0LP0,8249
+data_processing/data_access/data_access_local.py,sha256=5XozY_r_44ExjbFEPAzaIJDNEx3AKSk8h7zmf30Q2N0,8799
+data_processing/data_access/data_access_s3.py,sha256=6cPkLY4-yOhtoCryDDXbEWq4SHJpHEh2_VTnBbsQgsk,7815
 data_processing/data_access/snapshot_utils.py,sha256=5iT0sP5ZkXQpZmHQh_5Do2W846Xc04-6GPmociqxmAY,1207
 data_processing/runtime/__init__.py,sha256=E9jeZ28v-TOGGB5O91GZ9GG9dP86bK02-o4qWPj8TNw,389
 data_processing/runtime/execution_configuration.py,sha256=9EKln8O0bjKRbb7GysegxNQ6t9stI8GqbFIxscPYeak,3218
 data_processing/runtime/runtime_configuration.py,sha256=uAKr-aawKcr_5Yq9fG8SktHI-Z1nCoVsu84UF2NiV9U,2769
-data_processing/runtime/transform_file_processor.py,sha256=-Xdtq0sC99sIB9e6hssSvd_OOFBo9RCu1s8-6q16fi0,9916
+data_processing/runtime/transform_file_processor.py,sha256=SQ_ICPPXCLcBr_X2lmOLiddyjCfz4CCaFtvatu8tcSc,10898
 data_processing/runtime/transform_launcher.py,sha256=34144dkMHSt58PQ1wbAKnELnhi9lubJcNCBopWoLRqo,4593
 data_processing/runtime/pure_python/__init__.py,sha256=s0smp908uzeT1cd_6EERaI72WJqiOJ50SDC5yOgDBvc,744
 data_processing/runtime/pure_python/execution_configuration.py,sha256=C0DCP3YR9sirFQ1Zqz4KcZ03WA5FLXl_kky9PBHREYQ,2617
 data_processing/runtime/pure_python/runtime_configuration.py,sha256=a4vSY98HfRm2p6pIBNpT-wDEtqRX2RUbRpFYtXyq5Kk,1710
-data_processing/runtime/pure_python/transform_file_processor.py,sha256=PYWNUSeb6i6q6Ov7nE0jXQfHIhp1u9adArEU3mQ7B24,4394
+data_processing/runtime/pure_python/transform_file_processor.py,sha256=jN-8OnPgctfY80W13WOnsdxCpDHi4HAgv0bM0Ysx-H4,4586
 data_processing/runtime/pure_python/transform_invoker.py,sha256=lAG7tfyJyNqtwRB15-db4HJOQsBhT6JahLmjUFQFCRk,5192
 data_processing/runtime/pure_python/transform_launcher.py,sha256=BDctJnYlR9OVzGCzMwg2cEuGdnV3E9fvhUgoyslvK8k,2447
-data_processing/runtime/pure_python/transform_orchestrator.py,sha256=OIQzOL0jT-3ahT7aDs6suySkoEhmvM_T4C_qMDt0JSQ,9468
-data_processing/runtime/pure_python/transform_runtime.py,sha256=pWvuGJGAB6M798LJU3FZBG6l35VQCuhsh-SyzSf9ok0,2558
+data_processing/runtime/pure_python/transform_orchestrator.py,sha256=YtELV-ENqqklSGr1N2yqNaSsQZlR0o-lk16PNxDAHYE,10067
+data_processing/runtime/pure_python/transform_runtime.py,sha256=3fz3_c_pYJkShXj_EuCGg9ieIb9ysfc_yhttsSjSDi4,2806
 data_processing/test_support/__init__.py,sha256=O4lySih15vkOYUSa3uhTaoYw0RrV4rM_sUd691JEuVU,83
 data_processing/test_support/abstract_test.py,sha256=gZ51wnWITEAyb8BzA2WFCM0quJBxQrlD7WBwUfIsWEA,12875
 data_processing/test_support/data_access/__init__.py,sha256=q4xqedYF-a1Kkk64i1ToiEW_SZQDMw66mV1s-Owsqq0,69
-data_processing/test_support/data_access/data_access_factory_test.py,sha256=mKPPsGeXABjTaw5v1s5CGwELDgEBAJoIybLIFIsCtSM,2669
+data_processing/test_support/data_access/data_access_factory_test.py,sha256=AE1EWibaNtu3svA4s94GFd0RzJtdTyCsBYKnPM0U2Ww,3328
 data_processing/test_support/launch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 data_processing/test_support/launch/transform_test.py,sha256=vcqLeZZPSiJ0HPkr0qaBSLGhzKMaXeErtQdKu6fsiTs,5435
-data_processing/test_support/transform/__init__.py,sha256=pwrft_Q3NQZyySA9rKE-3ORo_Vi-Rq3nKtdD7w-bzXY,213
+data_processing/test_support/transform/__init__.py,sha256=zn8RWuDBFS1JNL1QCVGrGCBEMn6bcEqTaOsWp9XnGTg,502
 data_processing/test_support/transform/binary_transform_test.py,sha256=ea3ipfxUuOYYxZaNSV-2ad2Q2Oqg32uenMAqxHSbDB4,4573
-data_processing/test_support/transform/noop_transform.py,sha256=lDSceUbU_M40Zn2aDrhr3ZsWWB762iREVXmabg4JBX0,5638
+data_processing/test_support/transform/noop_folder_transform.py,sha256=0V74ZVCb6DgOKlgtDCsYPDeYIg6xJm1vLnCiTlAjqeM,4410
+data_processing/test_support/transform/noop_transform.py,sha256=9OnvZ5vhfveTjWSILnT8FyhvEL57yTIc_3wFi1R8YFw,5697
 data_processing/test_support/transform/table_transform_test.py,sha256=nTlomV3-X9kkiWaR7AF_qFp5T2fWWSoDCEi8p5SpqtI,4619
-data_processing/transform/__init__.py,sha256=tLmeKWCRKDEjMRyjOLbjmJFQHGcL0i-4YeTJF41bWjg,342
-data_processing/transform/binary_transform.py,sha256=Nh8C4qkvuxdku16CgTIHiVO1hnp2I9TzaYw5YN7qa9E,2916
+data_processing/transform/__init__.py,sha256=RqHj9FCbkKsYqeZRdFLiFapnJkWaj0AhPAdB0-Fy_BE,496
+data_processing/transform/abstract_transform.py,sha256=nSPaJiu-M1Ptzmixw3KgSlKXJyeFp7J8PIGkt9ULiog,738
+data_processing/transform/binary_transform.py,sha256=oM_v5_HsgM914FamzWP_IgpXxp4cxSGzer9io-G2c8k,2982
+data_processing/transform/folder_transform.py,sha256=XtdSHlvs4qbVTfXAWIVbpPtSnQjMQOYvJq8WZHeSdtU,1910
 data_processing/transform/table_transform.py,sha256=BliQKz7ShFTpXRe9WBYAeRM7n-AhEWYj5kb05QAxS2c,6281
-data_processing/transform/transform_configuration.py,sha256=uBrhCafxvy8bc5E11QwdQT9USrUZtiYjSp_PQH5OmFE,4413
+data_processing/transform/transform_configuration.py,sha256=4hLPy78eMpFw0yKQy2ICZcFicUz--kvgUFAXorUkkwE,4395
 data_processing/transform/transform_statistics.py,sha256=eeInqfFFxhmQoaMJXXaOq_zY49mmQYAA_9xmdwi5vDw,1437
 data_processing/utils/__init__.py,sha256=lL25FenOmye6EBtcQLbczJJ3NtrGlmW9v0hDQfKrCno,584
 data_processing/utils/cli_utils.py,sha256=vjs6qOxxYetEYCYx_8jX_KBVwA7z_iBYFPhyPyucCGM,3135
 data_processing/utils/config.py,sha256=GZgmmvjiSEL86Vk_wx7lJ-Y27ItQXujthb5mcdV4FMA,1695
 data_processing/utils/log.py,sha256=t7S_ITjWJicEaCPcSk3cKKXyzEe1u21Ov5aQz3CzrZQ,2226
+data_processing/utils/multilock.py,sha256=ra5Qy61zqmdPNhAqYNBXLG6L6LjTxTyTpDLMOuc99x0,5734
 data_processing/utils/params_utils.py,sha256=oAKY3wC8b17rDUJGqX19-rAQHDc9SQn1ksTeo3RFgCY,6668
 data_processing/utils/pipinstaller.py,sha256=PxFNwEy8v4FqjwYgrPhH0UTrCgsJvM5WAE2fKylsk2Q,2511
 data_processing/utils/transform_configuration.json,sha256=6YBw0Hk2mokY6JBn1kR6L9AkV_yivbFrpSoHecAJp9o,4562
 data_processing/utils/transform_configurator.py,sha256=9OHSCQ8rFSoDdMW6ZCHYdNe6thRwV9zOaRPnLkWNMYE,3601
 data_processing/utils/transform_utils.py,sha256=KGNioN35B1i1h-MIsfm3QvXLlU1aGXimheva7NbUhMM,8496
-data_processing/utils/unrecoverable.py,sha256=svNdVzQaArnf8GdLvB2nP9miv7kYe3bDfFRW--SWvbU,171
+data_processing/utils/unrecoverable.py,sha256=cbF74AGK1IdRor_L1w_hPwglV_b2blP6Ad4ET79xrl0,831
 data_processing_ray/runtime/ray/__init__.py,sha256=vjQOvb_OJNq3c1F_tG3WjO-pciY77Z1lETO2Ha_GVbw,784
 data_processing_ray/runtime/ray/execution_configuration.py,sha256=C9YFixlATr7PPpkVQ0WzjCCPTWFuP80W2rnzY1bbp5I,4628
 data_processing_ray/runtime/ray/ray_utils.py,sha256=eDPm-pybPOELjKkvoz3l-qFU-k1Iwh-giGlXULiZjEk,10212
 data_processing_ray/runtime/ray/runtime_configuration.py,sha256=js9dXwdxjYbSigMC49F07XmbLjmj9HiipPE6BDaIGfA,1691
-data_processing_ray/runtime/ray/transform_file_processor.py,sha256=eR814VvfmPOlvyv_FU7eyt0HRqIzkkvACURxJCK-xrM,2335
+data_processing_ray/runtime/ray/transform_file_processor.py,sha256=vciH2OViK_RLSzze34kaLkRNi5Bg2JmH-KQAYx5zoo0,2388
 data_processing_ray/runtime/ray/transform_invoker.py,sha256=apfH8uilpm9sJ4IpHgiNdIzcH_IHGQba5fui4cCfolk,4026
 data_processing_ray/runtime/ray/transform_launcher.py,sha256=oxI3MFZI_-LzTwHbrHBIUqJ0htnliKBuALt86qijRwU,4304
-data_processing_ray/runtime/ray/transform_orchestrator.py,sha256=FZl7NM0eU1SxOcavZm4lru3laCswACPB1rjk3KK3FtY,7102
-data_processing_ray/runtime/ray/transform_runtime.py,sha256=0-b5syOW9zNnZxmMHDdwPo_pvoqDBiM5dHCgSakZhGQ,2531
+data_processing_ray/runtime/ray/transform_orchestrator.py,sha256=ZY4Wu7luS3kiUWAsTPZAJg4upyNRwGl2Ez0K832vAIw,7552
+data_processing_ray/runtime/ray/transform_runtime.py,sha256=s-9lrPBg-ifRRYJPJfbL8P9cOpYdvQKcairxpaIbB4E,2779
 data_processing_ray/runtime/ray/transform_statistics.py,sha256=cxrSQVnzRBCGS68IoiVGLoRBWBxPBSFFMDiT29FNt0g,3749
-data_processing_ray/test_support/transform/__init__.py,sha256=CKk-J3aEwH7OgDardyUEbLjlWaZWLUBs93PdukT4Rbc,100
-data_processing_ray/test_support/transform/noop_transform.py,sha256=ZTx09M9vNOaqrVzeuT2VmWM-IF4Upip0g0EtbHaOn-0,1588
+data_processing_ray/test_support/transform/__init__.py,sha256=P9stuAcnpjeGrJ8HJKRTuT2BKwLtG_3uG1UCYpnkw8c,213
+data_processing_ray/test_support/transform/noop_folder_transform.py,sha256=Sw5heLb4EyPYpTjoCW1K53FOt7BH1Zob2bLZUQbwmfs,2075
+data_processing_ray/test_support/transform/noop_transform.py,sha256=97o8oizjhBR_dqdrXhwss5QvW8-FZ_QYfTQGXpyUpGk,1564
 data_processing_spark/runtime/spark/__init__.py,sha256=bhY1xI9lL0GR2v1APahlhC5sh5rdVcGhQbWN4yoXApw,1233
 data_processing_spark/runtime/spark/execution_configuration.py,sha256=BqxUlpXFdHRK-csO2jaJJtktyKbcMtjIn3sjAPBfO58,3643
-data_processing_spark/runtime/spark/runtime_configuration.py,sha256=uABzBvzzFZ5HA_lGYEBFRd1qViMj5sbpKtCSLy64riM,1705
-data_processing_spark/runtime/spark/transform_file_processor.py,sha256=sdDBZZyqCqyKaJmEqZh1QzIqCDkLQzqV0dcAI5TRBjo,2611
+data_processing_spark/runtime/spark/runtime_configuration.py,sha256=lAx7aGLq0_-ySJSc_fx5ek1bIgLcGUgfmAM-HcVLkqc,2377
+data_processing_spark/runtime/spark/transform_file_processor.py,sha256=E0JEoEvAqdygYBHkvlm5kdkNbPO4aPQljfk833_Fqv8,2682
 data_processing_spark/runtime/spark/transform_launcher.py,sha256=1PZ-N4Wy2Qqiqr2z9S1xV88cNsAoHrmmuPadiOakJLM,2479
-data_processing_spark/runtime/spark/transform_orchestrator.py,sha256=HadnLNx_icy2n7CXOwqLiUA7vjV-gOvajxE0AQU3_NM,6645
-data_processing_spark/runtime/spark/transform_runtime.py,sha256=IKChGY1uGxFlAqZaL-XeSv_J3BMm3nev9MAs0NTT8og,2506
-data_processing_spark/test_support/transform/__init__.py,sha256=v58HbP2x9KF8MG8SOGWjodrTjU57KXlL0aPPB7z8KQQ,755
+data_processing_spark/runtime/spark/transform_orchestrator.py,sha256=64lme-KZz-a1CQUThYnLtx8m99VMlYv8MJB7dHCIAo4,9470
+data_processing_spark/runtime/spark/transform_runtime.py,sha256=je27rTRdd-5Wtd8nc8ogUwxZqUd4ZgP5122tA8JtTKA,3258
+data_processing_spark/test_support/transform/__init__.py,sha256=FQJyj7z1hXQynngMVQlCTJxTh2bdc4jN4220CBmLTqE,872
+data_processing_spark/test_support/transform/noop_folder_transform.py,sha256=z0jXCVKJYHPqB9ZTfUxnQkUVDnmfWjvss4_I3QZ8JZ4,2187
 data_processing_spark/test_support/transform/noop_transform.py,sha256=0FR3o-LnXf-UFS5gU0j-i4LVlw1mHDxGaPI40dkkIKY,1694
-data_prep_toolkit-0.2.2.dev1.dist-info/METADATA,sha256=QiZEK2qc8or6csEZk_weYltxiDRFvDb0chVpwLMCMrU,1235
-data_prep_toolkit-0.2.2.dev1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-data_prep_toolkit-0.2.2.dev1.dist-info/top_level.txt,sha256=XGMDmY55_pe5KeRWvO0un9a640e2v99tzbBBtjNybPM,58
-data_prep_toolkit-0.2.2.dev1.dist-info/RECORD,,
+data_prep_toolkit-0.2.2.dev2.dist-info/METADATA,sha256=XgskYjPA5pddqDgaBrPpe1IeqOpHPB2WscNM4dRh7XQ,2240
+data_prep_toolkit-0.2.2.dev2.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+data_prep_toolkit-0.2.2.dev2.dist-info/top_level.txt,sha256=XGMDmY55_pe5KeRWvO0un9a640e2v99tzbBBtjNybPM,58
+data_prep_toolkit-0.2.2.dev2.dist-info/RECORD,,

{data_prep_toolkit-0.2.2.dev1.dist-info → data_prep_toolkit-0.2.2.dev2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.1.0)
+Generator: setuptools (75.3.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

data_processing/data_access/data_access.py

@@ -358,7 +358,10 @@ class DataAccess:
         :param path: input file location
         :return: output file location
         """
-        raise NotImplementedError("Subclasses should implement this!")
+        if self.get_output_folder() is None:
+            self.logger.error("Get out put location. S3 configuration is not provided, returning None")
+            return None
+        return path.replace(self.get_input_folder(), self.get_output_folder())
 
     def save_table(self, path: str, table: pa.Table) -> tuple[int, dict[str, Any], int]:
         """

data_processing/data_access/data_access_local.py

@@ -130,17 +130,6 @@ class DataAccessLocal(DataAccess):
             logger.error(f"Error reading table from {path}: {e}")
             return None, 0
 
-    def get_output_location(self, path: str) -> str:
-        """
-        Get output location based on input
-        :param path: input file location
-        :return: output file location
-        """
-        if self.output_folder is None:
-            logger.error("Get output location. local configuration is not defined, returning None")
-            return None
-        return path.replace(self.input_folder, self.output_folder)
-
     def save_table(self, path: str, table: pa.Table) -> tuple[int, dict[str, Any], int]:
         """
         Saves a pyarrow table to a file and returns information about the operation.

data_processing/data_access/data_access_s3.py

@@ -126,17 +126,6 @@ class DataAccessS3(DataAccess):
             self.logger.error(f"Exception reading table {path} from S3 - {e}")
             return None, 0
 
-    def get_output_location(self, path: str) -> str:
-        """
-        Get output location based on input
-        :param path: input file location
-        :return: output file location
-        """
-        if self.output_folder is None:
-            self.logger.error("Get out put location. S3 configuration is not provided, returning None")
-            return None
-        return path.replace(self.input_folder, self.output_folder)
-
     def save_table(self, path: str, table: pyarrow.Table) -> tuple[int, dict[str, Any], int]:
         """
         Save table to a given location

data_processing/runtime/pure_python/transform_file_processor.py

@@ -14,7 +14,7 @@ from typing import Any
 
 from data_processing.data_access import DataAccessFactoryBase
 from data_processing.runtime import AbstractTransformFileProcessor
-from data_processing.transform import AbstractBinaryTransform, TransformStatistics
+from data_processing.transform import AbstractTransform, TransformStatistics
 from data_processing.utils import UnrecoverableException
 
 
@@ -28,7 +28,8 @@ class PythonTransformFileProcessor(AbstractTransformFileProcessor):
         data_access_factory: DataAccessFactoryBase,
         statistics: TransformStatistics,
         transform_params: dict[str, Any],
-        transform_class: type[AbstractBinaryTransform],
+        transform_class: type[AbstractTransform],
+        is_folder: bool,
     ):
         """
         Init method
@@ -36,11 +37,13 @@ class PythonTransformFileProcessor(AbstractTransformFileProcessor):
         :param statistics - reference to statistics class
         :param transform_params - transform parameters
         :param transform_class: transform class
+        :param is_folder: folder transform flag
         """
         # invoke superclass
         super().__init__(
             data_access_factory=data_access_factory,
             transform_parameters=dict(transform_params),
+            is_folder=is_folder,
         )
         self.transform_params["statistics"] = statistics
         # Create local processor
@@ -65,17 +68,20 @@ class PythonPoolTransformFileProcessor(AbstractTransformFileProcessor):
         self,
         data_access_factory: DataAccessFactoryBase,
         transform_params: dict[str, Any],
-        transform_class: type[AbstractBinaryTransform],
+        transform_class: type[AbstractTransform],
+        is_folder: bool
     ):
         """
         Init method
         :param data_access_factory - data access factory
         :param transform_params - transform parameters
         :param transform_class: transform class
+        :param is_folder: folder tranform flag
         """
         super().__init__(
             data_access_factory=data_access_factory,
             transform_parameters=dict(transform_params),
+            is_folder=is_folder,
         )
         # Add data access and statistics to the processor parameters
         self.transform_params["data_access"] = self.data_access

data_processing/runtime/pure_python/transform_orchestrator.py

@@ -24,14 +24,13 @@ from data_processing.runtime.pure_python import (
     PythonTransformFileProcessor,
     PythonTransformRuntimeConfiguration,
 )
-from data_processing.transform import AbstractBinaryTransform, TransformStatistics
+from data_processing.transform import AbstractTransform, TransformStatistics, AbstractFolderTransform
 from data_processing.utils import GB, get_logger
 
 
 logger = get_logger(__name__)
 
 
-@staticmethod
 def _execution_resources() -> dict[str, Any]:
     """
     Get Execution resource
@@ -49,7 +48,6 @@ def _execution_resources() -> dict[str, Any]:
     }
 
 
-
 def orchestrate(
     data_access_factory: DataAccessFactoryBase,
     runtime_config: PythonTransformRuntimeConfiguration,
@@ -74,15 +72,21 @@ def orchestrate(
         return 1
     # create additional execution parameters
     runtime = runtime_config.create_transform_runtime()
+    is_folder = issubclass(runtime_config.get_transform_class(), AbstractFolderTransform)
     try:
-        # Get files to process
-        files, profile, retries = data_access.get_files_to_process()
-        if len(files) == 0:
-            logger.error("No input files to process - exiting")
-            return 0
-        if retries > 0:
-            statistics.add_stats({"data access retries": retries})
-        logger.info(f"Number of files is {len(files)}, source profile {profile}")
+        if is_folder:
+            # folder transform
+            files = runtime.get_folders(data_access=data_access)
+            logger.info(f"Number of folders is {len(files)}")
+        else:
+            # Get files to process
+            files, profile, retries = data_access.get_files_to_process()
+            if len(files) == 0:
+                logger.error("No input files to process - exiting")
+                return 0
+            if retries > 0:
+                statistics.add_stats({"data access retries": retries})
+            logger.info(f"Number of files is {len(files)}, source profile {profile}")
         # Print interval
         print_interval = int(len(files) / 100)
         if print_interval == 0:
@@ -99,6 +103,7 @@ def orchestrate(
                     data_access_factory=data_access_factory, statistics=statistics, files=files
                 ),
                 transform_class=runtime_config.get_transform_class(),
+                is_folder=is_folder,
             )
         else:
             # using sequential execution
@@ -111,6 +116,7 @@ def orchestrate(
                     data_access_factory=data_access_factory, statistics=statistics, files=files
                 ),
                 transform_class=runtime_config.get_transform_class(),
+                is_folder=is_folder,
             )
         status = "success"
         return_code = 0
@@ -139,7 +145,8 @@ def orchestrate(
             "job_input_params": input_params
             | data_access_factory.get_input_params()
             | execution_config.get_input_params(),
-            "execution_stats": _execution_resources() | {"execution time, min": round((time.time() - start_time) / 60.0, 3)},
+            "execution_stats": _execution_resources() |
+                               {"execution time, min": round((time.time() - start_time) / 60.0, 3)},
             "job_output_stats": stats,
         }
         logger.debug(f"Saving job metadata: {metadata}.")
@@ -157,7 +164,8 @@ def _process_transforms(
     data_access_factory: DataAccessFactoryBase,
     statistics: TransformStatistics,
     transform_params: dict[str, Any],
-    transform_class: type[AbstractBinaryTransform],
+    transform_class: type[AbstractTransform],
+    is_folder: bool,
 ) -> None:
     """
     Process transforms sequentially
@@ -167,9 +175,8 @@ def _process_transforms(
     :param data_access_factory: data access factory
     :param transform_params - transform parameters
     :param transform_class: transform class
+    :param is_folder: folder transform flag
     :return: metadata for the execution
-
-    :return: None
     """
     # create executor
     executor = PythonTransformFileProcessor(
@@ -177,6 +184,7 @@ def _process_transforms(
         statistics=statistics,
         transform_params=transform_params,
         transform_class=transform_class,
+        is_folder=is_folder,
     )
     # process data
     t_start = time.time()
@@ -202,7 +210,8 @@ def _process_transforms_multiprocessor(
     print_interval: int,
     data_access_factory: DataAccessFactoryBase,
     transform_params: dict[str, Any],
-    transform_class: type[AbstractBinaryTransform],
+    transform_class: type[AbstractTransform],
+    is_folder: bool
 ) -> TransformStatistics:
     """
     Process transforms using multiprocessing pool
@@ -212,13 +221,17 @@ def _process_transforms_multiprocessor(
     :param data_access_factory: data access factory
     :param transform_params - transform parameters
     :param transform_class: transform class
+    :param is_folder: folder transform class
     :return: metadata for the execution
     """
     # result statistics
     statistics = TransformStatistics()
     # create processor
     processor = PythonPoolTransformFileProcessor(
-        data_access_factory=data_access_factory, transform_params=transform_params, transform_class=transform_class
+        data_access_factory=data_access_factory,
+        transform_params=transform_params,
+        transform_class=transform_class,
+        is_folder=is_folder,
     )
     completed = 0
     t_start = time.time()

data_processing/runtime/pure_python/transform_runtime.py

@@ -12,7 +12,7 @@
 
 from typing import Any
 
-from data_processing.data_access import DataAccessFactoryBase
+from data_processing.data_access import DataAccessFactoryBase, DataAccess
 from data_processing.transform import TransformStatistics
 
 
@@ -28,6 +28,14 @@ class DefaultPythonTransformRuntime:
         """
         self.params = params
 
+    def get_folders(self, data_access: DataAccess) -> list[str]:
+        """
+        Get folders to process
+        :param data_access: data access
+        :return: list of folders to process
+        """
+        raise NotImplemented()
+
     def get_transform_config(
         self, data_access_factory: DataAccessFactoryBase, statistics: TransformStatistics, files: list[str]
     ) -> dict[str, Any]:

data_processing/runtime/transform_file_processor.py

@@ -26,11 +26,13 @@ class AbstractTransformFileProcessor:
         self,
         data_access_factory: DataAccessFactoryBase,
         transform_parameters: dict[str, Any],
+        is_folder: bool = False,
     ):
         """
         Init method
         :param data_access_factory: Data Access Factory
         :param transform_parameters: Transform parameters
+        :param is_folder: folder transform flag
         """
         self.logger = get_logger(__name__)
         # validate parameters
@@ -46,6 +48,7 @@ class AbstractTransformFileProcessor:
         # Add data access and statistics to the processor parameters
         self.transform_params = transform_parameters
         self.transform_params["data_access"] = self.data_access
+        self.is_folder = is_folder
 
     def process_file(self, f_name: str) -> None:
         """
@@ -58,25 +61,30 @@ class AbstractTransformFileProcessor:
             self.logger.warning("No data_access found. Returning.")
             return
         t_start = time.time()
-        # Read source file
-        filedata, retries = self.data_access.get_file(path=f_name)
-        if retries > 0:
-            self._publish_stats({"data access retries": retries})
-        if filedata is None:
-            self.logger.warning(f"File read resulted in None for {f_name}. Returning.")
-            self._publish_stats({"failed_reads": 1})
-            return
-        self._publish_stats({"source_files": 1, "source_size": len(filedata)})
+        if not self.is_folder:
+            # Read source file only if we are processing file
+            filedata, retries = self.data_access.get_file(path=f_name)
+            if retries > 0:
+                self._publish_stats({"data access retries": retries})
+            if filedata is None:
+                self.logger.warning(f"File read resulted in None for {f_name}. Returning.")
+                self._publish_stats({"failed_reads": 1})
+                return
+            self._publish_stats({"source_files": 1, "source_size": len(filedata)})
         # Process input file
         try:
-            # execute local processing
-            name_extension = TransformUtils.get_file_extension(f_name)
             self.logger.debug(f"Begin transforming file {f_name}")
-            out_files, stats = self.transform.transform_binary(file_name=f_name, byte_array=filedata)
+            if not self.is_folder:
+                # execute local processing
+                out_files, stats = self.transform.transform_binary(file_name=f_name, byte_array=filedata)
+                name_extension = TransformUtils.get_file_extension(f_name)
+                self.last_file_name = name_extension[0]
+                self.last_file_name_next_index = None
+                self.last_extension = name_extension[1]
+            else:
+                out_files, stats = self.transform.transform(folder_name=f_name)
+                self.last_file_name = f_name
             self.logger.debug(f"Done transforming file {f_name}, got {len(out_files)} files")
-            self.last_file_name = name_extension[0]
-            self.last_file_name_next_index = None
-            self.last_extension = name_extension[1]
             # save results
             self._submit_file(t_start=t_start, out_files=out_files, stats=stats)
         # Process unrecoverable exceptions
@@ -95,10 +103,10 @@ class AbstractTransformFileProcessor:
         the hook for them to return back locally stored data and their statistics.
         :return: None
         """
-        if self.last_file_name is None:
+        if self.last_file_name is None or self.is_folder:
             # for some reason a given worker never processed anything. Happens in testing
             # when the amount of workers is greater than the amount of files
-            self.logger.debug("skipping flush, no name for file is defined")
+            self.logger.debug("skipping flush, no name for file is defined or this is a folder transform")
             return
         try:
             t_start = time.time()
@@ -141,15 +149,21 @@ class AbstractTransformFileProcessor:
                 )
             case 1:
                 # we have exactly 1 output file
-                file_ext = out_files[0]
-                lfn = self.last_file_name
-                if self.last_file_name_next_index is not None:
-                    lfn = f"{lfn}_{self.last_file_name_next_index}"
-                output_name = self.data_access.get_output_location(path=f"{lfn}{file_ext[1]}")
+                if self.is_folder:
+                    # its folder
+                    output_name = out_files[0][1]
+                    dt = out_files[0][0]
+                else:
+                    file_ext = out_files[0]
+                    lfn = self.last_file_name
+                    if self.last_file_name_next_index is not None:
+                        lfn = f"{lfn}_{self.last_file_name_next_index}"
+                    output_name = self.data_access.get_output_location(path=f"{lfn}{file_ext[1]}")
+                    dt = file_ext[0]
                 self.logger.debug(
                     f"Writing transformed file {self.last_file_name}{self.last_extension} to {output_name}"
                 )
-                save_res, retries = self.data_access.save_file(path=output_name, data=file_ext[0])
+                save_res, retries = self.data_access.save_file(path=output_name, data=dt)
                 if retries > 0:
                     self._publish_stats({"data access retries": retries})
                 if save_res is None:
@@ -159,7 +173,7 @@ class AbstractTransformFileProcessor:
                 self._publish_stats(
                     {
                         "result_files": 1,
-                        "result_size": len(file_ext[0]),
+                        "result_size": len(dt),
                         "processing_time": time.time() - t_start,
                     }
                 )
@@ -176,14 +190,21 @@ class AbstractTransformFileProcessor:
                     start_index = 0
                 count = len(out_files)
                 for index in range(count):
-                    file_ext = out_files[index]
-                    output_name_indexed = f"{output_file_name}_{start_index + index}{file_ext[1]}"
-                    file_sizes += len(file_ext[0])
-                    self.logger.debug(
-                        f"Writing transformed file {self.last_file_name}{self.last_extension}, {index + 1} "
-                        f"of {count}  to {output_name_indexed}"
-                    )
-                    save_res, retries = self.data_access.save_file(path=output_name_indexed, data=file_ext[0])
+                    if self.is_folder:
+                        # its a folder
+                        output_name_indexed = out_files[index][1]
+                        dt = out_files[index][0]
+                    else:
+                        # files
+                        file_ext = out_files[index]
+                        output_name_indexed = f"{output_file_name}_{start_index + index}{file_ext[1]}"
+                        self.logger.debug(
+                            f"Writing transformed file {self.last_file_name}{self.last_extension}, {index + 1} "
+                            f"of {count}  to {output_name_indexed}"
+                        )
+                        dt = file_ext[0]
+                    file_sizes += len(dt)
+                    save_res, retries = self.data_access.save_file(path=output_name_indexed, data=dt)
                     if retries > 0:
                         self._publish_stats({"data access retries": retries})
                     if save_res is None:

data_processing/test_support/data_access/data_access_factory_test.py

@@ -1,3 +1,15 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
 import os
 import sys
 from argparse import ArgumentParser

data_processing/test_support/transform/__init__.py

@@ -1,6 +1,11 @@
-from .table_transform_test import AbstractTableTransformTest
-from .binary_transform_test import AbstractBinaryTransformTest
-from .noop_transform import (
+from data_processing.test_support.transform.table_transform_test import AbstractTableTransformTest
+from data_processing.test_support.transform.binary_transform_test import AbstractBinaryTransformTest
+from data_processing.test_support.transform.noop_transform import (
     NOOPTransform,
-    NOOPPythonTransformConfiguration,
+    NOOPTransformConfiguration,
+    NOOPPythonTransformConfiguration
 )
+from data_processing.test_support.transform.noop_folder_transform import (
+    NOOPFolderTransform,
+    NOOPFolderPythonTransformConfiguration
+)