data-prep-toolkit-transforms 0.2.1__tar.gz

data_prep_toolkit_transforms-0.2.1/Makefile

@@ -0,0 +1,62 @@
+# Define the root of the local git clone for the common rules to be able
+# know where they are running from.
+REPOROOT=../../..
+# Include a library of common .transform.* targets which most
+# transforms should be able to reuse.  However, feel free
+# to override/redefine the rules below.
+
+# $(REPOROOT)/.make.versions file contains the versions
+
+include $(REPOROOT)/transforms/.make.transforms
+include ../.make.packaging
+
+PACKAGING_RUN_TIME=python
+
+
+#Excluded List
+# ./code/malware
+# ./universal/html2parquet
+# ./universal/profiler      # Missing implementation 
+# ./universal/fdedup 	    # Missing implementation
+#  code/repo_level_ordering # Missing implementation
+
+
+TRANSFORMS_NAMES = code/code_quality \
+	    	code/code2parquet \
+    		code/header_cleanser \
+		code/proglang_select \
+		language/doc_chunk \
+		language/doc_quality \
+		language/lang_id \
+		language/pdf2parquet \
+		language/pii_redactor \
+		language/text_encoder \
+		universal/ededup \
+		universal/filter \
+		universal/resize \
+		universal/tokenization \
+		universal/doc_id
+
+
+# distribution versions is the same as image version.
+set-versions:
+	$(MAKE) TRANSFORM_PYTHON_VERSION=$(DPK_TRANSFORMS_VERSION) TOML_VERSION=$(DPK_TRANSFORMS_VERSION) .transforms.set-versions 
+
+test-src:: 
+	$(MAKE) src 
+	$(MAKE) .transforms.python-venv
+	$(MAKE) run-ut
+	@# Help: Do any default transform setup before running make src and setting up a test environment
+
+test-with-pypi:
+	-rm -fr venv
+	$(MAKE) .defaults.create-venv
+	source venv/bin/activate;       \
+#	$(PYTHON) -m pip install dist/data_prep_toolkit_transforms-$(DPK_TRANSFORMS_VERSION)-py3-none-any.whl
+#	$(PYTHON) -m pip install data_prep_toolkit_transforms==$(DPK_TRANSFORMS_VERSION) 
+	$(MAKE) run-ut
+	@# Help: Load wheel from pypi and run all unit tests: final step in verification after deploying to pypi)
+
+
+
+

data_prep_toolkit_transforms-0.2.1/PKG-INFO

@@ -0,0 +1,73 @@
+Metadata-Version: 2.1
+Name: data_prep_toolkit_transforms
+Version: 0.2.1
+Summary: Data Preparation Toolkit Transforms
+Author-email: Maroun Touma <touma@us.ibm.com>
+License: Apache-2.0
+Keywords: transforms,data preprocessing,data preparation,llm,generative,ai,fine-tuning,llmapps
+Requires-Python: <3.12,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: data-prep-toolkit>=0.2.1
+Requires-Dist: bs4==0.0.2
+Requires-Dist: docling-ibm-models==1.1.7
+Requires-Dist: deepsearch-glm==0.21.0
+Requires-Dist: docling==1.11.0
+Requires-Dist: filetype<2.0.0,>=1.2.0
+Requires-Dist: docling-core==1.3.0
+Requires-Dist: llama-index-core<0.12.0,>=0.11.0
+Requires-Dist: duckdb==0.10.1
+Requires-Dist: fasttext==0.9.2
+Requires-Dist: huggingface-hub<1.0.0,>=0.21.4
+Requires-Dist: langcodes==3.3.0
+Requires-Dist: mmh3==4.1.0
+Requires-Dist: numpy==1.26.4
+Requires-Dist: pandas
+Requires-Dist: parameterized
+Requires-Dist: scancode-toolkit==32.1.0; platform_system != "Darwin"
+Requires-Dist: sentence-transformers==3.0.1
+Requires-Dist: transformers==4.38.2
+Requires-Dist: xxhash==3.4.1
+Requires-Dist: presidio-analyzer>=2.2.355
+Requires-Dist: presidio-anonymizer>=2.2.355
+Requires-Dist: flair>=0.14.0
+Requires-Dist: pandas>=2.2.2
+
+# DPK Python Transforms
+
+## installation
+
+The [transforms](https://github.com/IBM/data-prep-kit/blob/dev/transforms/README.md) are delivered as a standard pyton library available on pypi and can be installed using pip install:
+
+`python -m pip install data-prep-toolkit-transforms`
+
+installing the python transforms will also install  `data-prep-toolkit`
+
+## List of Transforms in current package
+
+Note: This list includes the transforms that are part of the current release for 0.2.1.dev3 and will be maintained on best effort but may may not be always up to date. users are encourage to raise an issue in git when they discover missing components
+
+* code
+    * [code2parquet](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/code2parquet/python/README.md)
+    * [header_cleanser (Not available on MacOS)](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/header_cleanser/python/README.md)
+    * [code_quality](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/code_quality/python/README.md)
+    * [proglang_select](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/proglang_select/python/README.md)
+* language
+    * [doc_chunk](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/doc_chunk/python/README.md)
+	* [doc_quality](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/doc_quality/python/README.md)
+	* [lang_id](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/lang_id/python/README.md)
+	* [pdf2parquet](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/pdf2parquet/python/README.md)
+	* [text_encoder](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/text_encoder/python/README.md)
+	* [pii_redactor](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/pii_redactor/python/README.md)
+* universal
+    * [ededup](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/ededup/python/README.md)
+	* [filter](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/filter/python/README.md)
+	* [resize](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/resize/python/README.md)
+	* [tokenization](https://github.com/IBM/data-prep-kit/blob/dev/transforms/tokenization/doc_chunk/python/README.md)
+	* [doc_id](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/doc_id/python/README.md)
+
+	
+
+
+
+
+ 

data_prep_toolkit_transforms-0.2.1/README.md

@@ -0,0 +1,39 @@
+# DPK Python Transforms
+
+## installation
+
+The [transforms](https://github.com/IBM/data-prep-kit/blob/dev/transforms/README.md) are delivered as a standard pyton library available on pypi and can be installed using pip install:
+
+`python -m pip install data-prep-toolkit-transforms`
+
+installing the python transforms will also install  `data-prep-toolkit`
+
+## List of Transforms in current package
+
+Note: This list includes the transforms that are part of the current release for 0.2.1.dev3 and will be maintained on best effort but may may not be always up to date. users are encourage to raise an issue in git when they discover missing components
+
+* code
+    * [code2parquet](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/code2parquet/python/README.md)
+    * [header_cleanser (Not available on MacOS)](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/header_cleanser/python/README.md)
+    * [code_quality](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/code_quality/python/README.md)
+    * [proglang_select](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/proglang_select/python/README.md)
+* language
+    * [doc_chunk](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/doc_chunk/python/README.md)
+	* [doc_quality](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/doc_quality/python/README.md)
+	* [lang_id](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/lang_id/python/README.md)
+	* [pdf2parquet](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/pdf2parquet/python/README.md)
+	* [text_encoder](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/text_encoder/python/README.md)
+	* [pii_redactor](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/pii_redactor/python/README.md)
+* universal
+    * [ededup](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/ededup/python/README.md)
+	* [filter](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/filter/python/README.md)
+	* [resize](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/resize/python/README.md)
+	* [tokenization](https://github.com/IBM/data-prep-kit/blob/dev/transforms/tokenization/doc_chunk/python/README.md)
+	* [doc_id](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/doc_id/python/README.md)
+
+	
+
+
+
+
+ 

data_prep_toolkit_transforms-0.2.1/pyproject.toml

@@ -0,0 +1,39 @@
+[project]
+name = "data_prep_toolkit_transforms"
+version = "0.2.1"
+requires-python = ">=3.10,<3.12"
+keywords = ["transforms", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
+description = "Data Preparation Toolkit Transforms"
+license = {text = "Apache-2.0"}
+readme = {file = "README.md", content-type = "text/markdown"}
+authors = [
+    { name = "Maroun Touma", email = "touma@us.ibm.com" },
+]
+dynamic = ["dependencies"]
+
+[build-system]
+requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+
+[options]
+package_dir = ["src"]
+
+[options.packages.find]
+where = ["src/"]
+
+[tool.pytest.ini_options]
+# Currently we use low coverage since we have to run tests separately (see makefile)
+#addopts = "--cov --cov-report term-missing --cov-fail-under 25"
+markers = ["unit: unit tests", "integration: integration tests"]
+
+[tool.coverage.run]
+include = ["src/*"]
+
+
+
+
+
+

data_prep_toolkit_transforms-0.2.1/requirements.txt

@@ -0,0 +1,31 @@
+data-prep-toolkit>=0.2.1
+bs4==0.0.2
+#pdf2parquet
+# conflict with chunking....
+#docling-core==1.2.0,
+docling-ibm-models==1.1.7,
+deepsearch-glm==0.21.0,
+docling==1.11.0,
+filetype >=1.2.0, <2.0.0,
+#Doc chunking
+docling-core==1.3.0,
+llama-index-core>=0.11.0,<0.12.0,
+duckdb==0.10.1
+fasttext==0.9.2
+huggingface-hub >= 0.21.4, <1.0.0
+langcodes==3.3.0
+mmh3==4.1.0
+numpy==1.26.4
+pandas
+parameterized
+scancode-toolkit==32.1.0 ; platform_system != 'Darwin'
+sentence-transformers==3.0.1
+transformers==4.38.2
+xxhash==3.4.1
+# PII-redactor
+presidio-analyzer>=2.2.355
+presidio-anonymizer>=2.2.355
+flair>=0.14.0
+pandas>=2.2.2
+
+

data_prep_toolkit_transforms-0.2.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

data_prep_toolkit_transforms-0.2.1/src/cc_net_prepro.py

@@ -0,0 +1,168 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+"""
+@ Create on: 2023/04/25
+@ Description:
+    To incorporate preprocessing steps from cc_net/
+
+@ Reference:
+    https://github.com/facebookresearch/cc_net/blob/main/cc_net/text_normalizer.py
+    https://github.com/bigscience-workshop/data_tooling/blob/master/kenlm_training/cc_net/text_normalizer.py
+"""
+
+import re
+import unicodedata
+from typing import Dict
+
+
+unicode_punct_dict: Dict[str, str] = {
+    "，": ",",
+    "。": ".",
+    "、": ",",
+    "„": '"',
+    "”": '"',
+    "“": '"',
+    "«": '"',
+    "»": '"',
+    "１": '"',
+    "」": '"',
+    "「": '"',
+    "《": '"',
+    "》": '"',
+    "´": "'",
+    "∶": ":",
+    "：": ":",
+    "？": "?",
+    "！": "!",
+    "（": "(",
+    "）": ")",
+    "；": ";",
+    "–": "-",
+    "—": " - ",
+    "．": ". ",
+    "～": "~",
+    "’": "'",
+    "…": "...",
+    "━": "-",
+    "〈": "<",
+    "〉": ">",
+    "【": "[",
+    "】": "]",
+    "％": "%",
+    "►": "-",
+}
+
+_unicode_punct_re = re.compile(f"[{''.join(unicode_punct_dict.keys())}]")
+
+"""
+Generate regex pattern obj for later searching using re.search() or re.match()
+(r'[\x00\x01...\x9e\x9f]',re.UNICODE)
+"""
+_non_printing_chars_re = re.compile(f"[{''.join(map(chr, list(range(0, 32)) + list(range(127, 160))))}]")
+
+
+def unicode_normalization(line: str, language="en") -> str:
+    """
+    For some languages like 'ja' or 'en', its text should be normalized with the right unicode format
+    prior to calling tokenization by sentence piece tokenizer
+    """
+    if language == "ja":
+        line = unicodedata.normalize("NFKC", line)
+    elif language == "en":
+        line = unicodedata.normalize("NFD", line)  # normalize line using Unicode Normalization Form D
+    else:
+        """
+        TODO: add relevant unicodedata normalization here for other languages as needed
+        """
+        return line
+
+    return line
+
+
+def _strip_accents(line) -> str:
+    """
+    This currently is applied for `en` and `ja` language to strip out accents from text.
+    The given text 'line' should be normalized with the right unicode format prior to calling this method.
+    For example:
+        line = "Café élevàtor ôperàtor naïve Noël façade don't"
+            -> "Cafe elevator operator naive Noel facade don't"  if line was normalized with unicode format NFD
+            -> "Cafe elevator operator naïve Noël façade don't"  if line was NOT normalized with any unicode format
+    """
+
+    """Keep char whose category is NOT "Mn", i.e, 
+    "Mn" is category for Mark or Non-Spacing characters like diacritic/accent or non-spacing marks
+    Example of some diacritic/non-spacing marks: ^,´,` as they don't occupy space
+    """
+    output = [c for c in line if unicodedata.category(c) != "Mn"]  # decompose line into chars and diacritical marks
+    if len(output) == line:
+        return line
+    return "".join(output)
+
+
+def _replace_unicode_punct(line: str) -> str:
+    """
+    Replace unicode punctuations defined in `unicode_punct_dict'
+    """
+    return "".join(unicode_punct_dict.get(c, c) for c in line)
+
+
+def _remove_unicode_punct(line: str) -> str:
+    """
+    More aggressive _replace_unicode_punct
+    """
+    return _unicode_punct_re.sub("", line)
+
+
+def _remove_non_printing_char(line: str) -> str:
+    return _non_printing_chars_re.sub("", line)
+
+
+def cc_net_normalize(
+    line: str,
+    strip_accent: bool = True,
+    lower_case: bool = True,
+    digit_2_zero: bool = True,
+    punct_level: int = 1,
+    language: str = "en",
+) -> str:
+    line = line.strip()
+
+    if not line:
+        return line
+
+    line = unicode_normalization(line=line, language=language)
+
+    if lower_case:
+        line = line.lower()
+
+    if strip_accent:
+        line = _strip_accents(line)
+
+    if digit_2_zero:
+        # eg, "int 10 float 2.01 scientific 1.2e10" -> "int 00 float 0.00 scientific 0.0e00"
+        line = re.compile(r"\d").sub("0", line)
+
+    if punct_level == 1:
+        line = _replace_unicode_punct(line)
+
+    elif punct_level == 2:
+        line = _remove_unicode_punct(line)
+
+    line = _remove_non_printing_char(line)
+    return line
+
+
+if __name__ == "__main__":
+    line = "Int 10 float 2.01 scientific 1.2e10 Café ôperàtor"
+    new_line = cc_net_normalize(line)
+    print(f"== {line} -> {new_line}")

data_prep_toolkit_transforms-0.2.1/src/code2parquet_local.py

@@ -0,0 +1,51 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import ast
+import os
+
+from code2parquet_transform import (  # domain_key,; snapshot_key,
+    CodeToParquetTransform,
+    data_factory_key,
+    detect_programming_lang_key,
+    supported_langs_file_key,
+)
+from data_processing.data_access import DataAccessFactory, DataAccessLocal
+
+
+supported_languages_file = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "../../ray/test-data/languages/lang_extensions.json")
+)
+input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input"))
+
+params = {
+    supported_langs_file_key: supported_languages_file,
+    detect_programming_lang_key: True,
+    # snapshot_key: "github",
+    # domain_key: "code",
+    "data_files_to_use": ast.literal_eval("['.zip']"),
+    data_factory_key: DataAccessFactory(),  # Expect to create DataAccessLocal
+}
+
+if __name__ == "__main__":
+    # Here we show how to run outside of ray
+    # Create and configure the transform.
+    # transform = CodeToParquetPythonTransform(params)
+    transform = CodeToParquetTransform(params)
+    # Use the local data access to read a parquet table.
+    data_access = DataAccessLocal()
+    file_to_process = os.path.join(input_folder, "application-java.zip")
+    byte_array, _ = data_access.get_file(file_to_process)
+    # Transform the table
+    files_list, metadata = transform.transform_binary(file_name=file_to_process, byte_array=byte_array)
+    print(f"Got {len(files_list)} output files")
+    print(f"output metadata : {metadata}")

data_prep_toolkit_transforms-0.2.1/src/code2parquet_local_python.py

@@ -0,0 +1,60 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import ast
+import os
+import sys
+
+from code2parquet_transform import (  # domain_key,; snapshot_key,
+    detect_programming_lang_cli_key,
+    supported_langs_file_cli_key,
+)
+from code2parquet_transform_python import CodeToParquetPythonConfiguration
+from data_processing.runtime.pure_python import PythonTransformLauncher
+from data_processing.utils import ParamsUtils
+
+
+# create parameters
+supported_languages_file = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "../test-data/languages/lang_extensions.json")
+)
+input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input"))
+output_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../output"))
+local_conf = {
+    "input_folder": input_folder,
+    "output_folder": output_folder,
+}
+worker_options = {"num_cpus": 0.8}
+code_location = {"github": "github", "commit_hash": "12345", "path": "path"}
+ingest_config = {
+    supported_langs_file_cli_key: supported_languages_file,
+    detect_programming_lang_cli_key: True,
+    # snapshot_key: "github",
+    # domain_key: "code",
+}
+
+params = {
+    # Data access. Only required parameters are specified
+    "data_local_config": ParamsUtils.convert_to_ast(local_conf),
+    "data_files_to_use": ast.literal_eval("['.zip']"),
+    # orchestrator
+    "runtime_pipeline_id": "pipeline_id",
+    "runtime_job_id": "job_id",
+    "runtime_code_location": ParamsUtils.convert_to_ast(code_location),
+}
+
+if __name__ == "__main__":
+    sys.argv = ParamsUtils.dict_to_req(d=(params | ingest_config))
+    # create launcher
+    launcher = PythonTransformLauncher(CodeToParquetPythonConfiguration())
+    # launch
+    launcher.launch()

data_prep_toolkit_transforms-0.2.1/src/code2parquet_s3_python.py

@@ -0,0 +1,61 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import ast
+import sys
+
+from code2parquet_transform import (  # domain_key,; snapshot_key,
+    detect_programming_lang_cli_key,
+    supported_langs_file_cli_key,
+)
+from code2parquet_transform_python import CodeToParquetPythonConfiguration
+from data_processing.runtime.pure_python import PythonTransformLauncher
+from data_processing.utils import GB, ParamsUtils
+
+
+# create parameters
+s3_cred = {
+    "access_key": "localminioaccesskey",
+    "secret_key": "localminiosecretkey",
+    "url": "http://localhost:9000",
+}
+s3_conf = {
+    "input_folder": "test/ingest_2_parquet/input",
+    "output_folder": "test/ingest_2_parquet/output",
+}
+worker_options = {"num_cpus": 0.8, "memory": 2 * GB}
+code_location = {"github": "github", "commit_hash": "12345", "path": "path"}
+ingest_config = {
+    supported_langs_file_cli_key: "test/ingest_2_parquet/languages/lang_extensions.json",
+    detect_programming_lang_cli_key: True,
+    # snapshot_key: "github",
+    # domain_key: "code",
+    "code2parquet_s3_cred": ParamsUtils.convert_to_ast(s3_cred),
+}
+
+params = {
+    # Data access. Only required parameters are specified
+    "data_s3_cred": ParamsUtils.convert_to_ast(s3_cred),
+    "data_s3_config": ParamsUtils.convert_to_ast(s3_conf),
+    "data_files_to_use": ast.literal_eval("['.zip']"),
+    # orchestrator
+    "runtime_pipeline_id": "pipeline_id",
+    "runtime_job_id": "job_id",
+    "runtime_code_location": ParamsUtils.convert_to_ast(code_location),
+}
+
+if __name__ == "__main__":
+    sys.argv = ParamsUtils.dict_to_req(d=(params | ingest_config))
+    # create launcher
+    launcher = PythonTransformLauncher(CodeToParquetPythonConfiguration())
+    # launch
+    launcher.launch()