data-prep-toolkit-transforms 0.2.2__1-py3-none-any.whl

cc_net_prepro.py

@@ -0,0 +1,168 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+"""
+@ Create on: 2023/04/25
+@ Description:
+    To incorporate preprocessing steps from cc_net/
+
+@ Reference:
+    https://github.com/facebookresearch/cc_net/blob/main/cc_net/text_normalizer.py
+    https://github.com/bigscience-workshop/data_tooling/blob/master/kenlm_training/cc_net/text_normalizer.py
+"""
+
+import re
+import unicodedata
+from typing import Dict
+
+
+unicode_punct_dict: Dict[str, str] = {
+    "，": ",",
+    "。": ".",
+    "、": ",",
+    "„": '"',
+    "”": '"',
+    "“": '"',
+    "«": '"',
+    "»": '"',
+    "１": '"',
+    "」": '"',
+    "「": '"',
+    "《": '"',
+    "》": '"',
+    "´": "'",
+    "∶": ":",
+    "：": ":",
+    "？": "?",
+    "！": "!",
+    "（": "(",
+    "）": ")",
+    "；": ";",
+    "–": "-",
+    "—": " - ",
+    "．": ". ",
+    "～": "~",
+    "’": "'",
+    "…": "...",
+    "━": "-",
+    "〈": "<",
+    "〉": ">",
+    "【": "[",
+    "】": "]",
+    "％": "%",
+    "►": "-",
+}
+
+_unicode_punct_re = re.compile(f"[{''.join(unicode_punct_dict.keys())}]")
+
+"""
+Generate regex pattern obj for later searching using re.search() or re.match()
+(r'[\x00\x01...\x9e\x9f]',re.UNICODE)
+"""
+_non_printing_chars_re = re.compile(f"[{''.join(map(chr, list(range(0, 32)) + list(range(127, 160))))}]")
+
+
+def unicode_normalization(line: str, language="en") -> str:
+    """
+    For some languages like 'ja' or 'en', its text should be normalized with the right unicode format
+    prior to calling tokenization by sentence piece tokenizer
+    """
+    if language == "ja":
+        line = unicodedata.normalize("NFKC", line)
+    elif language == "en":
+        line = unicodedata.normalize("NFD", line)  # normalize line using Unicode Normalization Form D
+    else:
+        """
+        TODO: add relevant unicodedata normalization here for other languages as needed
+        """
+        return line
+
+    return line
+
+
+def _strip_accents(line) -> str:
+    """
+    This currently is applied for `en` and `ja` language to strip out accents from text.
+    The given text 'line' should be normalized with the right unicode format prior to calling this method.
+    For example:
+        line = "Café élevàtor ôperàtor naïve Noël façade don't"
+            -> "Cafe elevator operator naive Noel facade don't"  if line was normalized with unicode format NFD
+            -> "Cafe elevator operator naïve Noël façade don't"  if line was NOT normalized with any unicode format
+    """
+
+    """Keep char whose category is NOT "Mn", i.e, 
+    "Mn" is category for Mark or Non-Spacing characters like diacritic/accent or non-spacing marks
+    Example of some diacritic/non-spacing marks: ^,´,` as they don't occupy space
+    """
+    output = [c for c in line if unicodedata.category(c) != "Mn"]  # decompose line into chars and diacritical marks
+    if len(output) == line:
+        return line
+    return "".join(output)
+
+
+def _replace_unicode_punct(line: str) -> str:
+    """
+    Replace unicode punctuations defined in `unicode_punct_dict'
+    """
+    return "".join(unicode_punct_dict.get(c, c) for c in line)
+
+
+def _remove_unicode_punct(line: str) -> str:
+    """
+    More aggressive _replace_unicode_punct
+    """
+    return _unicode_punct_re.sub("", line)
+
+
+def _remove_non_printing_char(line: str) -> str:
+    return _non_printing_chars_re.sub("", line)
+
+
+def cc_net_normalize(
+    line: str,
+    strip_accent: bool = True,
+    lower_case: bool = True,
+    digit_2_zero: bool = True,
+    punct_level: int = 1,
+    language: str = "en",
+) -> str:
+    line = line.strip()
+
+    if not line:
+        return line
+
+    line = unicode_normalization(line=line, language=language)
+
+    if lower_case:
+        line = line.lower()
+
+    if strip_accent:
+        line = _strip_accents(line)
+
+    if digit_2_zero:
+        # eg, "int 10 float 2.01 scientific 1.2e10" -> "int 00 float 0.00 scientific 0.0e00"
+        line = re.compile(r"\d").sub("0", line)
+
+    if punct_level == 1:
+        line = _replace_unicode_punct(line)
+
+    elif punct_level == 2:
+        line = _remove_unicode_punct(line)
+
+    line = _remove_non_printing_char(line)
+    return line
+
+
+if __name__ == "__main__":
+    line = "Int 10 float 2.01 scientific 1.2e10 Café ôperàtor"
+    new_line = cc_net_normalize(line)
+    print(f"== {line} -> {new_line}")

cluster_estimator.py

@@ -0,0 +1,59 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import math
+
+from data_processing.utils import GB
+
+
+"""
+This is a small code, that should be run locally to get the estimate of required cluster based on the amount 
+the documents to be processed and Ray worker nodes numbers of CPUs. These values can 
+be modified based on the job you need to run. The rest of the values are based on the experience of running
+exact dedup and should not be modified.
+"""
+
+# change these parameters for specific use case
+av_table_size = 189056510  # bytes
+number_of_docs = 4179209729
+
+# Number of workers
+n_workers = 1000
+# pod cpus
+ray_node_cpu = 15
+# cpu sizes fo actors
+worker_cpu = 0.75
+hash_cpu = 0.5
+
+print(f"suggested number of workers {n_workers}")
+
+n_hashes = math.ceil(number_of_docs * 32 / GB)
+required_hash_cpu = n_hashes * hash_cpu
+required_hash_mem = n_hashes * 2
+
+print(
+    f"Estimated Required hashes {n_hashes}, cpus for hashes {required_hash_cpu}, "
+    f"memory for hashes {required_hash_mem} GB"
+)
+
+execution_required_cpus = math.ceil((worker_cpu * n_workers + required_hash_cpu) / 0.85)
+
+print(f"required cpus for execution: {execution_required_cpus}")
+n_ray_nodes = math.ceil(execution_required_cpus / ray_node_cpu)
+print(f"Required ray nodes with {ray_node_cpu} cpus is {n_ray_nodes}")
+
+execution_required_memory = math.ceil((n_workers * av_table_size * 4 / GB + required_hash_cpu) / 0.6)
+print(f"required memory for execution: {execution_required_memory}")
+print(
+    f"Minimal required memory per ray pod is {math.ceil(execution_required_memory / n_ray_nodes)} GB. "
+    f"Practically use the larger number"
+)

code2parquet_local.py

@@ -0,0 +1,51 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import ast
+import os
+
+from code2parquet_transform import (  # domain_key,; snapshot_key,
+    CodeToParquetTransform,
+    data_factory_key,
+    detect_programming_lang_key,
+    supported_langs_file_key,
+)
+from data_processing.data_access import DataAccessFactory, DataAccessLocal
+
+
+supported_languages_file = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "../../ray/test-data/languages/lang_extensions.json")
+)
+input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input"))
+
+params = {
+    supported_langs_file_key: supported_languages_file,
+    detect_programming_lang_key: True,
+    # snapshot_key: "github",
+    # domain_key: "code",
+    "data_files_to_use": ast.literal_eval("['.zip']"),
+    data_factory_key: DataAccessFactory(),  # Expect to create DataAccessLocal
+}
+
+if __name__ == "__main__":
+    # Here we show how to run outside of ray
+    # Create and configure the transform.
+    # transform = CodeToParquetPythonTransform(params)
+    transform = CodeToParquetTransform(params)
+    # Use the local data access to read a parquet table.
+    data_access = DataAccessLocal()
+    file_to_process = os.path.join(input_folder, "application-java.zip")
+    byte_array, _ = data_access.get_file(file_to_process)
+    # Transform the table
+    files_list, metadata = transform.transform_binary(file_name=file_to_process, byte_array=byte_array)
+    print(f"Got {len(files_list)} output files")
+    print(f"output metadata : {metadata}")

code2parquet_local_python.py

@@ -0,0 +1,60 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import ast
+import os
+import sys
+
+from code2parquet_transform import (  # domain_key,; snapshot_key,
+    detect_programming_lang_cli_key,
+    supported_langs_file_cli_key,
+)
+from code2parquet_transform_python import CodeToParquetPythonConfiguration
+from data_processing.runtime.pure_python import PythonTransformLauncher
+from data_processing.utils import ParamsUtils
+
+
+# create parameters
+supported_languages_file = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "../test-data/languages/lang_extensions.json")
+)
+input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input"))
+output_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../output"))
+local_conf = {
+    "input_folder": input_folder,
+    "output_folder": output_folder,
+}
+worker_options = {"num_cpus": 0.8}
+code_location = {"github": "github", "commit_hash": "12345", "path": "path"}
+ingest_config = {
+    supported_langs_file_cli_key: supported_languages_file,
+    detect_programming_lang_cli_key: True,
+    # snapshot_key: "github",
+    # domain_key: "code",
+}
+
+params = {
+    # Data access. Only required parameters are specified
+    "data_local_config": ParamsUtils.convert_to_ast(local_conf),
+    "data_files_to_use": ast.literal_eval("['.zip']"),
+    # orchestrator
+    "runtime_pipeline_id": "pipeline_id",
+    "runtime_job_id": "job_id",
+    "runtime_code_location": ParamsUtils.convert_to_ast(code_location),
+}
+
+if __name__ == "__main__":
+    sys.argv = ParamsUtils.dict_to_req(d=(params | ingest_config))
+    # create launcher
+    launcher = PythonTransformLauncher(CodeToParquetPythonConfiguration())
+    # launch
+    launcher.launch()

code2parquet_local_ray.py

@@ -0,0 +1,63 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import ast
+import os
+import sys
+
+from code2parquet_transform import (
+    detect_programming_lang_cli_key,
+    supported_langs_file_cli_key,
+)
+from code2parquet_transform_ray import CodeToParquetRayConfiguration
+from data_processing.utils import GB, ParamsUtils
+from data_processing_ray.runtime.ray import RayTransformLauncher
+
+
+# create parameters
+supported_languages_file = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "../test-data/languages/lang_extensions.json")
+)
+input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input"))
+output_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../output"))
+local_conf = {
+    "input_folder": input_folder,
+    "output_folder": output_folder,
+}
+worker_options = {"num_cpus": 0.8, "memory": 2 * GB}
+code_location = {"github": "github", "commit_hash": "12345", "path": "path"}
+ingest_config = {
+    supported_langs_file_cli_key: supported_languages_file,
+    detect_programming_lang_cli_key: True,
+}
+
+params = {
+    # where to run
+    "run_locally": True,
+    # Data access. Only required parameters are specified
+    "data_local_config": ParamsUtils.convert_to_ast(local_conf),
+    "data_files_to_use": ast.literal_eval("['.zip']"),
+    # orchestrator
+    "runtime_worker_options": ParamsUtils.convert_to_ast(worker_options),
+    "runtime_num_workers": 3,
+    "runtime_pipeline_id": "pipeline_id",
+    "runtime_job_id": "job_id",
+    "runtime_creation_delay": 0,
+    "runtime_code_location": ParamsUtils.convert_to_ast(code_location),
+}
+
+if __name__ == "__main__":
+    sys.argv = ParamsUtils.dict_to_req(d=(params | ingest_config))
+    # create launcher
+    launcher = RayTransformLauncher(CodeToParquetRayConfiguration())
+    # launch
+    launcher.launch()

code2parquet_s3_python.py

@@ -0,0 +1,61 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import ast
+import sys
+
+from code2parquet_transform import (  # domain_key,; snapshot_key,
+    detect_programming_lang_cli_key,
+    supported_langs_file_cli_key,
+)
+from code2parquet_transform_python import CodeToParquetPythonConfiguration
+from data_processing.runtime.pure_python import PythonTransformLauncher
+from data_processing.utils import GB, ParamsUtils
+
+
+# create parameters
+s3_cred = {
+    "access_key": "localminioaccesskey",
+    "secret_key": "localminiosecretkey",
+    "url": "http://localhost:9000",
+}
+s3_conf = {
+    "input_folder": "test/ingest_2_parquet/input",
+    "output_folder": "test/ingest_2_parquet/output",
+}
+worker_options = {"num_cpus": 0.8, "memory": 2 * GB}
+code_location = {"github": "github", "commit_hash": "12345", "path": "path"}
+ingest_config = {
+    supported_langs_file_cli_key: "test/ingest_2_parquet/languages/lang_extensions.json",
+    detect_programming_lang_cli_key: True,
+    # snapshot_key: "github",
+    # domain_key: "code",
+    "code2parquet_s3_cred": ParamsUtils.convert_to_ast(s3_cred),
+}
+
+params = {
+    # Data access. Only required parameters are specified
+    "data_s3_cred": ParamsUtils.convert_to_ast(s3_cred),
+    "data_s3_config": ParamsUtils.convert_to_ast(s3_conf),
+    "data_files_to_use": ast.literal_eval("['.zip']"),
+    # orchestrator
+    "runtime_pipeline_id": "pipeline_id",
+    "runtime_job_id": "job_id",
+    "runtime_code_location": ParamsUtils.convert_to_ast(code_location),
+}
+
+if __name__ == "__main__":
+    sys.argv = ParamsUtils.dict_to_req(d=(params | ingest_config))
+    # create launcher
+    launcher = PythonTransformLauncher(CodeToParquetPythonConfiguration())
+    # launch
+    launcher.launch()

code2parquet_s3_ray.py

@@ -0,0 +1,64 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import ast
+import sys
+
+from code2parquet_transform import (
+    detect_programming_lang_cli_key,
+    supported_langs_file_cli_key,
+)
+from code2parquet_transform_ray import CodeToParquetRayConfiguration
+from data_processing.utils import GB, ParamsUtils
+from data_processing_ray.runtime.ray import RayTransformLauncher
+
+
+# create parameters
+s3_cred = {
+    "access_key": "localminioaccesskey",
+    "secret_key": "localminiosecretkey",
+    "url": "http://localhost:9000",
+}
+s3_conf = {
+    "input_folder": "test/ingest_2_parquet/input",
+    "output_folder": "test/ingest_2_parquet/output",
+}
+worker_options = {"num_cpus": 0.8, "memory": 2 * GB}
+code_location = {"github": "github", "commit_hash": "12345", "path": "path"}
+ingest_config = {
+    supported_langs_file_cli_key: "test/ingest_2_parquet/languages/lang_extensions.json",
+    detect_programming_lang_cli_key: True,
+    "code2parquet_s3_cred": ParamsUtils.convert_to_ast(s3_cred),
+}
+
+params = {
+    # where to run
+    "run_locally": True,
+    # Data access. Only required parameters are specified
+    "data_s3_cred": ParamsUtils.convert_to_ast(s3_cred),
+    "data_s3_config": ParamsUtils.convert_to_ast(s3_conf),
+    "data_files_to_use": ast.literal_eval("['.zip']"),
+    # orchestrator
+    "runtime_worker_options": ParamsUtils.convert_to_ast(worker_options),
+    "runtime_num_workers": 3,
+    "runtime_pipeline_id": "pipeline_id",
+    "runtime_job_id": "job_id",
+    "runtime_creation_delay": 0,
+    "runtime_code_location": ParamsUtils.convert_to_ast(code_location),
+}
+
+if __name__ == "__main__":
+    sys.argv = ParamsUtils.dict_to_req(d=(params | ingest_config))
+    # create launcher
+    launcher = RayTransformLauncher(CodeToParquetRayConfiguration())
+    # launch
+    launcher.launch()