data-prep-toolkit-transforms 0.2.1__py3-none-any.whl

cc_net_prepro.py

@@ -0,0 +1,168 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+"""
+@ Create on: 2023/04/25
+@ Description:
+    To incorporate preprocessing steps from cc_net/
+
+@ Reference:
+    https://github.com/facebookresearch/cc_net/blob/main/cc_net/text_normalizer.py
+    https://github.com/bigscience-workshop/data_tooling/blob/master/kenlm_training/cc_net/text_normalizer.py
+"""
+
+import re
+import unicodedata
+from typing import Dict
+
+
+unicode_punct_dict: Dict[str, str] = {
+    "，": ",",
+    "。": ".",
+    "、": ",",
+    "„": '"',
+    "”": '"',
+    "“": '"',
+    "«": '"',
+    "»": '"',
+    "１": '"',
+    "」": '"',
+    "「": '"',
+    "《": '"',
+    "》": '"',
+    "´": "'",
+    "∶": ":",
+    "：": ":",
+    "？": "?",
+    "！": "!",
+    "（": "(",
+    "）": ")",
+    "；": ";",
+    "–": "-",
+    "—": " - ",
+    "．": ". ",
+    "～": "~",
+    "’": "'",
+    "…": "...",
+    "━": "-",
+    "〈": "<",
+    "〉": ">",
+    "【": "[",
+    "】": "]",
+    "％": "%",
+    "►": "-",
+}
+
+_unicode_punct_re = re.compile(f"[{''.join(unicode_punct_dict.keys())}]")
+
+"""
+Generate regex pattern obj for later searching using re.search() or re.match()
+(r'[\x00\x01...\x9e\x9f]',re.UNICODE)
+"""
+_non_printing_chars_re = re.compile(f"[{''.join(map(chr, list(range(0, 32)) + list(range(127, 160))))}]")
+
+
+def unicode_normalization(line: str, language="en") -> str:
+    """
+    For some languages like 'ja' or 'en', its text should be normalized with the right unicode format
+    prior to calling tokenization by sentence piece tokenizer
+    """
+    if language == "ja":
+        line = unicodedata.normalize("NFKC", line)
+    elif language == "en":
+        line = unicodedata.normalize("NFD", line)  # normalize line using Unicode Normalization Form D
+    else:
+        """
+        TODO: add relevant unicodedata normalization here for other languages as needed
+        """
+        return line
+
+    return line
+
+
+def _strip_accents(line) -> str:
+    """
+    This currently is applied for `en` and `ja` language to strip out accents from text.
+    The given text 'line' should be normalized with the right unicode format prior to calling this method.
+    For example:
+        line = "Café élevàtor ôperàtor naïve Noël façade don't"
+            -> "Cafe elevator operator naive Noel facade don't"  if line was normalized with unicode format NFD
+            -> "Cafe elevator operator naïve Noël façade don't"  if line was NOT normalized with any unicode format
+    """
+
+    """Keep char whose category is NOT "Mn", i.e, 
+    "Mn" is category for Mark or Non-Spacing characters like diacritic/accent or non-spacing marks
+    Example of some diacritic/non-spacing marks: ^,´,` as they don't occupy space
+    """
+    output = [c for c in line if unicodedata.category(c) != "Mn"]  # decompose line into chars and diacritical marks
+    if len(output) == line:
+        return line
+    return "".join(output)
+
+
+def _replace_unicode_punct(line: str) -> str:
+    """
+    Replace unicode punctuations defined in `unicode_punct_dict'
+    """
+    return "".join(unicode_punct_dict.get(c, c) for c in line)
+
+
+def _remove_unicode_punct(line: str) -> str:
+    """
+    More aggressive _replace_unicode_punct
+    """
+    return _unicode_punct_re.sub("", line)
+
+
+def _remove_non_printing_char(line: str) -> str:
+    return _non_printing_chars_re.sub("", line)
+
+
+def cc_net_normalize(
+    line: str,
+    strip_accent: bool = True,
+    lower_case: bool = True,
+    digit_2_zero: bool = True,
+    punct_level: int = 1,
+    language: str = "en",
+) -> str:
+    line = line.strip()
+
+    if not line:
+        return line
+
+    line = unicode_normalization(line=line, language=language)
+
+    if lower_case:
+        line = line.lower()
+
+    if strip_accent:
+        line = _strip_accents(line)
+
+    if digit_2_zero:
+        # eg, "int 10 float 2.01 scientific 1.2e10" -> "int 00 float 0.00 scientific 0.0e00"
+        line = re.compile(r"\d").sub("0", line)
+
+    if punct_level == 1:
+        line = _replace_unicode_punct(line)
+
+    elif punct_level == 2:
+        line = _remove_unicode_punct(line)
+
+    line = _remove_non_printing_char(line)
+    return line
+
+
+if __name__ == "__main__":
+    line = "Int 10 float 2.01 scientific 1.2e10 Café ôperàtor"
+    new_line = cc_net_normalize(line)
+    print(f"== {line} -> {new_line}")

code2parquet_local.py

@@ -0,0 +1,51 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import ast
+import os
+
+from code2parquet_transform import (  # domain_key,; snapshot_key,
+    CodeToParquetTransform,
+    data_factory_key,
+    detect_programming_lang_key,
+    supported_langs_file_key,
+)
+from data_processing.data_access import DataAccessFactory, DataAccessLocal
+
+
+supported_languages_file = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "../../ray/test-data/languages/lang_extensions.json")
+)
+input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input"))
+
+params = {
+    supported_langs_file_key: supported_languages_file,
+    detect_programming_lang_key: True,
+    # snapshot_key: "github",
+    # domain_key: "code",
+    "data_files_to_use": ast.literal_eval("['.zip']"),
+    data_factory_key: DataAccessFactory(),  # Expect to create DataAccessLocal
+}
+
+if __name__ == "__main__":
+    # Here we show how to run outside of ray
+    # Create and configure the transform.
+    # transform = CodeToParquetPythonTransform(params)
+    transform = CodeToParquetTransform(params)
+    # Use the local data access to read a parquet table.
+    data_access = DataAccessLocal()
+    file_to_process = os.path.join(input_folder, "application-java.zip")
+    byte_array, _ = data_access.get_file(file_to_process)
+    # Transform the table
+    files_list, metadata = transform.transform_binary(file_name=file_to_process, byte_array=byte_array)
+    print(f"Got {len(files_list)} output files")
+    print(f"output metadata : {metadata}")

code2parquet_local_python.py

@@ -0,0 +1,60 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import ast
+import os
+import sys
+
+from code2parquet_transform import (  # domain_key,; snapshot_key,
+    detect_programming_lang_cli_key,
+    supported_langs_file_cli_key,
+)
+from code2parquet_transform_python import CodeToParquetPythonConfiguration
+from data_processing.runtime.pure_python import PythonTransformLauncher
+from data_processing.utils import ParamsUtils
+
+
+# create parameters
+supported_languages_file = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "../test-data/languages/lang_extensions.json")
+)
+input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input"))
+output_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../output"))
+local_conf = {
+    "input_folder": input_folder,
+    "output_folder": output_folder,
+}
+worker_options = {"num_cpus": 0.8}
+code_location = {"github": "github", "commit_hash": "12345", "path": "path"}
+ingest_config = {
+    supported_langs_file_cli_key: supported_languages_file,
+    detect_programming_lang_cli_key: True,
+    # snapshot_key: "github",
+    # domain_key: "code",
+}
+
+params = {
+    # Data access. Only required parameters are specified
+    "data_local_config": ParamsUtils.convert_to_ast(local_conf),
+    "data_files_to_use": ast.literal_eval("['.zip']"),
+    # orchestrator
+    "runtime_pipeline_id": "pipeline_id",
+    "runtime_job_id": "job_id",
+    "runtime_code_location": ParamsUtils.convert_to_ast(code_location),
+}
+
+if __name__ == "__main__":
+    sys.argv = ParamsUtils.dict_to_req(d=(params | ingest_config))
+    # create launcher
+    launcher = PythonTransformLauncher(CodeToParquetPythonConfiguration())
+    # launch
+    launcher.launch()

code2parquet_s3_python.py

@@ -0,0 +1,61 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import ast
+import sys
+
+from code2parquet_transform import (  # domain_key,; snapshot_key,
+    detect_programming_lang_cli_key,
+    supported_langs_file_cli_key,
+)
+from code2parquet_transform_python import CodeToParquetPythonConfiguration
+from data_processing.runtime.pure_python import PythonTransformLauncher
+from data_processing.utils import GB, ParamsUtils
+
+
+# create parameters
+s3_cred = {
+    "access_key": "localminioaccesskey",
+    "secret_key": "localminiosecretkey",
+    "url": "http://localhost:9000",
+}
+s3_conf = {
+    "input_folder": "test/ingest_2_parquet/input",
+    "output_folder": "test/ingest_2_parquet/output",
+}
+worker_options = {"num_cpus": 0.8, "memory": 2 * GB}
+code_location = {"github": "github", "commit_hash": "12345", "path": "path"}
+ingest_config = {
+    supported_langs_file_cli_key: "test/ingest_2_parquet/languages/lang_extensions.json",
+    detect_programming_lang_cli_key: True,
+    # snapshot_key: "github",
+    # domain_key: "code",
+    "code2parquet_s3_cred": ParamsUtils.convert_to_ast(s3_cred),
+}
+
+params = {
+    # Data access. Only required parameters are specified
+    "data_s3_cred": ParamsUtils.convert_to_ast(s3_cred),
+    "data_s3_config": ParamsUtils.convert_to_ast(s3_conf),
+    "data_files_to_use": ast.literal_eval("['.zip']"),
+    # orchestrator
+    "runtime_pipeline_id": "pipeline_id",
+    "runtime_job_id": "job_id",
+    "runtime_code_location": ParamsUtils.convert_to_ast(code_location),
+}
+
+if __name__ == "__main__":
+    sys.argv = ParamsUtils.dict_to_req(d=(params | ingest_config))
+    # create launcher
+    launcher = PythonTransformLauncher(CodeToParquetPythonConfiguration())
+    # launch
+    launcher.launch()

code2parquet_transform.py

@@ -0,0 +1,222 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import io
+import json
+import logging
+import os
+import uuid
+import zipfile
+from argparse import ArgumentParser, Namespace
+from datetime import datetime
+from typing import Any
+
+import pyarrow as pa
+from data_processing.data_access import DataAccess, DataAccessFactory
+from data_processing.transform import AbstractBinaryTransform, TransformConfiguration
+from data_processing.utils import CLIArgumentProvider, TransformUtils, str2bool
+
+
+shortname = "code2parquet"
+cli_prefix = f"{shortname}_"
+
+supported_langs_file_key = "supported_langs_file"
+supported_langs_file_cli_key = f"{cli_prefix}{supported_langs_file_key}"
+
+supported_languages_key = "supported_languages"
+supported_languages_cli_key = f"{cli_prefix}{supported_languages_key}"
+
+detect_programming_lang_key = "detect_programming_lang"
+detect_programming_lang_cli_key = f"{cli_prefix}{detect_programming_lang_key}"
+detect_programming_lang_default = True
+
+data_factory_key = "data_factory"
+
+domain_key = "domain"
+domain_cli_key = f"{cli_prefix}{domain_key}"
+snapshot_key = "snapshot"
+snapshot_cli_key = f"{cli_prefix}{snapshot_key}"
+
+
+def get_supported_languages(lang_file: str, data_access: DataAccess, logger: logging.Logger) -> dict[str, str]:
+    logger.debug(f"Getting supported languages from file {lang_file}")
+    json_data, _ = data_access.get_file(lang_file)
+    lang_dict = json.loads(json_data.decode("utf-8"))
+    reversed_dict = {ext: langs for langs, exts in lang_dict.items() for ext in exts}
+    logger.debug(f"Supported languages {reversed_dict}")
+    return reversed_dict
+
+
+class CodeToParquetTransform(AbstractBinaryTransform):
+    def __init__(self, config: dict):
+        """
+
+        Args:
+            config: dictionary of configuration data
+                supported_langs - dictionary of file extenstions to language names.
+                supported_langs_file - if supported_langs, is not provided, then read a map
+                    of language names keyed to a list of extensions, from this json file.  The file is read using
+                    the DataAccessFactory, under the code2parquet_data_factory key.
+        """
+        from data_processing.utils import get_logger
+
+        self.logger = get_logger(__name__)
+        super().__init__(config)
+        self.languages_supported = config.get(supported_languages_key, None)
+        if self.languages_supported is None:
+            path = config.get(supported_langs_file_key, None)
+            if path is not None:
+                daf = config.get(data_factory_key, None)
+                if daf is None:
+                    raise ValueError(f"Neither {supported_languages_key} nor {data_factory_key} were provided.")
+                data_access = daf.create_data_access()
+                self.languages_supported = get_supported_languages(
+                    lang_file=path, data_access=data_access, logger=self.logger
+                )
+            self.detect_programming_lang = config.get(detect_programming_lang_key, detect_programming_lang_default)
+            if self.detect_programming_lang and self.languages_supported is None:
+                raise RuntimeError(
+                    "Programming language detection requested without providing a mapping of extensions to languages"
+                )
+        domain = config.get(domain_key, None)
+        snapshot = config.get(domain_key, None)
+        self.shared_columns = {}
+        if domain is not None:
+            self.shared_columns["domain"] = domain
+        if snapshot is not None:
+            self.shared_columns["snapshot"] = snapshot
+
+    def _get_lang_from_ext(self, ext):
+        lang = "unknown"
+        if ext is not None:
+            lang = self.languages_supported.get(ext, lang)
+        return lang
+
+    def transform_binary(self, file_name: str, byte_array: bytes) -> tuple[list[tuple[bytes, str]], dict[str, Any]]:
+        """
+        Converts raw data file (ZIP) to Parquet format
+        """
+        # We currently only process .zip files
+        if TransformUtils.get_file_extension(file_name)[1] != ".zip":
+            self.logger.warning(f"Got unsupported file type {file_name}, skipping")
+            return [], {}
+        data = []
+        number_of_rows = 0
+        with zipfile.ZipFile(io.BytesIO(bytes(byte_array))) as opened_zip:
+            # Loop through each file member in the ZIP archive
+            for member in opened_zip.infolist():
+                if not member.is_dir():
+                    with opened_zip.open(member) as file:
+                        try:
+                            # Read the content of the file
+                            content_bytes = file.read()
+                            # Decode the content
+                            content_string = TransformUtils.decode_content(content_bytes)
+                            if content_string and len(content_string) > 0:
+                                ext = TransformUtils.get_file_extension(member.filename)[1]
+                                row_data = {
+                                    "title": member.filename,
+                                    "document": TransformUtils.get_file_basename(file_name),
+                                    "contents": content_string,
+                                    "document_id": str(uuid.uuid4()),
+                                    "ext": ext,
+                                    "hash": TransformUtils.str_to_hash(content_string),
+                                    "size": len(content_string),
+                                    "date_acquired": datetime.now().isoformat(),
+                                    "repo_name": os.path.splitext(os.path.basename(file_name))[0],
+                                } | self.shared_columns
+                                if self.detect_programming_lang:
+                                    lang = self._get_lang_from_ext(ext)
+                                    row_data["programming_language"] = lang  # TODO column name should be configurable
+                                data.append(row_data)
+                                number_of_rows += 1
+                            else:
+                                self.logger.warning(
+                                    f"file {member.filename} is empty. content {content_string}, skipping"
+                                )
+                        except Exception as e:
+                            self.logger.warning(f"Exception {str(e)} processing file {member.filename}, skipping")
+        table = pa.Table.from_pylist(data)
+        return [(TransformUtils.convert_arrow_to_binary(table=table), ".parquet")], {"number of rows": number_of_rows}
+
+
+class CodeToParquetTransformConfiguration(TransformConfiguration):
+    """
+    Provides support for configuring and using the associated Transform class include
+    configuration with CLI args and combining of metadata.
+    """
+
+    def __init__(self, transform_class: type[AbstractBinaryTransform] = CodeToParquetTransform):
+        super().__init__(
+            name=shortname,
+            transform_class=transform_class,
+            remove_from_metadata=[data_factory_key],
+        )
+        from data_processing.utils import get_logger
+
+        self.logger = get_logger(__name__)
+        self.daf = None
+
+    def add_input_params(self, parser: ArgumentParser) -> None:
+        """
+        Add Transform-specific arguments to the given parser.
+        This will be included in a dictionary used to initialize the ProgLangMatchTransform.
+        By convention a common prefix should be used for all mutator-specific CLI args
+        (e.g, noop_, pii_, etc.)
+        """
+        parser.add_argument(
+            f"--{cli_prefix}{supported_langs_file_key}",
+            type=str,
+            default=None,
+            help="Path to file containing the list of supported languages",
+        )
+        parser.add_argument(
+            f"--{cli_prefix}{detect_programming_lang_key}",
+            type=lambda x: bool(str2bool(x)),
+            default=detect_programming_lang_default,
+            help="Infer the programming lang from the file extension using the file of supported languages",
+        )
+        parser.add_argument(
+            f"--{snapshot_cli_key}", type=str, help="Snapshot value assigned to all imported documents.", default=None
+        )
+        parser.add_argument(
+            f"--{domain_cli_key}",
+            type=str,
+            help="Domain value assigned to all imported documents.",
+            default=None,
+        )
+        # Create the DataAccessFactor to use CLI args
+        self.daf = DataAccessFactory(cli_prefix, False)
+        # Add the DataAccessFactory parameters to the transform's configuration parameters.
+        self.daf.add_input_params(parser)
+
+    def apply_input_params(self, args: Namespace) -> bool:
+        """
+        Validate and apply the arguments that have been parsed
+        :param args: user defined arguments.
+        :return: True, if validate pass or False otherwise
+        """
+        captured = CLIArgumentProvider.capture_parameters(args, cli_prefix, False)
+        if captured.get(supported_langs_file_key, None) is None:
+            self.logger.warning(f"{supported_langs_file_key} is required, but got None")
+            return False
+        self.params = captured | {
+            # detect_programming_lang_key: captured.get(detect_programming_lang_key, None),
+            # supported_langs_file_key: captured.get(supported_langs_file_key, ""),
+            #            domain_key: dargs.get(domain_key, ""),
+            # snapshot_key: dargs.get(snapshot_key, ""),
+            data_factory_key: self.daf,
+        }
+        # self.logger.info(f"Transform configuration {self.params}") # Uhmm, let's NOT print out S3 keys please!
+
+        # Validate and populate the transform's DataAccessFactory
+        return self.daf.apply_input_params(args)

code2parquet_transform_python.py

@@ -0,0 +1,39 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from code2parquet_transform import (
+    CodeToParquetTransform,
+    CodeToParquetTransformConfiguration,
+    data_factory_key,
+    get_supported_languages,
+    supported_langs_file_key,
+)
+from data_processing.runtime.pure_python import PythonTransformLauncher
+from data_processing.runtime.pure_python.runtime_configuration import (
+    PythonTransformRuntimeConfiguration,
+)
+from data_processing.utils import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class CodeToParquetPythonConfiguration(PythonTransformRuntimeConfiguration):
+    def __init__(self):
+        super().__init__(transform_config=CodeToParquetTransformConfiguration(transform_class=CodeToParquetTransform))
+
+
+if __name__ == "__main__":
+    # launcher = NOOPRayLauncher()
+    launcher = PythonTransformLauncher(CodeToParquetPythonConfiguration())
+    logger.info("Launching noop transform")
+    launcher.launch()

code_quality_local.py

@@ -0,0 +1,37 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import os
+
+from code_quality_transform import CodeQualityTransform
+from data_processing.data_access import DataAccessLocal
+
+
+input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input"))
+
+if __name__ == "__main__":
+    codequality_params = {
+        "code_quality_params": {
+            "contents_column_name": "contents",
+            "language_column_name": "language",
+            "tokenizer": "codeparrot/codeparrot",
+            "hf_token": None,
+        }
+    }
+    transform = CodeQualityTransform(codequality_params)
+
+    data_access = DataAccessLocal()
+    table, _ = data_access.get_table(os.path.join(input_folder, "sample_1.parquet"))
+    print(f"input table: {table}")
+    # Transform the table
+    table_list, metadata = transform.transform(table)
+    print(f"\noutput table: {table_list}")