data-prep-toolkit-transforms 0.2.1__tar.gz → 0.2.1.dev0__tar.gz

data_prep_toolkit_transforms-0.2.1.dev0/PKG-INFO

@@ -0,0 +1,33 @@
+Metadata-Version: 2.1
+Name: data_prep_toolkit_transforms
+Version: 0.2.1.dev0
+Summary: Data Preparation Toolkit Transforms
+Author-email: Maroun Touma <touma@us.ibm.com>
+License: Apache-2.0
+Keywords: transforms,data preprocessing,data preparation,llm,generative,ai,fine-tuning,llmapps
+Requires-Python: <3.12,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: data-prep-toolkit==0.2.1.dev0
+Requires-Dist: argparse
+Requires-Dist: boto3==1.34.69
+Requires-Dist: bs4==0.0.2
+Requires-Dist: clamd==1.0.2
+Requires-Dist: docling[ocr]==1.1.2
+Requires-Dist: duckdb==0.10.1
+Requires-Dist: fasttext==0.9.2
+Requires-Dist: filetype<2.0.0,>=1.2.0
+Requires-Dist: huggingface-hub<1.0.0,>=0.21.4
+Requires-Dist: langcodes==3.3.0
+Requires-Dist: mmh3==4.1.0
+Requires-Dist: numpy==1.26.4
+Requires-Dist: pandas
+Requires-Dist: parameterized
+Requires-Dist: pyarrow==16.1.0
+Requires-Dist: python-dateutil>=2.8.2
+Requires-Dist: pytz>=2020.1
+Requires-Dist: quackling==0.1.0
+Requires-Dist: scancode-toolkit==32.1.0; platform_system != "Darwin"
+Requires-Dist: sentence-transformers==3.0.1
+Requires-Dist: transformers==4.38.2
+Requires-Dist: tzdata>=2022.7
+Requires-Dist: xxhash==3.4.1

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit_transforms"
-version = "0.2.1"
+version = "0.2.1.dev0"
 requires-python = ">=3.10,<3.12"
 keywords = ["transforms", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 description = "Data Preparation Toolkit Transforms"
@@ -9,14 +9,38 @@ readme = {file = "README.md", content-type = "text/markdown"}
 authors = [
     { name = "Maroun Touma", email = "touma@us.ibm.com" },
 ]
-dynamic = ["dependencies"]
+
+dependencies = [
+	"data-prep-toolkit==0.2.1.dev0",
+	"argparse",
+	"boto3==1.34.69",
+	"bs4==0.0.2",
+	"clamd==1.0.2",
+	"docling[ocr]==1.1.2",
+	"duckdb==0.10.1",
+	"fasttext==0.9.2",
+	"filetype >=1.2.0, <2.0.0",
+	"huggingface-hub >= 0.21.4, <1.0.0",
+	"langcodes==3.3.0",
+	"mmh3==4.1.0",
+	"numpy==1.26.4",
+	"pandas",
+	"parameterized",
+	"pyarrow==16.1.0",
+	"python-dateutil>=2.8.2",
+	"pytz>=2020.1",
+	"quackling==0.1.0",
+	"scancode-toolkit==32.1.0 ; platform_system != 'Darwin'",
+	"sentence-transformers==3.0.1",
+	"transformers==4.38.2",
+	"tzdata>=2022.7",
+	"xxhash==3.4.1",
+]
 
 [build-system]
 requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
 build-backend = "setuptools.build_meta"
 
-[tool.setuptools.dynamic]
-dependencies = {file = ["requirements.txt"]}
 
 [options]
 package_dir = ["src"]

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev0}/src/code2parquet_transform.py

@@ -13,16 +13,20 @@
 import io
 import json
 import logging
-import os
 import uuid
 import zipfile
 from argparse import ArgumentParser, Namespace
 from datetime import datetime
 from typing import Any
+import os
 
 import pyarrow as pa
 from data_processing.data_access import DataAccess, DataAccessFactory
-from data_processing.transform import AbstractBinaryTransform, TransformConfiguration
+from data_processing.transform import (
+    AbstractBinaryTransform,
+    AbstractTransform,
+    TransformConfiguration,
+)
 from data_processing.utils import CLIArgumentProvider, TransformUtils, str2bool
 
 
@@ -132,7 +136,7 @@ class CodeToParquetTransform(AbstractBinaryTransform):
                                     "hash": TransformUtils.str_to_hash(content_string),
                                     "size": len(content_string),
                                     "date_acquired": datetime.now().isoformat(),
-                                    "repo_name": os.path.splitext(os.path.basename(file_name))[0],
+                                    "repo_name":os.path.splitext(os.path.basename(file_name))[0]
                                 } | self.shared_columns
                                 if self.detect_programming_lang:
                                     lang = self._get_lang_from_ext(ext)
@@ -155,7 +159,7 @@ class CodeToParquetTransformConfiguration(TransformConfiguration):
     configuration with CLI args and combining of metadata.
     """
 
-    def __init__(self, transform_class: type[AbstractBinaryTransform] = CodeToParquetTransform):
+    def __init__(self, transform_class: type[AbstractTransform] = CodeToParquetTransform):
         super().__init__(
             name=shortname,
             transform_class=transform_class,

data_prep_toolkit_transforms-0.2.1.dev0/src/data_prep_toolkit_transforms.egg-info/PKG-INFO

@@ -0,0 +1,33 @@
+Metadata-Version: 2.1
+Name: data_prep_toolkit_transforms
+Version: 0.2.1.dev0
+Summary: Data Preparation Toolkit Transforms
+Author-email: Maroun Touma <touma@us.ibm.com>
+License: Apache-2.0
+Keywords: transforms,data preprocessing,data preparation,llm,generative,ai,fine-tuning,llmapps
+Requires-Python: <3.12,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: data-prep-toolkit==0.2.1.dev0
+Requires-Dist: argparse
+Requires-Dist: boto3==1.34.69
+Requires-Dist: bs4==0.0.2
+Requires-Dist: clamd==1.0.2
+Requires-Dist: docling[ocr]==1.1.2
+Requires-Dist: duckdb==0.10.1
+Requires-Dist: fasttext==0.9.2
+Requires-Dist: filetype<2.0.0,>=1.2.0
+Requires-Dist: huggingface-hub<1.0.0,>=0.21.4
+Requires-Dist: langcodes==3.3.0
+Requires-Dist: mmh3==4.1.0
+Requires-Dist: numpy==1.26.4
+Requires-Dist: pandas
+Requires-Dist: parameterized
+Requires-Dist: pyarrow==16.1.0
+Requires-Dist: python-dateutil>=2.8.2
+Requires-Dist: pytz>=2020.1
+Requires-Dist: quackling==0.1.0
+Requires-Dist: scancode-toolkit==32.1.0; platform_system != "Darwin"
+Requires-Dist: sentence-transformers==3.0.1
+Requires-Dist: transformers==4.38.2
+Requires-Dist: tzdata>=2022.7
+Requires-Dist: xxhash==3.4.1

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev0}/src/data_prep_toolkit_transforms.egg-info/SOURCES.txt

@@ -1,7 +1,5 @@
-Makefile
-README.md
 pyproject.toml
-requirements.txt
+src/__init__.py
 src/cc_net_prepro.py
 src/code2parquet_local.py
 src/code2parquet_local_python.py
@@ -19,10 +17,6 @@ src/doc_chunk_local.py
 src/doc_chunk_local_python.py
 src/doc_chunk_transform.py
 src/doc_chunk_transform_python.py
-src/doc_id_local.py
-src/doc_id_local_python.py
-src/doc_id_transform_base.py
-src/doc_id_transform_python.py
 src/doc_quality_local.py
 src/doc_quality_local_python.py
 src/doc_quality_transform.py
@@ -30,7 +24,6 @@ src/doc_quality_transform_python.py
 src/doc_quality_utils.py
 src/ededup_local.py
 src/ededup_local_python.py
-src/ededup_local_python_incremental.py
 src/ededup_transform_base.py
 src/ededup_transform_python.py
 src/filter_local.py
@@ -38,7 +31,6 @@ src/filter_local_python.py
 src/filter_test_support.py
 src/filter_transform.py
 src/filter_transform_python.py
-src/flair_recognizer.py
 src/header_cleanser_local.py
 src/header_cleanser_local_python.py
 src/header_cleanser_test_support.py
@@ -54,12 +46,6 @@ src/pdf2parquet_local.py
 src/pdf2parquet_local_python.py
 src/pdf2parquet_transform.py
 src/pdf2parquet_transform_python.py
-src/pii_analyzer.py
-src/pii_anonymizer.py
-src/pii_redactor_local.py
-src/pii_redactor_local_python.py
-src/pii_redactor_transform.py
-src/pii_redactor_transform_python.py
 src/proglang_select_local.py
 src/proglang_select_local_python.py
 src/proglang_select_transform.py

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev0}/src/data_prep_toolkit_transforms.egg-info/requires.txt

@@ -1,26 +1,26 @@
-data-prep-toolkit>=0.2.1
+data-prep-toolkit==0.2.1.dev0
+argparse
+boto3==1.34.69
 bs4==0.0.2
-docling-ibm-models==1.1.7
-deepsearch-glm==0.21.0
-docling==1.11.0
-filetype<2.0.0,>=1.2.0
-docling-core==1.3.0
-llama-index-core<0.12.0,>=0.11.0
+clamd==1.0.2
+docling[ocr]==1.1.2
 duckdb==0.10.1
 fasttext==0.9.2
+filetype<2.0.0,>=1.2.0
 huggingface-hub<1.0.0,>=0.21.4
 langcodes==3.3.0
 mmh3==4.1.0
 numpy==1.26.4
 pandas
 parameterized
+pyarrow==16.1.0
+python-dateutil>=2.8.2
+pytz>=2020.1
+quackling==0.1.0
 sentence-transformers==3.0.1
 transformers==4.38.2
+tzdata>=2022.7
 xxhash==3.4.1
-presidio-analyzer>=2.2.355
-presidio-anonymizer>=2.2.355
-flair>=0.14.0
-pandas>=2.2.2
 
 [:platform_system != "Darwin"]
 scancode-toolkit==32.1.0

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev0}/src/data_prep_toolkit_transforms.egg-info/top_level.txt

@@ -1,3 +1,4 @@
+__init__
 cc_net_prepro
 code2parquet_local
 code2parquet_local_python
@@ -15,10 +16,6 @@ doc_chunk_local
 doc_chunk_local_python
 doc_chunk_transform
 doc_chunk_transform_python
-doc_id_local
-doc_id_local_python
-doc_id_transform_base
-doc_id_transform_python
 doc_quality_local
 doc_quality_local_python
 doc_quality_transform
@@ -26,7 +23,6 @@ doc_quality_transform_python
 doc_quality_utils
 ededup_local
 ededup_local_python
-ededup_local_python_incremental
 ededup_transform_base
 ededup_transform_python
 filter_local
@@ -34,7 +30,6 @@ filter_local_python
 filter_test_support
 filter_transform
 filter_transform_python
-flair_recognizer
 header_cleanser_local
 header_cleanser_local_python
 header_cleanser_test_support
@@ -50,12 +45,6 @@ pdf2parquet_local
 pdf2parquet_local_python
 pdf2parquet_transform
 pdf2parquet_transform_python
-pii_analyzer
-pii_anonymizer
-pii_redactor_local
-pii_redactor_local_python
-pii_redactor_transform
-pii_redactor_transform_python
 proglang_select_local
 proglang_select_local_python
 proglang_select_transform

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev0}/src/doc_Gopher_statistics.py

@@ -49,9 +49,7 @@ def compute_word_statistics(text: str, symbols: list = ["#", "..."]) -> tuple[in
     return total_words, mean_word_len, symbol_to_word_ratio
 
 
-def compute_bullet_point_ellipsis_alphabet_word_ratio(
-    text: str, bullets: list = ["-", "*"]
-) -> tuple[float, float, float]:
+def compute_bullet_point_ellipsis_alphabet_word_ratio(text: str, bullets: list = ["-", "*"]) -> tuple[float, float, float]:
     """
     Given a text document:
         - Compute the ratio of lines starting with a bullet point (should be <=90%)

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev0}/src/doc_chunk_chunkers.py

@@ -10,13 +10,14 @@
 # limitations under the License.
 ################################################################################
 
+import math
 from abc import ABCMeta, abstractmethod
-from typing import Iterator, Optional
+from typing import Iterator
 
 from docling_core.types import Document as DLDocument
+from quackling.core.chunkers.hierarchical_chunker import HierarchicalChunker
 from llama_index.core import Document as LIDocument
 from llama_index.core.node_parser import MarkdownNodeParser
-from docling_core.transforms.chunker import HierarchicalChunker
 
 
 class ChunkingExecutor(metaclass=ABCMeta):
@@ -24,25 +25,13 @@ class ChunkingExecutor(metaclass=ABCMeta):
     def chunk(self, content: str) -> Iterator[dict]:
         raise NotImplemented("The chunk() method must be implemented")
 
-
 class DLJsonChunker(ChunkingExecutor):
-    def __init__(
-        self,
-        min_chunk_len: Optional[int],
-        output_chunk_column_name: str,
-        output_jsonpath_column_name: str,
-        output_pageno_column_name_key: str,
-        output_bbox_column_name_key: str,
-    ):
+    def __init__(self, output_chunk_column_name: str, output_jsonpath_column_name: str, output_pageno_column_name_key: str, output_bbox_column_name_key: str):
         self.output_chunk_column_name = output_chunk_column_name
         self.output_jsonpath_column_name = output_jsonpath_column_name
         self.output_pageno_column_name_key = output_pageno_column_name_key
         self.output_bbox_column_name_key = output_bbox_column_name_key
-
-        chunker_kwargs = dict(include_metadata=True)
-        if min_chunk_len is not None:
-            chunker_kwargs["min_chunk_len"] = min_chunk_len
-        self._chunker = HierarchicalChunker(**chunker_kwargs)
+        self._chunker = HierarchicalChunker(include_metadata=True)
 
     def chunk(self, content: str) -> Iterator[dict]:
         doc = DLDocument.model_validate_json(content)
@@ -54,7 +43,6 @@ class DLJsonChunker(ChunkingExecutor):
                 self.output_bbox_column_name_key: chunk.bbox,
             }
 
-
 class LIMarkdown(ChunkingExecutor):
     def __init__(self, output_chunk_column_name: str):
         self.output_chunk_column_name = output_chunk_column_name
@@ -66,3 +54,4 @@ class LIMarkdown(ChunkingExecutor):
             yield {
                 self.output_chunk_column_name: node.text,
             }
+

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev0}/src/doc_chunk_transform.py

@@ -24,20 +24,14 @@ from doc_chunk_chunkers import ChunkingExecutor, DLJsonChunker, LIMarkdown
 short_name = "doc_chunk"
 cli_prefix = f"{short_name}_"
 content_column_name_key = "content_column_name"
-doc_id_column_name_key = "doc_id_column_name"
 chunking_type_key = "chunking_type"
-dl_min_chunk_len_key = "dl_min_chunk_len"
 output_chunk_column_name_key = "output_chunk_column_name"
-output_source_doc_id_column_name_key = "output_source_doc_id_column_name"
 output_jsonpath_column_name_key = "output_jsonpath_column_name"
 output_pageno_column_name_key = "output_pageno_column_name"
 output_bbox_column_name_key = "output_bbox_column_name"
 content_column_name_cli_param = f"{cli_prefix}{content_column_name_key}"
-doc_id_column_name_cli_param = f"{cli_prefix}{doc_id_column_name_key}"
 chunking_type_cli_param = f"{cli_prefix}{chunking_type_key}"
-dl_min_chunk_len_cli_param = f"{cli_prefix}{dl_min_chunk_len_key}"
 output_chunk_column_name_cli_param = f"{cli_prefix}{output_chunk_column_name_key}"
-output_source_doc_id_column_name_cli_param = f"{cli_prefix}{output_source_doc_id_column_name_key}"
 output_jsonpath_column_name_cli_param = f"{cli_prefix}{output_jsonpath_column_name_key}"
 output_pageno_column_name_cli_param = f"{cli_prefix}{output_pageno_column_name_key}"
 output_bbox_column_name_cli_param = f"{cli_prefix}{output_bbox_column_name_key}"
@@ -52,11 +46,8 @@ class chunking_types(str, enum.Enum):
 
 
 default_content_column_name = "contents"
-default_doc_id_column_name = "document_id"
 default_chunking_type = chunking_types.DL_JSON
-default_dl_min_chunk_len = None
 default_output_chunk_column_name = "contents"
-default_output_source_doc_id_column_name = "source_document_id"
 default_output_jsonpath_column_name = "doc_jsonpath"
 default_output_pageno_column_name = "page_number"
 default_output_bbox_column_name = "bbox"
@@ -82,12 +73,9 @@ class DocChunkTransform(AbstractTableTransform):
         self.chunking_type = config.get(chunking_type_key, default_chunking_type)
 
         self.content_column_name = config.get(content_column_name_key, default_content_column_name)
-        self.doc_id_column_name = config.get(doc_id_column_name_key, default_doc_id_column_name)
         self.output_chunk_column_name = config.get(output_chunk_column_name_key, default_output_chunk_column_name)
-        self.output_source_doc_id_column_name = config.get(output_source_doc_id_column_name_key, default_output_source_doc_id_column_name)
 
         # Parameters for Docling JSON chunking
-        self.dl_min_chunk_len = config.get(dl_min_chunk_len_key, default_dl_min_chunk_len)
         self.output_jsonpath_column_name = config.get(
             output_jsonpath_column_name_key, default_output_jsonpath_column_name
         )
@@ -101,7 +89,6 @@ class DocChunkTransform(AbstractTableTransform):
         self.chunker: ChunkingExecutor
         if self.chunking_type == chunking_types.DL_JSON:
             self.chunker = DLJsonChunker(
-                min_chunk_len=self.dl_min_chunk_len,
                 output_chunk_column_name=self.output_chunk_column_name,
                 output_jsonpath_column_name=self.output_jsonpath_column_name,
                 output_pageno_column_name_key=self.output_pageno_column_name_key,
@@ -125,11 +112,8 @@ class DocChunkTransform(AbstractTableTransform):
         for batch in table.to_batches():
             for row in batch.to_pylist():
                 content: str = row[self.content_column_name]
-                new_row = {k: v for k, v in row.items() if k not in (self.content_column_name, self.doc_id_column_name)}
-                if self.doc_id_column_name in row:
-                    new_row[self.output_source_doc_id_column_name] = row[self.doc_id_column_name]
+                new_row = {k: v for k, v in row.items() if k not in (self.content_column_name,)}
                 for chunk in self.chunker.chunk(content):
-                    chunk[self.doc_id_column_name] = TransformUtils.str_to_hash(chunk[self.output_chunk_column_name])
                     data.append(
                         {
                             **new_row,
@@ -178,26 +162,11 @@ class DocChunkTransformConfiguration(TransformConfiguration):
             default=default_content_column_name,
             help="Name of the column containing the text to be chunked",
         )
-        parser.add_argument(
-            f"--{doc_id_column_name_cli_param}",
-            default=default_doc_id_column_name,
-            help="Name of the column containing the doc_id to be propagated in the output",
-        )
-        parser.add_argument(
-            f"--{dl_min_chunk_len_cli_param}",
-            default=default_dl_min_chunk_len,
-            help="Minimum number of characters for the chunk in the dl_json chunker. Setting to None is using the library defaults, i.e. a min_chunk_len=64.",
-        )
         parser.add_argument(
             f"--{output_chunk_column_name_cli_param}",
             default=default_output_chunk_column_name,
             help="Column name to store the chunks",
         )
-        parser.add_argument(
-            f"--{output_source_doc_id_column_name_cli_param}",
-            default=default_output_source_doc_id_column_name,
-            help="Column name to store the `document_id` from the input table",
-        )
         parser.add_argument(
             f"--{output_jsonpath_column_name_cli_param}",
             default=default_output_jsonpath_column_name,

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev0}/src/doc_quality_local_python.py

@@ -16,13 +16,12 @@ import sys
 from data_processing.runtime.pure_python import PythonTransformLauncher
 from data_processing.utils import ParamsUtils
 from doc_quality_transform import (
-    bad_word_filepath_cli_param,
-    doc_content_column_cli_param,
     text_lang_cli_param,
+    doc_content_column_cli_param,
+    bad_word_filepath_cli_param,
 )
 from doc_quality_transform_python import DocQualityPythonTransformConfiguration
 
-
 # create parameters
 input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "test-data", "input"))
 output_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "output"))
@@ -32,7 +31,7 @@ local_conf = {
 }
 code_location = {"github": "github", "commit_hash": "12345", "path": "path"}
 basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
-model_path = os.path.join(basedir, "models")
+model_path=os.path.join(basedir, "models")
 if not os.path.exists(model_path):
     model_path = os.path.abspath(os.path.join(basedir, "..", "models"))
 

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev0}/src/doc_quality_transform.py

@@ -10,12 +10,12 @@
 # limitations under the License.
 ################################################################################
 
-import os
 from argparse import ArgumentParser, Namespace
 from typing import Any
 
+import os
 import pyarrow as pa
-from data_processing.data_access import DataAccess, DataAccessFactory
+from data_processing.data_access import DataAccessFactory, DataAccess
 from data_processing.transform import AbstractTableTransform, TransformConfiguration
 from data_processing.utils import CLIArgumentProvider, TransformUtils, get_logger
 from doc_c4_statistics import (
@@ -32,7 +32,6 @@ from doc_Gopher_statistics import (
     find_first_japanese_alphabet_position,
 )
 
-
 logger = get_logger(__name__)
 
 short_name = "docq"
@@ -50,7 +49,6 @@ default_doc_content_column = "contents"
 data_factory_internal_key = f"{cli_prefix}data_factory"
 files_to_use_internal_key = f"{cli_prefix}files_to_use"
 
-
 class DocQualityTransform(AbstractTableTransform):
     """
     Implements a transform to calculate document quality.
@@ -67,7 +65,7 @@ class DocQualityTransform(AbstractTableTransform):
         super().__init__(config)
         self.text_lang = config.get(text_lang_key, default_text_lang)
         self.doc_content_column = config.get(doc_content_column_key, default_doc_content_column)
-
+        
         daf = config.get(data_factory_internal_key, None)
         bad_word_filepath = config.get(bad_word_filepath_key, None)
         if bad_word_filepath is not None:
@@ -75,14 +73,11 @@ class DocQualityTransform(AbstractTableTransform):
                 logger.info(f"Load badwords found locally from {bad_word_filepath}")
                 self.re_pattern = c4_load_ldnoobw_words(ft_lang=self.text_lang, file_path=bad_word_filepath)
             else:
-                if daf is None:
-                    raise RuntimeError(
-                        f"Did not find DataAccessFactory instance under {data_factory_internal_key} key. This is required when bad word file is not in the local file system."
-                    )
+                if daf is None: 
+                    raise RuntimeError(f"Did not find DataAccessFactory instance under {data_factory_internal_key} key. This is required when bad word file is not in the local file system.")
                 logger.info(f"Load badwords from remote")
                 data_access = daf.create_data_access()
                 import tempfile
-
                 with tempfile.TemporaryDirectory() as temp_dir:
                     # use a temporary directory until model is loaded to memory
                     bad_word_filepath = self._write_locally(data_access, bad_word_filepath, temp_dir)
@@ -92,7 +87,7 @@ class DocQualityTransform(AbstractTableTransform):
         filename = os.path.basename(path)
         content, _ = data_access.get_file(path)
         temp_file_path = os.path.join(temp_dir, filename)
-        with open(temp_file_path, "wb") as temp_file:
+        with open(temp_file_path, 'wb') as temp_file:
             temp_file.write(content)
         return temp_file_path
 
@@ -190,7 +185,6 @@ class DocQualityTransformConfiguration(TransformConfiguration):
     Provides support for configuring and using the associated Transform class include
     configuration with CLI args.
     """
-
     def __init__(self):
         super().__init__(
             name=short_name,
@@ -207,7 +201,9 @@ class DocQualityTransformConfiguration(TransformConfiguration):
         (e.g, noop_, pii_, etc.)
         """
         parser.add_argument(
-            f"--{text_lang_cli_param}", default=default_text_lang, help="language used in the text content"
+            f"--{text_lang_cli_param}",
+            default=default_text_lang,
+            help="language used in the text content"
         )
         parser.add_argument(
             f"--{doc_content_column_cli_param}",
@@ -229,13 +225,9 @@ class DocQualityTransformConfiguration(TransformConfiguration):
         :return: True, if validate pass or False otherwise
         """
         captured = CLIArgumentProvider.capture_parameters(args, cli_prefix, False)
-        self.params = (
-            self.params
-            | captured
-            | {
-                data_factory_internal_key: self.daf,
-            }
-        )
+        self.params = self.params | captured | {
+            data_factory_internal_key: self.daf,
+        }
         logger.info(f"doc_quality parameters are : {self.params}")
         # Validate and populate the transform's DataAccessFactory
         return self.daf.apply_input_params(args)

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev0}/src/ededup_local.py

@@ -13,10 +13,7 @@
 import os
 
 from data_processing.data_access import DataAccessLocal
-from ededup_transform_base import HashFilter
-from ededup_transform_python import EdedupTransform
-from ededup_transform_base import doc_column_name_key, int_column_name_key
-
+from ededup_transform_python import EdedupPythonTransform
 
 # create parameters
 input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input"))
@@ -26,18 +23,18 @@ local_conf = {
     "output_folder": output_folder,
 }
 
-ededup_params = {doc_column_name_key: "contents", int_column_name_key: "document_id", "filter": HashFilter({})}
+ededup_params = {"doc_column": "contents"}
 
 if __name__ == "__main__":
     # Here we show how to run outside of ray
     # Filter transform needs a DataAccess to ready the domain list.
     data_access = DataAccessLocal(local_conf)
     # Create and configure the transform.
-    transform = EdedupTransform(ededup_params)
+    transform = EdedupPythonTransform(ededup_params)
     # Use the local data access to read a parquet table.
     table, _ = data_access.get_table(os.path.join(input_folder, "sample1.parquet"))
-    print(f"input table has {table.num_rows} rows and {table.num_columns} columns")
+    print(f"input table has {table.num_rows} rows")
     # Transform the table
     table_list, metadata = transform.transform(table)
-    print(f"\noutput table has {table_list[0].num_rows} rows and {table_list[0].num_columns} columns")
+    print(f"\noutput table has {table_list[0].num_rows} rows")
     print(f"output metadata : {metadata}")

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev0}/src/ededup_local_python.py

@@ -13,14 +13,13 @@
 import os
 import sys
 
-from data_processing.runtime.pure_python import PythonTransformLauncher
 from data_processing.utils import ParamsUtils
-from ededup_transform_python import EdedupPythonTransformRuntimeConfiguration
-from ededup_transform_base import doc_column_name_cli_param, int_column_name_cli_param
+from data_processing.runtime.pure_python import PythonTransformLauncher
+from ededup_transform_python import EdedupPythonTransformConfiguration
 
 
 # create launcher
-launcher = PythonTransformLauncher(EdedupPythonTransformRuntimeConfiguration())
+launcher = PythonTransformLauncher(EdedupPythonTransformConfiguration())
 # create parameters
 input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input"))
 output_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../output"))
@@ -37,8 +36,7 @@ params = {
     "runtime_job_id": "job_id",
     "runtime_code_location": ParamsUtils.convert_to_ast(code_location),
     # ededup parameters
-    doc_column_name_cli_param: "contents",
-    int_column_name_cli_param: "document_id",
+    "ededup_doc_column": "contents",
 }
 sys.argv = ParamsUtils.dict_to_req(d=params)