data-prep-toolkit-transforms 0.2.1__tar.gz → 0.2.1.dev1__tar.gz

data_prep_toolkit_transforms-0.2.1.dev1/Makefile

@@ -0,0 +1,77 @@
+# Define the root of the local git clone for the common rules to be able 
+# know where they are running from.
+REPOROOT=../../..
+# Include a library of common .transform.* targets which most
+# transforms should be able to reuse.  However, feel free
+# to override/redefine the rules below. 
+
+# $(REPOROOT)/.make.versions file contains the versions
+
+#TRANSFORM_NAME=doc_quality
+
+include $(REPOROOT)/transforms/.make.transforms
+
+TRANSFORMS_NAMES = code/code_quality \
+    	code/code2parquet \
+    	code/header_cleanser \
+		code/code_quality \
+		code/proglang_select \
+		language/doc_chunk \
+		language/doc_quality \
+		language/lang_id \
+		language/pdf2parquet \
+		language/text_encoder \
+		universal/ededup \
+		universal/filter \
+		universal/resize \
+		universal/tokenization
+
+venv: 
+	$(MAKE) .defaults.create-venv
+	source venv/bin/activate;       \
+	$(PYTHON) -m pip install . 
+
+
+test::	setup venv test-src
+
+clean:: .transforms.clean
+	-rm -fr src
+
+image:: .transforms.python-image
+
+test-src:: 
+	source venv/bin/activate;       \
+	for T in $(TRANSFORMS_NAMES); do                    \
+	    echo running unit test on: $$T ; \
+		$(PYTEST) $(REPOROOT)/transforms/$$T/python/test; \
+	done;
+
+test-with-pypi:
+	$(MAKE) .defaults.create-venv
+	source venv/bin/activate;       \
+	$(PYTHON) -m pip install data_prep_toolkit_transforms==0.2.1.dev0 
+	$(MAKE) test-src
+
+
+setup: .transforms.setup 
+	$(MAKE) src
+
+src:
+	for T in $(TRANSFORMS_NAMES); do                    \
+	    echo copy src from  $$T ; \
+		cp -R $(REPOROOT)/transforms/$$T/python/src/ src/ ; \
+		rm -fr *.egg-info ; \
+		rm -fr dist ; \
+		rm -fr build ; \
+	done;
+
+
+build:: build-dist
+        
+publish:: publish-dist
+        
+build-dist:: setup .defaults.build-dist 
+
+publish-dist:: .defaults.publish-dist
+
+

data_prep_toolkit_transforms-0.2.1.dev1/PKG-INFO

@@ -0,0 +1,67 @@
+Metadata-Version: 2.1
+Name: data_prep_toolkit_transforms
+Version: 0.2.1.dev1
+Summary: Data Preparation Toolkit Transforms
+Author-email: Maroun Touma <touma@us.ibm.com>
+License: Apache-2.0
+Keywords: transforms,data preprocessing,data preparation,llm,generative,ai,fine-tuning,llmapps
+Requires-Python: <3.12,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: data-prep-toolkit==0.2.1.dev0
+Requires-Dist: argparse
+Requires-Dist: boto3==1.34.69
+Requires-Dist: bs4==0.0.2
+Requires-Dist: clamd==1.0.2
+Requires-Dist: docling[ocr]==1.1.2
+Requires-Dist: duckdb==0.10.1
+Requires-Dist: fasttext==0.9.2
+Requires-Dist: filetype<2.0.0,>=1.2.0
+Requires-Dist: huggingface-hub<1.0.0,>=0.21.4
+Requires-Dist: langcodes==3.3.0
+Requires-Dist: mmh3==4.1.0
+Requires-Dist: numpy==1.26.4
+Requires-Dist: pandas
+Requires-Dist: parameterized
+Requires-Dist: pyarrow==16.1.0
+Requires-Dist: python-dateutil>=2.8.2
+Requires-Dist: pytz>=2020.1
+Requires-Dist: quackling==0.1.0
+Requires-Dist: scancode-toolkit==32.1.0; platform_system != "Darwin"
+Requires-Dist: sentence-transformers==3.0.1
+Requires-Dist: transformers==4.38.2
+Requires-Dist: tzdata>=2022.7
+Requires-Dist: xxhash==3.4.1
+
+# DPK Python Transforms
+
+## installation
+
+The [transforms](https://github.com/IBM/data-prep-kit/blob/dev/transforms/README.md) are delivered as a standard pyton library available on pypi and can be installed using pip install:
+
+`python -m pip install data-prep-toolkit-transforms`
+
+installing the python transforms will also install  `data-prep-toolkit`
+
+## List of Transforms in current package
+
+* code
+    * [code2parquet](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/code2parquet/python/README.md)
+    * header_cleanser (Not available on MacOS)
+    * code_quality
+    * proglang_select
+* language
+    * doc_chunk
+	* *doc_quality
+	* lang_id
+	* pdf2parquet
+	* text_encoder
+* universal
+    * ededup
+	* filter
+	* resize
+	* tokenization
+
+
+
+
+ 

data_prep_toolkit_transforms-0.2.1.dev1/README.md

@@ -0,0 +1,33 @@
+# DPK Python Transforms
+
+## installation
+
+The [transforms](https://github.com/IBM/data-prep-kit/blob/dev/transforms/README.md) are delivered as a standard pyton library available on pypi and can be installed using pip install:
+
+`python -m pip install data-prep-toolkit-transforms`
+
+installing the python transforms will also install  `data-prep-toolkit`
+
+## List of Transforms in current package
+
+* code
+    * [code2parquet](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/code2parquet/python/README.md)
+    * header_cleanser (Not available on MacOS)
+    * code_quality
+    * proglang_select
+* language
+    * doc_chunk
+	* *doc_quality
+	* lang_id
+	* pdf2parquet
+	* text_encoder
+* universal
+    * ededup
+	* filter
+	* resize
+	* tokenization
+
+
+
+
+ 

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit_transforms"
-version = "0.2.1"
+version = "0.2.1.dev1"
 requires-python = ">=3.10,<3.12"
 keywords = ["transforms", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 description = "Data Preparation Toolkit Transforms"
@@ -9,14 +9,38 @@ readme = {file = "README.md", content-type = "text/markdown"}
 authors = [
     { name = "Maroun Touma", email = "touma@us.ibm.com" },
 ]
-dynamic = ["dependencies"]
+
+dependencies = [
+	"data-prep-toolkit==0.2.1.dev0",
+	"argparse",
+	"boto3==1.34.69",
+	"bs4==0.0.2",
+	"clamd==1.0.2",
+	"docling[ocr]==1.1.2",
+	"duckdb==0.10.1",
+	"fasttext==0.9.2",
+	"filetype >=1.2.0, <2.0.0",
+	"huggingface-hub >= 0.21.4, <1.0.0",
+	"langcodes==3.3.0",
+	"mmh3==4.1.0",
+	"numpy==1.26.4",
+	"pandas",
+	"parameterized",
+	"pyarrow==16.1.0",
+	"python-dateutil>=2.8.2",
+	"pytz>=2020.1",
+	"quackling==0.1.0",
+	"scancode-toolkit==32.1.0 ; platform_system != 'Darwin'",
+	"sentence-transformers==3.0.1",
+	"transformers==4.38.2",
+	"tzdata>=2022.7",
+	"xxhash==3.4.1",
+]
 
 [build-system]
 requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
 build-backend = "setuptools.build_meta"
 
-[tool.setuptools.dynamic]
-dependencies = {file = ["requirements.txt"]}
 
 [options]
 package_dir = ["src"]

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev1}/src/code2parquet_transform.py

@@ -13,16 +13,20 @@
 import io
 import json
 import logging
-import os
 import uuid
 import zipfile
 from argparse import ArgumentParser, Namespace
 from datetime import datetime
 from typing import Any
+import os
 
 import pyarrow as pa
 from data_processing.data_access import DataAccess, DataAccessFactory
-from data_processing.transform import AbstractBinaryTransform, TransformConfiguration
+from data_processing.transform import (
+    AbstractBinaryTransform,
+    AbstractTransform,
+    TransformConfiguration,
+)
 from data_processing.utils import CLIArgumentProvider, TransformUtils, str2bool
 
 
@@ -132,7 +136,7 @@ class CodeToParquetTransform(AbstractBinaryTransform):
                                     "hash": TransformUtils.str_to_hash(content_string),
                                     "size": len(content_string),
                                     "date_acquired": datetime.now().isoformat(),
-                                    "repo_name": os.path.splitext(os.path.basename(file_name))[0],
+                                    "repo_name":os.path.splitext(os.path.basename(file_name))[0]
                                 } | self.shared_columns
                                 if self.detect_programming_lang:
                                     lang = self._get_lang_from_ext(ext)
@@ -155,7 +159,7 @@ class CodeToParquetTransformConfiguration(TransformConfiguration):
     configuration with CLI args and combining of metadata.
     """
 
-    def __init__(self, transform_class: type[AbstractBinaryTransform] = CodeToParquetTransform):
+    def __init__(self, transform_class: type[AbstractTransform] = CodeToParquetTransform):
         super().__init__(
             name=shortname,
             transform_class=transform_class,

data_prep_toolkit_transforms-0.2.1.dev1/src/data_prep_toolkit_transforms.egg-info/PKG-INFO

@@ -0,0 +1,67 @@
+Metadata-Version: 2.1
+Name: data_prep_toolkit_transforms
+Version: 0.2.1.dev1
+Summary: Data Preparation Toolkit Transforms
+Author-email: Maroun Touma <touma@us.ibm.com>
+License: Apache-2.0
+Keywords: transforms,data preprocessing,data preparation,llm,generative,ai,fine-tuning,llmapps
+Requires-Python: <3.12,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: data-prep-toolkit==0.2.1.dev0
+Requires-Dist: argparse
+Requires-Dist: boto3==1.34.69
+Requires-Dist: bs4==0.0.2
+Requires-Dist: clamd==1.0.2
+Requires-Dist: docling[ocr]==1.1.2
+Requires-Dist: duckdb==0.10.1
+Requires-Dist: fasttext==0.9.2
+Requires-Dist: filetype<2.0.0,>=1.2.0
+Requires-Dist: huggingface-hub<1.0.0,>=0.21.4
+Requires-Dist: langcodes==3.3.0
+Requires-Dist: mmh3==4.1.0
+Requires-Dist: numpy==1.26.4
+Requires-Dist: pandas
+Requires-Dist: parameterized
+Requires-Dist: pyarrow==16.1.0
+Requires-Dist: python-dateutil>=2.8.2
+Requires-Dist: pytz>=2020.1
+Requires-Dist: quackling==0.1.0
+Requires-Dist: scancode-toolkit==32.1.0; platform_system != "Darwin"
+Requires-Dist: sentence-transformers==3.0.1
+Requires-Dist: transformers==4.38.2
+Requires-Dist: tzdata>=2022.7
+Requires-Dist: xxhash==3.4.1
+
+# DPK Python Transforms
+
+## installation
+
+The [transforms](https://github.com/IBM/data-prep-kit/blob/dev/transforms/README.md) are delivered as a standard pyton library available on pypi and can be installed using pip install:
+
+`python -m pip install data-prep-toolkit-transforms`
+
+installing the python transforms will also install  `data-prep-toolkit`
+
+## List of Transforms in current package
+
+* code
+    * [code2parquet](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/code2parquet/python/README.md)
+    * header_cleanser (Not available on MacOS)
+    * code_quality
+    * proglang_select
+* language
+    * doc_chunk
+	* *doc_quality
+	* lang_id
+	* pdf2parquet
+	* text_encoder
+* universal
+    * ededup
+	* filter
+	* resize
+	* tokenization
+
+
+
+
+ 

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev1}/src/data_prep_toolkit_transforms.egg-info/SOURCES.txt

@@ -1,7 +1,6 @@
 Makefile
 README.md
 pyproject.toml
-requirements.txt
 src/cc_net_prepro.py
 src/code2parquet_local.py
 src/code2parquet_local_python.py
@@ -19,10 +18,6 @@ src/doc_chunk_local.py
 src/doc_chunk_local_python.py
 src/doc_chunk_transform.py
 src/doc_chunk_transform_python.py
-src/doc_id_local.py
-src/doc_id_local_python.py
-src/doc_id_transform_base.py
-src/doc_id_transform_python.py
 src/doc_quality_local.py
 src/doc_quality_local_python.py
 src/doc_quality_transform.py
@@ -30,7 +25,6 @@ src/doc_quality_transform_python.py
 src/doc_quality_utils.py
 src/ededup_local.py
 src/ededup_local_python.py
-src/ededup_local_python_incremental.py
 src/ededup_transform_base.py
 src/ededup_transform_python.py
 src/filter_local.py
@@ -38,7 +32,6 @@ src/filter_local_python.py
 src/filter_test_support.py
 src/filter_transform.py
 src/filter_transform_python.py
-src/flair_recognizer.py
 src/header_cleanser_local.py
 src/header_cleanser_local_python.py
 src/header_cleanser_test_support.py
@@ -54,12 +47,6 @@ src/pdf2parquet_local.py
 src/pdf2parquet_local_python.py
 src/pdf2parquet_transform.py
 src/pdf2parquet_transform_python.py
-src/pii_analyzer.py
-src/pii_anonymizer.py
-src/pii_redactor_local.py
-src/pii_redactor_local_python.py
-src/pii_redactor_transform.py
-src/pii_redactor_transform_python.py
 src/proglang_select_local.py
 src/proglang_select_local_python.py
 src/proglang_select_transform.py

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev1}/src/data_prep_toolkit_transforms.egg-info/requires.txt

@@ -1,26 +1,26 @@
-data-prep-toolkit>=0.2.1
+data-prep-toolkit==0.2.1.dev0
+argparse
+boto3==1.34.69
 bs4==0.0.2
-docling-ibm-models==1.1.7
-deepsearch-glm==0.21.0
-docling==1.11.0
-filetype<2.0.0,>=1.2.0
-docling-core==1.3.0
-llama-index-core<0.12.0,>=0.11.0
+clamd==1.0.2
+docling[ocr]==1.1.2
 duckdb==0.10.1
 fasttext==0.9.2
+filetype<2.0.0,>=1.2.0
 huggingface-hub<1.0.0,>=0.21.4
 langcodes==3.3.0
 mmh3==4.1.0
 numpy==1.26.4
 pandas
 parameterized
+pyarrow==16.1.0
+python-dateutil>=2.8.2
+pytz>=2020.1
+quackling==0.1.0
 sentence-transformers==3.0.1
 transformers==4.38.2
+tzdata>=2022.7
 xxhash==3.4.1
-presidio-analyzer>=2.2.355
-presidio-anonymizer>=2.2.355
-flair>=0.14.0
-pandas>=2.2.2
 
 [:platform_system != "Darwin"]
 scancode-toolkit==32.1.0

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev1}/src/data_prep_toolkit_transforms.egg-info/top_level.txt

@@ -15,10 +15,6 @@ doc_chunk_local
 doc_chunk_local_python
 doc_chunk_transform
 doc_chunk_transform_python
-doc_id_local
-doc_id_local_python
-doc_id_transform_base
-doc_id_transform_python
 doc_quality_local
 doc_quality_local_python
 doc_quality_transform
@@ -26,7 +22,6 @@ doc_quality_transform_python
 doc_quality_utils
 ededup_local
 ededup_local_python
-ededup_local_python_incremental
 ededup_transform_base
 ededup_transform_python
 filter_local
@@ -34,7 +29,6 @@ filter_local_python
 filter_test_support
 filter_transform
 filter_transform_python
-flair_recognizer
 header_cleanser_local
 header_cleanser_local_python
 header_cleanser_test_support
@@ -50,12 +44,6 @@ pdf2parquet_local
 pdf2parquet_local_python
 pdf2parquet_transform
 pdf2parquet_transform_python
-pii_analyzer
-pii_anonymizer
-pii_redactor_local
-pii_redactor_local_python
-pii_redactor_transform
-pii_redactor_transform_python
 proglang_select_local
 proglang_select_local_python
 proglang_select_transform

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev1}/src/doc_Gopher_statistics.py

@@ -49,9 +49,7 @@ def compute_word_statistics(text: str, symbols: list = ["#", "..."]) -> tuple[in
     return total_words, mean_word_len, symbol_to_word_ratio
 
 
-def compute_bullet_point_ellipsis_alphabet_word_ratio(
-    text: str, bullets: list = ["-", "*"]
-) -> tuple[float, float, float]:
+def compute_bullet_point_ellipsis_alphabet_word_ratio(text: str, bullets: list = ["-", "*"]) -> tuple[float, float, float]:
     """
     Given a text document:
         - Compute the ratio of lines starting with a bullet point (should be <=90%)

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev1}/src/doc_chunk_chunkers.py

@@ -10,13 +10,14 @@
 # limitations under the License.
 ################################################################################
 
+import math
 from abc import ABCMeta, abstractmethod
-from typing import Iterator, Optional
+from typing import Iterator
 
 from docling_core.types import Document as DLDocument
+from quackling.core.chunkers.hierarchical_chunker import HierarchicalChunker
 from llama_index.core import Document as LIDocument
 from llama_index.core.node_parser import MarkdownNodeParser
-from docling_core.transforms.chunker import HierarchicalChunker
 
 
 class ChunkingExecutor(metaclass=ABCMeta):
@@ -24,25 +25,13 @@ class ChunkingExecutor(metaclass=ABCMeta):
     def chunk(self, content: str) -> Iterator[dict]:
         raise NotImplemented("The chunk() method must be implemented")
 
-
 class DLJsonChunker(ChunkingExecutor):
-    def __init__(
-        self,
-        min_chunk_len: Optional[int],
-        output_chunk_column_name: str,
-        output_jsonpath_column_name: str,
-        output_pageno_column_name_key: str,
-        output_bbox_column_name_key: str,
-    ):
+    def __init__(self, output_chunk_column_name: str, output_jsonpath_column_name: str, output_pageno_column_name_key: str, output_bbox_column_name_key: str):
         self.output_chunk_column_name = output_chunk_column_name
         self.output_jsonpath_column_name = output_jsonpath_column_name
         self.output_pageno_column_name_key = output_pageno_column_name_key
         self.output_bbox_column_name_key = output_bbox_column_name_key
-
-        chunker_kwargs = dict(include_metadata=True)
-        if min_chunk_len is not None:
-            chunker_kwargs["min_chunk_len"] = min_chunk_len
-        self._chunker = HierarchicalChunker(**chunker_kwargs)
+        self._chunker = HierarchicalChunker(include_metadata=True)
 
     def chunk(self, content: str) -> Iterator[dict]:
         doc = DLDocument.model_validate_json(content)
@@ -54,7 +43,6 @@ class DLJsonChunker(ChunkingExecutor):
                 self.output_bbox_column_name_key: chunk.bbox,
             }
 
-
 class LIMarkdown(ChunkingExecutor):
     def __init__(self, output_chunk_column_name: str):
         self.output_chunk_column_name = output_chunk_column_name
@@ -66,3 +54,4 @@ class LIMarkdown(ChunkingExecutor):
             yield {
                 self.output_chunk_column_name: node.text,
             }
+

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev1}/src/doc_chunk_transform.py

@@ -24,20 +24,14 @@ from doc_chunk_chunkers import ChunkingExecutor, DLJsonChunker, LIMarkdown
 short_name = "doc_chunk"
 cli_prefix = f"{short_name}_"
 content_column_name_key = "content_column_name"
-doc_id_column_name_key = "doc_id_column_name"
 chunking_type_key = "chunking_type"
-dl_min_chunk_len_key = "dl_min_chunk_len"
 output_chunk_column_name_key = "output_chunk_column_name"
-output_source_doc_id_column_name_key = "output_source_doc_id_column_name"
 output_jsonpath_column_name_key = "output_jsonpath_column_name"
 output_pageno_column_name_key = "output_pageno_column_name"
 output_bbox_column_name_key = "output_bbox_column_name"
 content_column_name_cli_param = f"{cli_prefix}{content_column_name_key}"
-doc_id_column_name_cli_param = f"{cli_prefix}{doc_id_column_name_key}"
 chunking_type_cli_param = f"{cli_prefix}{chunking_type_key}"
-dl_min_chunk_len_cli_param = f"{cli_prefix}{dl_min_chunk_len_key}"
 output_chunk_column_name_cli_param = f"{cli_prefix}{output_chunk_column_name_key}"
-output_source_doc_id_column_name_cli_param = f"{cli_prefix}{output_source_doc_id_column_name_key}"
 output_jsonpath_column_name_cli_param = f"{cli_prefix}{output_jsonpath_column_name_key}"
 output_pageno_column_name_cli_param = f"{cli_prefix}{output_pageno_column_name_key}"
 output_bbox_column_name_cli_param = f"{cli_prefix}{output_bbox_column_name_key}"
@@ -52,11 +46,8 @@ class chunking_types(str, enum.Enum):
 
 
 default_content_column_name = "contents"
-default_doc_id_column_name = "document_id"
 default_chunking_type = chunking_types.DL_JSON
-default_dl_min_chunk_len = None
 default_output_chunk_column_name = "contents"
-default_output_source_doc_id_column_name = "source_document_id"
 default_output_jsonpath_column_name = "doc_jsonpath"
 default_output_pageno_column_name = "page_number"
 default_output_bbox_column_name = "bbox"
@@ -82,12 +73,9 @@ class DocChunkTransform(AbstractTableTransform):
         self.chunking_type = config.get(chunking_type_key, default_chunking_type)
 
         self.content_column_name = config.get(content_column_name_key, default_content_column_name)
-        self.doc_id_column_name = config.get(doc_id_column_name_key, default_doc_id_column_name)
         self.output_chunk_column_name = config.get(output_chunk_column_name_key, default_output_chunk_column_name)
-        self.output_source_doc_id_column_name = config.get(output_source_doc_id_column_name_key, default_output_source_doc_id_column_name)
 
         # Parameters for Docling JSON chunking
-        self.dl_min_chunk_len = config.get(dl_min_chunk_len_key, default_dl_min_chunk_len)
         self.output_jsonpath_column_name = config.get(
             output_jsonpath_column_name_key, default_output_jsonpath_column_name
         )
@@ -101,7 +89,6 @@ class DocChunkTransform(AbstractTableTransform):
         self.chunker: ChunkingExecutor
         if self.chunking_type == chunking_types.DL_JSON:
             self.chunker = DLJsonChunker(
-                min_chunk_len=self.dl_min_chunk_len,
                 output_chunk_column_name=self.output_chunk_column_name,
                 output_jsonpath_column_name=self.output_jsonpath_column_name,
                 output_pageno_column_name_key=self.output_pageno_column_name_key,
@@ -125,11 +112,8 @@ class DocChunkTransform(AbstractTableTransform):
         for batch in table.to_batches():
             for row in batch.to_pylist():
                 content: str = row[self.content_column_name]
-                new_row = {k: v for k, v in row.items() if k not in (self.content_column_name, self.doc_id_column_name)}
-                if self.doc_id_column_name in row:
-                    new_row[self.output_source_doc_id_column_name] = row[self.doc_id_column_name]
+                new_row = {k: v for k, v in row.items() if k not in (self.content_column_name,)}
                 for chunk in self.chunker.chunk(content):
-                    chunk[self.doc_id_column_name] = TransformUtils.str_to_hash(chunk[self.output_chunk_column_name])
                     data.append(
                         {
                             **new_row,
@@ -178,26 +162,11 @@ class DocChunkTransformConfiguration(TransformConfiguration):
             default=default_content_column_name,
             help="Name of the column containing the text to be chunked",
         )
-        parser.add_argument(
-            f"--{doc_id_column_name_cli_param}",
-            default=default_doc_id_column_name,
-            help="Name of the column containing the doc_id to be propagated in the output",
-        )
-        parser.add_argument(
-            f"--{dl_min_chunk_len_cli_param}",
-            default=default_dl_min_chunk_len,
-            help="Minimum number of characters for the chunk in the dl_json chunker. Setting to None is using the library defaults, i.e. a min_chunk_len=64.",
-        )
         parser.add_argument(
             f"--{output_chunk_column_name_cli_param}",
             default=default_output_chunk_column_name,
             help="Column name to store the chunks",
         )
-        parser.add_argument(
-            f"--{output_source_doc_id_column_name_cli_param}",
-            default=default_output_source_doc_id_column_name,
-            help="Column name to store the `document_id` from the input table",
-        )
         parser.add_argument(
             f"--{output_jsonpath_column_name_cli_param}",
             default=default_output_jsonpath_column_name,

{data_prep_toolkit_transforms-0.2.1 → data_prep_toolkit_transforms-0.2.1.dev1}/src/doc_quality_local_python.py

@@ -16,13 +16,12 @@ import sys
 from data_processing.runtime.pure_python import PythonTransformLauncher
 from data_processing.utils import ParamsUtils
 from doc_quality_transform import (
-    bad_word_filepath_cli_param,
-    doc_content_column_cli_param,
     text_lang_cli_param,
+    doc_content_column_cli_param,
+    bad_word_filepath_cli_param,
 )
 from doc_quality_transform_python import DocQualityPythonTransformConfiguration
 
-
 # create parameters
 input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "test-data", "input"))
 output_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "output"))
@@ -32,7 +31,7 @@ local_conf = {
 }
 code_location = {"github": "github", "commit_hash": "12345", "path": "path"}
 basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
-model_path = os.path.join(basedir, "models")
+model_path=os.path.join(basedir, "models")
 if not os.path.exists(model_path):
     model_path = os.path.abspath(os.path.join(basedir, "..", "models"))