dalla-data-processing 0.0.2__tar.gz → 0.0.3__tar.gz

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dalla-data-processing
-Version: 0.0.2
+Version: 0.0.3
 Summary: data processing pipeline with deduplication, stemming, quality checking, and readability scoring, used for the DALLA Models
 Author-email: Hadi Hamoud <hhamoud@dohainstitute.edu.qa>, Digital Research Unit - Arab Center <dru@dohainstitute.edu.qa>
 Project-URL: Homepage, https://github.com/U4RASD/dalla-data-processing
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Text Processing :: Linguistic
-Requires-Python: >=3.12
+Requires-Python: <3.13,>=3.12
 Description-Content-Type: text/markdown
 Requires-Dist: datasets>=2.14.0
 Requires-Dist: transformers>=4.30.0
@@ -28,18 +28,17 @@ Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
 Requires-Dist: ruff>=0.1.0; extra == "dev"
 Requires-Dist: pre-commit>=3.0.0; extra == "dev"
 Provides-Extra: dedup
-Requires-Dist: camel-tools>=1.5.0; extra == "dedup"
+Requires-Dist: camel-tools==1.5.7; extra == "dedup"
 Provides-Extra: dedup-native
 Requires-Dist: cffi>=1.15.0; extra == "dedup-native"
 Provides-Extra: stem
-Requires-Dist: camel-tools>=1.5.0; extra == "stem"
+Requires-Dist: camel-tools==1.5.7; extra == "stem"
 Provides-Extra: quality
-Requires-Dist: camel-tools>=1.5.0; extra == "quality"
+Requires-Dist: camel-tools==1.5.7; extra == "quality"
 Provides-Extra: readability
 Requires-Dist: textstat>=0.7.0; extra == "readability"
 Provides-Extra: pack
 Requires-Dist: sentencepiece>=0.2.0; extra == "pack"
-Requires-Dist: rbpe; extra == "pack"
 Requires-Dist: pyyaml; extra == "pack"
 Provides-Extra: all
 Requires-Dist: dalla-data-processing[dedup,dedup-native,dev,pack,quality,readability,stem]; extra == "all"

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.3}/dalla_data_processing/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.0.2'
-__version_tuple__ = version_tuple = (0, 0, 2)
+__version__ = version = '0.0.3'
+__version_tuple__ = version_tuple = (0, 0, 3)
 
-__commit_id__ = commit_id = 'g68cee46a4'
+__commit_id__ = commit_id = 'g37580acc9'

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.3}/dalla_data_processing/cli.py

@@ -651,7 +651,15 @@ def pack(
             tokenizer = RBPETokenizer.from_pretrained(config_data["tokenizer_path"])
         except ImportError:
             logger.error("Missing rbpe package")
-            logger.error("Install with: pip install rbpe")
+            logger.error(
+                "rbpe is not included in the default installation due to "
+                "dependency conflicts with camel-tools (transformers version requirements)"
+            )
+            logger.error("Install separately with: pip install rbpe")
+            logger.error(
+                "Note: Installing rbpe may require a separate environment "
+                "if you also use dedup/stem/quality features"
+            )
             sys.exit(1)
     else:
         try:

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.3}/dalla_data_processing/quality/checker.py

@@ -11,6 +11,7 @@ from concurrent.futures import TimeoutError as FutureTimeoutError
 from types import MethodType
 from typing import Any
 
+from camel_tools.data.catalogue import Catalogue
 from camel_tools.disambig.bert import BERTUnfactoredDisambiguator
 from camel_tools.disambig.mle import MLEDisambiguator
 from datasets import Dataset
@@ -53,6 +54,25 @@ class QualityChecker:
 
     def _init_disambiguator(self):
         """Initialize and configure the disambiguator with caching."""
+        # Install required CAMeL Tools packages based on model type
+        logger.info("Checking CAMeL Tools data packages...")
+        catalogue = Catalogue.load_catalogue()
+
+        try:
+            catalogue.download_package("morphology-db-msa-r13")
+            catalogue.download_package("disambig-mle-calima-msa-r13")
+            logger.info("msa-r13 packages installed")
+        except Exception as e:
+            logger.warning(f"Package installation warning: {e}")
+
+        # Install BERT package if using BERT model
+        if self.model == "bert":
+            try:
+                catalogue.download_package("disambig-bert-unfactored-all")
+                logger.info("BERT package installed")
+            except Exception as e:
+                logger.warning(f"BERT package installation warning: {e}")
+
         if self.model == "mle":
             self.disambiguator = MLEDisambiguator.pretrained()
             logger.info("MLE disambiguator loaded")

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.3}/dalla_data_processing/stemming/stemmer.py

@@ -473,12 +473,19 @@ def stem_dataset(
     catalogue = Catalogue.load_catalogue()
     try:
         catalogue.download_package("morphology-db-msa-r13")
-        if model == "mle":
-            catalogue.download_package("disambig-mle-calima-msa-r13")
-        # For BERT, let it download automatically when pretrained() is called
-        logger.info("CAMeL Tools data packages ready")
+        catalogue.download_package("disambig-mle-calima-msa-r13")
+        logger.info("msa-r13 packages installed")
     except Exception as e:
-        logger.warning(f"Could not verify CAMeL packages: {e}")
+        logger.warning(f"Package installation warning: {e}")
+
+    if model == "bert":
+        try:
+            catalogue.download_package("disambig-bert-unfactored-all")
+            logger.info("BERT package installed")
+        except Exception as e:
+            logger.warning(f"BERT package installation warning: {e}")
+
+    logger.info("CAMeL Tools data packages ready")
 
     logger.info("Loading additional words lists...")
     words_dir = os.path.join(os.path.dirname(__file__), "data")
@@ -597,15 +604,21 @@ def stem(
     if not all(isinstance(t, str) for t in text_list):
         raise TypeError("All items in text list must be strings")
 
-    # Initialize disambiguator (cached globally if possible)
     logger.info(f"Initializing {model.upper()} disambiguator...")
     catalogue = Catalogue.load_catalogue()
     try:
         catalogue.download_package("morphology-db-msa-r13")
-        if model == "mle":
-            catalogue.download_package("disambig-mle-calima-msa-r13")
+        catalogue.download_package("disambig-mle-calima-msa-r13")
+        logger.info("msa-r13 packages installed")
     except Exception as e:
-        logger.warning(f"Could not verify CAMeL packages: {e}")
+        logger.warning(f"Package installation warning: {e}")
+
+    if model == "bert":
+        try:
+            catalogue.download_package("disambig-bert-unfactored-all")
+            logger.info("BERT package installed")
+        except Exception as e:
+            logger.warning(f"BERT package installation warning: {e}")
 
     if model == "mle":
         disambiguator = MLEDisambiguator.pretrained("calima-msa-r13", cache_size=1_000_000)

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.3}/dalla_data_processing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dalla-data-processing
-Version: 0.0.2
+Version: 0.0.3
 Summary: data processing pipeline with deduplication, stemming, quality checking, and readability scoring, used for the DALLA Models
 Author-email: Hadi Hamoud <hhamoud@dohainstitute.edu.qa>, Digital Research Unit - Arab Center <dru@dohainstitute.edu.qa>
 Project-URL: Homepage, https://github.com/U4RASD/dalla-data-processing
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Text Processing :: Linguistic
-Requires-Python: >=3.12
+Requires-Python: <3.13,>=3.12
 Description-Content-Type: text/markdown
 Requires-Dist: datasets>=2.14.0
 Requires-Dist: transformers>=4.30.0
@@ -28,18 +28,17 @@ Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
 Requires-Dist: ruff>=0.1.0; extra == "dev"
 Requires-Dist: pre-commit>=3.0.0; extra == "dev"
 Provides-Extra: dedup
-Requires-Dist: camel-tools>=1.5.0; extra == "dedup"
+Requires-Dist: camel-tools==1.5.7; extra == "dedup"
 Provides-Extra: dedup-native
 Requires-Dist: cffi>=1.15.0; extra == "dedup-native"
 Provides-Extra: stem
-Requires-Dist: camel-tools>=1.5.0; extra == "stem"
+Requires-Dist: camel-tools==1.5.7; extra == "stem"
 Provides-Extra: quality
-Requires-Dist: camel-tools>=1.5.0; extra == "quality"
+Requires-Dist: camel-tools==1.5.7; extra == "quality"
 Provides-Extra: readability
 Requires-Dist: textstat>=0.7.0; extra == "readability"
 Provides-Extra: pack
 Requires-Dist: sentencepiece>=0.2.0; extra == "pack"
-Requires-Dist: rbpe; extra == "pack"
 Requires-Dist: pyyaml; extra == "pack"
 Provides-Extra: all
 Requires-Dist: dalla-data-processing[dedup,dedup-native,dev,pack,quality,readability,stem]; extra == "all"

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.3}/dalla_data_processing.egg-info/requires.txt

@@ -9,7 +9,7 @@ structlog>=24.0.0
 dalla-data-processing[dedup,dedup-native,dev,pack,quality,readability,stem]
 
 [dedup]
-camel-tools>=1.5.0
+camel-tools==1.5.7
 
 [dedup-native]
 cffi>=1.15.0
@@ -22,14 +22,13 @@ pre-commit>=3.0.0
 
 [pack]
 sentencepiece>=0.2.0
-rbpe
 pyyaml
 
 [quality]
-camel-tools>=1.5.0
+camel-tools==1.5.7
 
 [readability]
 textstat>=0.7.0
 
 [stem]
-camel-tools>=1.5.0
+camel-tools==1.5.7

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.3}/pyproject.toml

@@ -11,7 +11,7 @@ authors = [
     {name = "Digital Research Unit - Arab Center", email = "dru@dohainstitute.edu.qa"}
 ]
 readme = "README.md"
-requires-python = ">=3.12"
+requires-python = ">=3.12,<3.13"
 keywords = ["arabic", "nlp", "data-processing", "deduplication", "stemming", "readability", "quality"]
 classifiers = [
     "Intended Audience :: Developers",
@@ -39,23 +39,24 @@ dev = [
     "pre-commit>=3.0.0",
 ]
 dedup = [
-    "camel-tools>=1.5.0",
+    "camel-tools==1.5.7",
 ]
 dedup-native = [
     "cffi>=1.15.0",
 ]
 stem = [
-    "camel-tools>=1.5.0",
+    "camel-tools==1.5.7",
 ]
 quality = [
-    "camel-tools>=1.5.0",
+    "camel-tools==1.5.7",
 ]
 readability = [
     "textstat>=0.7.0",
 ]
 pack = [
     "sentencepiece>=0.2.0",
-    "rbpe",
+    # "rbpe",  # excluded due to transformers version conflict with camel-tools
+                # users should install separately if needed: pip install rbpe
     "pyyaml",
 ]
 all = [