dalla-data-processing 0.0.2__tar.gz → 0.0.4__tar.gz

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.4}/.github/workflows/ci.yml

@@ -55,8 +55,10 @@ jobs:
         include:
           - os: ubuntu-latest
             artifact_name: onion-linux-x86_64
-          - os: macos-latest
+          - os: macos-13
             artifact_name: onion-darwin-x86_64
+          - os: macos-14
+            artifact_name: onion-darwin-arm64
     
     steps:
     - uses: actions/checkout@v4

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.4}/.github/workflows/release.yml

@@ -17,8 +17,10 @@ jobs:
         include:
           - os: ubuntu-latest
             artifact_name: onion-linux-x86_64
-          - os: macos-latest
+          - os: macos-13
             artifact_name: onion-darwin-x86_64
+          - os: macos-14
+            artifact_name: onion-darwin-arm64
     
     steps:
     - uses: actions/checkout@v4
@@ -46,19 +48,10 @@ jobs:
         path: dalla_data_processing/deduplication/bin/onion-*
         if-no-files-found: error
 
-  build-wheels:
-    name: Build wheels on ${{ matrix.os }}
+  build-wheel:
+    name: Build universal wheel with all platform binaries
     needs: build-onion
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        include:
-          - os: ubuntu-latest
-            onion_artifact: onion-linux-x86_64
-          - os: macos-latest
-            onion_artifact: onion-darwin-x86_64
-          - os: windows-latest
-            onion_artifact: none
+    runs-on: ubuntu-latest
     
     steps:
     - uses: actions/checkout@v4
@@ -67,24 +60,30 @@ jobs:
     
     - name: Create bin directory
       run: mkdir -p dalla_data_processing/deduplication/bin
-      shell: bash
     
-    - name: Download Onion binary
-      if: matrix.onion_artifact != 'none'
+    - name: Download Linux binary
+      uses: actions/download-artifact@v4
+      with:
+        name: onion-linux-x86_64
+        path: dalla_data_processing/deduplication/bin/
+    
+    - name: Download macOS Intel binary
+      uses: actions/download-artifact@v4
+      with:
+        name: onion-darwin-x86_64
+        path: dalla_data_processing/deduplication/bin/
+    
+    - name: Download macOS ARM binary
       uses: actions/download-artifact@v4
       with:
-        name: ${{ matrix.onion_artifact }}
+        name: onion-darwin-arm64
         path: dalla_data_processing/deduplication/bin/
     
     - name: Set binary permissions
-      if: matrix.onion_artifact != 'none'
       run: chmod +x dalla_data_processing/deduplication/bin/onion-*
-      shell: bash
     
-    - name: List binary files
-      if: matrix.onion_artifact != 'none'
+    - name: List all binaries
       run: ls -lah dalla_data_processing/deduplication/bin/
-      shell: bash
     
     - name: Set up Python
       uses: actions/setup-python@v5
@@ -101,13 +100,16 @@ jobs:
     
     - name: List wheel contents
       run: |
-        python -m zipfile -l dist/*.whl | head -50
+        echo "=== Wheel contents (binaries) ==="
+        python -m zipfile -l dist/*.whl | grep onion-
+        echo "=== Wheel info ==="
+        ls -lh dist/
     
-    - name: Upload wheels
+    - name: Upload wheel
       uses: actions/upload-artifact@v4
       with:
-        name: wheels-${{ matrix.os }}
-        path: dist/*
+        name: wheel
+        path: dist/*.whl
 
   build-sdist:
     name: Build source distribution
@@ -136,15 +138,15 @@ jobs:
 
   publish-pypi:
     name: Publish to PyPI
-    needs: [build-wheels, build-sdist]
+    needs: [build-wheel, build-sdist]
     runs-on: ubuntu-latest
     environment: release
     
     steps:
-    # Only use Linux wheel to avoid conflicts
+    # Download the universal wheel with all platform binaries
     - uses: actions/download-artifact@v4
       with:
-        name: wheels-ubuntu-latest
+        name: wheel
         path: dist/
     
     - uses: actions/download-artifact@v4
@@ -168,7 +170,7 @@ jobs:
 
   create-release:
     name: Create GitHub Release
-    needs: [build-wheels, build-sdist]
+    needs: [build-wheel, build-sdist]
     runs-on: ubuntu-latest
     
     steps:
@@ -178,8 +180,7 @@ jobs:
     
     - uses: actions/download-artifact@v4
       with:
-        pattern: wheels-*
-        merge-multiple: true
+        name: wheel
         path: dist/
     
     - uses: actions/download-artifact@v4

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dalla-data-processing
-Version: 0.0.2
+Version: 0.0.4
 Summary: data processing pipeline with deduplication, stemming, quality checking, and readability scoring, used for the DALLA Models
 Author-email: Hadi Hamoud <hhamoud@dohainstitute.edu.qa>, Digital Research Unit - Arab Center <dru@dohainstitute.edu.qa>
 Project-URL: Homepage, https://github.com/U4RASD/dalla-data-processing
@@ -11,10 +11,13 @@ Keywords: arabic,nlp,data-processing,deduplication,stemming,readability,quality
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Text Processing :: Linguistic
-Requires-Python: >=3.12
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: datasets>=2.14.0
 Requires-Dist: transformers>=4.30.0
@@ -28,18 +31,17 @@ Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
 Requires-Dist: ruff>=0.1.0; extra == "dev"
 Requires-Dist: pre-commit>=3.0.0; extra == "dev"
 Provides-Extra: dedup
-Requires-Dist: camel-tools>=1.5.0; extra == "dedup"
+Requires-Dist: camel-tools==1.5.7; extra == "dedup"
 Provides-Extra: dedup-native
 Requires-Dist: cffi>=1.15.0; extra == "dedup-native"
 Provides-Extra: stem
-Requires-Dist: camel-tools>=1.5.0; extra == "stem"
+Requires-Dist: camel-tools==1.5.7; extra == "stem"
 Provides-Extra: quality
-Requires-Dist: camel-tools>=1.5.0; extra == "quality"
+Requires-Dist: camel-tools==1.5.7; extra == "quality"
 Provides-Extra: readability
 Requires-Dist: textstat>=0.7.0; extra == "readability"
 Provides-Extra: pack
 Requires-Dist: sentencepiece>=0.2.0; extra == "pack"
-Requires-Dist: rbpe; extra == "pack"
 Requires-Dist: pyyaml; extra == "pack"
 Provides-Extra: all
 Requires-Dist: dalla-data-processing[dedup,dedup-native,dev,pack,quality,readability,stem]; extra == "all"

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.4}/dalla_data_processing/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.0.2'
-__version_tuple__ = version_tuple = (0, 0, 2)
+__version__ = version = '0.0.4'
+__version_tuple__ = version_tuple = (0, 0, 4)
 
-__commit_id__ = commit_id = 'g68cee46a4'
+__commit_id__ = commit_id = 'g89939c5cc'

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.4}/dalla_data_processing/cli.py

@@ -651,7 +651,15 @@ def pack(
             tokenizer = RBPETokenizer.from_pretrained(config_data["tokenizer_path"])
         except ImportError:
             logger.error("Missing rbpe package")
-            logger.error("Install with: pip install rbpe")
+            logger.error(
+                "rbpe is not included in the default installation due to "
+                "dependency conflicts with camel-tools (transformers version requirements)"
+            )
+            logger.error("Install separately with: pip install rbpe")
+            logger.error(
+                "Note: Installing rbpe may require a separate environment "
+                "if you also use dedup/stem/quality features"
+            )
             sys.exit(1)
     else:
         try:

{dalla_data_processing-0.0.2/dalla_data_processing/deduplication/onion/src_sc → dalla_data_processing-0.0.4/dalla_data_processing/deduplication/onion/src}/Makefile

@@ -1,7 +1,7 @@
 include ../Makefile.config
 
 CC=g++
-CFLAGS=-Wall -O3
+CFLAGS=-Wall -O3 -std=c++11 -I/opt/homebrew/opt/google-sparsehash/include
 
 OBJS=version.o buzhash.o
 TARGETS=hashgen hashdup onion

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.4}/dalla_data_processing/deduplication/onion/src/Makefile.g

@@ -2,7 +2,7 @@ include ../Makefile.config
 
 CC=g++
 #CFLAGS=-Wall -O3
-CFLAGS=-Wall -g
+CFLAGS=-Wall -g -std=c++11 -I/opt/homebrew/opt/google-sparsehash/include
 
 OBJS=version.o buzhash.o
 TARGETS=hashgen hashdup onion

{dalla_data_processing-0.0.2/dalla_data_processing/deduplication/onion/src → dalla_data_processing-0.0.4/dalla_data_processing/deduplication/onion/src_sc}/Makefile

@@ -1,7 +1,7 @@
 include ../Makefile.config
 
 CC=g++
-CFLAGS=-Wall -O3
+CFLAGS=-Wall -O3 -std=c++11 -I/opt/homebrew/opt/google-sparsehash/include
 
 OBJS=version.o buzhash.o
 TARGETS=hashgen hashdup onion

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.4}/dalla_data_processing/deduplication/onion/src_sc/Makefile.g

@@ -2,7 +2,7 @@ include ../Makefile.config
 
 CC=g++
 #CFLAGS=-Wall -O3
-CFLAGS=-Wall -g
+CFLAGS=-Wall -g -std=c++11 -I/opt/homebrew/opt/google-sparsehash/include
 
 OBJS=version.o buzhash.o
 TARGETS=hashgen hashdup onion

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.4}/dalla_data_processing/quality/checker.py

@@ -11,6 +11,7 @@ from concurrent.futures import TimeoutError as FutureTimeoutError
 from types import MethodType
 from typing import Any
 
+from camel_tools.data.catalogue import Catalogue
 from camel_tools.disambig.bert import BERTUnfactoredDisambiguator
 from camel_tools.disambig.mle import MLEDisambiguator
 from datasets import Dataset
@@ -53,6 +54,25 @@ class QualityChecker:
 
     def _init_disambiguator(self):
         """Initialize and configure the disambiguator with caching."""
+        # Install required CAMeL Tools packages based on model type
+        logger.info("Checking CAMeL Tools data packages...")
+        catalogue = Catalogue.load_catalogue()
+
+        try:
+            catalogue.download_package("morphology-db-msa-r13")
+            catalogue.download_package("disambig-mle-calima-msa-r13")
+            logger.info("msa-r13 packages installed")
+        except Exception as e:
+            logger.warning(f"Package installation warning: {e}")
+
+        # Install BERT package if using BERT model
+        if self.model == "bert":
+            try:
+                catalogue.download_package("disambig-bert-unfactored-all")
+                logger.info("BERT package installed")
+            except Exception as e:
+                logger.warning(f"BERT package installation warning: {e}")
+
         if self.model == "mle":
             self.disambiguator = MLEDisambiguator.pretrained()
             logger.info("MLE disambiguator loaded")

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.4}/dalla_data_processing/stemming/stemmer.py

@@ -473,12 +473,19 @@ def stem_dataset(
     catalogue = Catalogue.load_catalogue()
     try:
         catalogue.download_package("morphology-db-msa-r13")
-        if model == "mle":
-            catalogue.download_package("disambig-mle-calima-msa-r13")
-        # For BERT, let it download automatically when pretrained() is called
-        logger.info("CAMeL Tools data packages ready")
+        catalogue.download_package("disambig-mle-calima-msa-r13")
+        logger.info("msa-r13 packages installed")
     except Exception as e:
-        logger.warning(f"Could not verify CAMeL packages: {e}")
+        logger.warning(f"Package installation warning: {e}")
+
+    if model == "bert":
+        try:
+            catalogue.download_package("disambig-bert-unfactored-all")
+            logger.info("BERT package installed")
+        except Exception as e:
+            logger.warning(f"BERT package installation warning: {e}")
+
+    logger.info("CAMeL Tools data packages ready")
 
     logger.info("Loading additional words lists...")
     words_dir = os.path.join(os.path.dirname(__file__), "data")
@@ -597,15 +604,21 @@ def stem(
     if not all(isinstance(t, str) for t in text_list):
         raise TypeError("All items in text list must be strings")
 
-    # Initialize disambiguator (cached globally if possible)
     logger.info(f"Initializing {model.upper()} disambiguator...")
     catalogue = Catalogue.load_catalogue()
     try:
         catalogue.download_package("morphology-db-msa-r13")
-        if model == "mle":
-            catalogue.download_package("disambig-mle-calima-msa-r13")
+        catalogue.download_package("disambig-mle-calima-msa-r13")
+        logger.info("msa-r13 packages installed")
     except Exception as e:
-        logger.warning(f"Could not verify CAMeL packages: {e}")
+        logger.warning(f"Package installation warning: {e}")
+
+    if model == "bert":
+        try:
+            catalogue.download_package("disambig-bert-unfactored-all")
+            logger.info("BERT package installed")
+        except Exception as e:
+            logger.warning(f"BERT package installation warning: {e}")
 
     if model == "mle":
         disambiguator = MLEDisambiguator.pretrained("calima-msa-r13", cache_size=1_000_000)

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.4}/dalla_data_processing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dalla-data-processing
-Version: 0.0.2
+Version: 0.0.4
 Summary: data processing pipeline with deduplication, stemming, quality checking, and readability scoring, used for the DALLA Models
 Author-email: Hadi Hamoud <hhamoud@dohainstitute.edu.qa>, Digital Research Unit - Arab Center <dru@dohainstitute.edu.qa>
 Project-URL: Homepage, https://github.com/U4RASD/dalla-data-processing
@@ -11,10 +11,13 @@ Keywords: arabic,nlp,data-processing,deduplication,stemming,readability,quality
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Text Processing :: Linguistic
-Requires-Python: >=3.12
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: datasets>=2.14.0
 Requires-Dist: transformers>=4.30.0
@@ -28,18 +31,17 @@ Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
 Requires-Dist: ruff>=0.1.0; extra == "dev"
 Requires-Dist: pre-commit>=3.0.0; extra == "dev"
 Provides-Extra: dedup
-Requires-Dist: camel-tools>=1.5.0; extra == "dedup"
+Requires-Dist: camel-tools==1.5.7; extra == "dedup"
 Provides-Extra: dedup-native
 Requires-Dist: cffi>=1.15.0; extra == "dedup-native"
 Provides-Extra: stem
-Requires-Dist: camel-tools>=1.5.0; extra == "stem"
+Requires-Dist: camel-tools==1.5.7; extra == "stem"
 Provides-Extra: quality
-Requires-Dist: camel-tools>=1.5.0; extra == "quality"
+Requires-Dist: camel-tools==1.5.7; extra == "quality"
 Provides-Extra: readability
 Requires-Dist: textstat>=0.7.0; extra == "readability"
 Provides-Extra: pack
 Requires-Dist: sentencepiece>=0.2.0; extra == "pack"
-Requires-Dist: rbpe; extra == "pack"
 Requires-Dist: pyyaml; extra == "pack"
 Provides-Extra: all
 Requires-Dist: dalla-data-processing[dedup,dedup-native,dev,pack,quality,readability,stem]; extra == "all"

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.4}/dalla_data_processing.egg-info/SOURCES.txt

@@ -14,6 +14,7 @@ dalla_data_processing.egg-info/PKG-INFO
 dalla_data_processing.egg-info/SOURCES.txt
 dalla_data_processing.egg-info/dependency_links.txt
 dalla_data_processing.egg-info/entry_points.txt
+dalla_data_processing.egg-info/not-zip-safe
 dalla_data_processing.egg-info/requires.txt
 dalla_data_processing.egg-info/top_level.txt
 dalla_data_processing/core/README.md

dalla_data_processing-0.0.4/dalla_data_processing.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.4}/dalla_data_processing.egg-info/requires.txt

@@ -9,7 +9,7 @@ structlog>=24.0.0
 dalla-data-processing[dedup,dedup-native,dev,pack,quality,readability,stem]
 
 [dedup]
-camel-tools>=1.5.0
+camel-tools==1.5.7
 
 [dedup-native]
 cffi>=1.15.0
@@ -22,14 +22,13 @@ pre-commit>=3.0.0
 
 [pack]
 sentencepiece>=0.2.0
-rbpe
 pyyaml
 
 [quality]
-camel-tools>=1.5.0
+camel-tools==1.5.7
 
 [readability]
 textstat>=0.7.0
 
 [stem]
-camel-tools>=1.5.0
+camel-tools==1.5.7

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.4}/pyproject.toml

@@ -11,13 +11,16 @@ authors = [
     {name = "Digital Research Unit - Arab Center", email = "dru@dohainstitute.edu.qa"}
 ]
 readme = "README.md"
-requires-python = ">=3.12"
+requires-python = ">=3.10"
 keywords = ["arabic", "nlp", "data-processing", "deduplication", "stemming", "readability", "quality"]
 classifiers = [
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
     "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
     "Topic :: Text Processing :: Linguistic",
 ]
@@ -39,23 +42,24 @@ dev = [
     "pre-commit>=3.0.0",
 ]
 dedup = [
-    "camel-tools>=1.5.0",
+    "camel-tools==1.5.7",
 ]
 dedup-native = [
     "cffi>=1.15.0",
 ]
 stem = [
-    "camel-tools>=1.5.0",
+    "camel-tools==1.5.7",
 ]
 quality = [
-    "camel-tools>=1.5.0",
+    "camel-tools==1.5.7",
 ]
 readability = [
     "textstat>=0.7.0",
 ]
 pack = [
     "sentencepiece>=0.2.0",
-    "rbpe",
+    # "rbpe",  # excluded due to transformers version conflict with camel-tools
+                # users should install separately if needed: pip install rbpe
     "pyyaml",
 ]
 all = [
@@ -75,6 +79,8 @@ Repository = "https://github.com/U4RASD/dalla-data-processing"
 packages = ["dalla_data_processing", "dalla_data_processing.core", "dalla_data_processing.deduplication", "dalla_data_processing.packing", "dalla_data_processing.stemming", "dalla_data_processing.quality", "dalla_data_processing.readability", "dalla_data_processing.utils"]
 include-package-data = true
 
+zip-safe = false
+
 [tool.setuptools.package-data]
 dalla_data_processing = ["py.typed"]
 "dalla_data_processing.stemming" = ["data/*.txt"]
@@ -82,7 +88,7 @@ dalla_data_processing = ["py.typed"]
 
 [tool.ruff]
 line-length = 100
-target-version = "py312"
+target-version = "py310"
 src = ["dalla_data_processing"]
 
 [tool.ruff.lint]

{dalla_data_processing-0.0.2 → dalla_data_processing-0.0.4}/scripts/build_onion.sh

@@ -63,14 +63,16 @@ if [[ "$OSTYPE" == "darwin"* ]]; then
     # On macOS, add Homebrew paths for sparsehash
     if command -v brew &> /dev/null; then
         BREW_PREFIX=$(brew --prefix)
-        EXTRA_CFLAGS="-I${BREW_PREFIX}/include"
+        SPARSEHASH_PREFIX=$(brew --prefix google-sparsehash 2>/dev/null || echo "${BREW_PREFIX}")
+        EXTRA_CFLAGS="-I${SPARSEHASH_PREFIX}/include"
         echo -e "${YELLOW}Using Homebrew prefix: ${BREW_PREFIX}${NC}"
+        echo -e "${YELLOW}Using sparsehash include: ${SPARSEHASH_PREFIX}/include${NC}"
     fi
 fi
 
 # Build onion
 echo -e "${YELLOW}Compiling Onion...${NC}"
-if make CFLAGS="-Wall -O3 ${EXTRA_CFLAGS}"; then
+if make CFLAGS="-Wall -O3 -std=c++11 ${EXTRA_CFLAGS}"; then
     echo -e "${GREEN}✓ Compilation successful${NC}"
 else
     echo -e "${RED}✗ Compilation failed${NC}"