dalla-data-processing 0.0.1__tar.gz → 0.0.3__tar.gz

dalla_data_processing-0.0.3/.dockerignore

@@ -0,0 +1,63 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+venv/
+env/
+ENV/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+
+# Git
+.git/
+.gitignore
+
+# Documentation
+docs/
+*.md
+!README.md
+
+# CI/CD
+.github/
+
+# Temporary
+tmp/
+temp/
+*.tmp
+
+# Data files (users should mount these)
+*.arrow
+*.parquet
+*.csv
+*.json
+data/

dalla_data_processing-0.0.3/.github/workflows/ci.yml

@@ -0,0 +1,116 @@
+name: CI
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main, develop ]
+
+jobs:
+  test:
+    name: Test Python ${{ matrix.python-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: ['3.12']
+    
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0  # Fetch full history for setuptools-scm
+    
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    
+    - name: Install UV
+      run: |
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+    
+    - name: Install dependencies
+      run: |
+        uv pip install --system -e ".[dev]"
+    
+    - name: Lint with ruff
+      run: |
+        ruff check dalla_data_processing/ --output-format=github
+        ruff format --check dalla_data_processing/
+    
+    - name: Upload coverage
+      if: matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest'
+      uses: codecov/codecov-action@v3
+      with:
+        files: ./coverage.xml
+        fail_ci_if_error: false
+
+  build-onion:
+    name: Build Onion Binary for ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+            artifact_name: onion-linux-x86_64
+          - os: macos-latest
+            artifact_name: onion-darwin-x86_64
+    
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0  # Fetch full history for setuptools-scm
+    
+    - name: Install build dependencies
+      run: |
+        if [ "$RUNNER_OS" == "Linux" ]; then
+          sudo apt-get update
+          sudo apt-get install -y build-essential libsparsehash-dev
+        elif [ "$RUNNER_OS" == "macOS" ]; then
+          brew install google-sparsehash
+        fi
+    
+    - name: Build Onion
+      run: |
+        chmod +x scripts/build_onion.sh
+        ./scripts/build_onion.sh
+    
+    - name: Upload Onion Binary
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ matrix.artifact_name }}
+        path: dalla_data_processing/deduplication/bin/onion-*
+        if-no-files-found: error
+
+  quality-checks:
+    name: Code Quality
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0  # Fetch full history for setuptools-scm
+    
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    
+    - name: Install UV
+      run: |
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+    
+    - name: Install pre-commit
+      run: |
+        uv pip install --system pre-commit
+    
+    - name: Clean pre-commit cache
+      run: |
+        pre-commit clean
+    
+    - name: Run pre-commit hooks
+      run: |
+        pre-commit run --all-files

dalla_data_processing-0.0.3/.github/workflows/release.yml

@@ -0,0 +1,253 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+
+permissions:
+  contents: write
+
+jobs:
+  build-onion:
+    name: Build Onion Binary for ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+            artifact_name: onion-linux-x86_64
+          - os: macos-latest
+            artifact_name: onion-darwin-x86_64
+    
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0  # Fetch full history for setuptools-scm
+    
+    - name: Install build dependencies
+      run: |
+        if [ "$RUNNER_OS" == "Linux" ]; then
+          sudo apt-get update
+          sudo apt-get install -y build-essential libsparsehash-dev
+        elif [ "$RUNNER_OS" == "macOS" ]; then
+          brew install google-sparsehash
+        fi
+    
+    - name: Build Onion
+      run: |
+        chmod +x scripts/build_onion.sh
+        ./scripts/build_onion.sh
+    
+    - name: Upload Onion Binary
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ matrix.artifact_name }}
+        path: dalla_data_processing/deduplication/bin/onion-*
+        if-no-files-found: error
+
+  build-wheels:
+    name: Build wheels on ${{ matrix.os }}
+    needs: build-onion
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+            onion_artifact: onion-linux-x86_64
+          - os: macos-latest
+            onion_artifact: onion-darwin-x86_64
+          - os: windows-latest
+            onion_artifact: none
+    
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0  # Fetch full history for setuptools-scm
+    
+    - name: Create bin directory
+      run: mkdir -p dalla_data_processing/deduplication/bin
+      shell: bash
+    
+    - name: Download Onion binary
+      if: matrix.onion_artifact != 'none'
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ matrix.onion_artifact }}
+        path: dalla_data_processing/deduplication/bin/
+    
+    - name: Set binary permissions
+      if: matrix.onion_artifact != 'none'
+      run: chmod +x dalla_data_processing/deduplication/bin/onion-*
+      shell: bash
+    
+    - name: List binary files
+      if: matrix.onion_artifact != 'none'
+      run: ls -lah dalla_data_processing/deduplication/bin/
+      shell: bash
+    
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    
+    - name: Install build dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build twine
+    
+    - name: Build wheel
+      run: python -m build
+    
+    - name: List wheel contents
+      run: |
+        python -m zipfile -l dist/*.whl | head -50
+    
+    - name: Upload wheels
+      uses: actions/upload-artifact@v4
+      with:
+        name: wheels-${{ matrix.os }}
+        path: dist/*
+
+  build-sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0  # Fetch full history for setuptools-scm
+    
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    
+    - name: Build sdist
+      run: |
+        pip install build
+        python -m build --sdist
+    
+    - name: Upload sdist
+      uses: actions/upload-artifact@v4
+      with:
+        name: sdist
+        path: dist/*.tar.gz
+
+  publish-pypi:
+    name: Publish to PyPI
+    needs: [build-wheels, build-sdist]
+    runs-on: ubuntu-latest
+    environment: release
+    
+    steps:
+    # Only use Linux wheel to avoid conflicts
+    - uses: actions/download-artifact@v4
+      with:
+        name: wheels-ubuntu-latest
+        path: dist/
+    
+    - uses: actions/download-artifact@v4
+      with:
+        name: sdist
+        path: dist/
+    
+    - name: List distribution files
+      run: ls -lh dist/
+    
+    - name: Verify wheel integrity
+      run: |
+        pip install twine
+        twine check dist/*
+    
+    - name: Publish to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.PYPI_API_TOKEN }}
+        skip-existing: true
+
+  create-release:
+    name: Create GitHub Release
+    needs: [build-wheels, build-sdist]
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0  # Fetch full history for setuptools-scm
+    
+    - uses: actions/download-artifact@v4
+      with:
+        pattern: wheels-*
+        merge-multiple: true
+        path: dist/
+    
+    - uses: actions/download-artifact@v4
+      with:
+        name: sdist
+        path: dist/
+    
+    - name: Extract version from tag
+      id: version
+      run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
+    
+    - name: Generate release notes
+      id: notes
+      run: |
+        # Extract changelog for this version
+        VERSION=${{ steps.version.outputs.VERSION }}
+        echo "NOTES<<EOF" >> $GITHUB_OUTPUT
+        echo "## Dalla Data Processing v${VERSION}" >> $GITHUB_OUTPUT
+        echo "" >> $GITHUB_OUTPUT
+        echo "### Installation" >> $GITHUB_OUTPUT
+        echo "\`\`\`bash" >> $GITHUB_OUTPUT
+        echo "pip install dalla-data-processing==${VERSION}" >> $GITHUB_OUTPUT
+        echo "\`\`\`" >> $GITHUB_OUTPUT
+        echo "" >> $GITHUB_OUTPUT
+        echo "### What's Changed" >> $GITHUB_OUTPUT
+        # TODO: Add actual changelog extraction
+        echo "See CHANGELOG.md for details" >> $GITHUB_OUTPUT
+        echo "EOF" >> $GITHUB_OUTPUT
+    
+    - name: Create Release
+      uses: softprops/action-gh-release@v1
+      with:
+        files: dist/*
+        body: ${{ steps.notes.outputs.NOTES }}
+        draft: false
+        prerelease: false
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  # Commented out for now - uncomment when ready to publish Docker images
+  # build-docker:
+  #   name: Build and Push Docker Image
+  #   needs: [publish-pypi]
+  #   runs-on: ubuntu-latest
+  #   
+  #   steps:
+  #   - uses: actions/checkout@v4
+  #   
+  #   - name: Set up Docker Buildx
+  #     uses: docker/setup-buildx-action@v3
+  #   
+  #   - name: Log in to Docker Hub
+  #     uses: docker/login-action@v3
+  #     with:
+  #       username: ${{ secrets.DOCKER_USERNAME }}
+  #       password: ${{ secrets.DOCKER_PASSWORD }}
+  #   
+  #   - name: Extract version
+  #     id: version
+  #     run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
+  #   
+  #   - name: Build and push
+  #     uses: docker/build-push-action@v5
+  #     with:
+  #       context: .
+  #       push: true
+  #       tags: |
+  #         ${{ secrets.DOCKER_USERNAME }}/dalla-data-processing:latest
+  #         ${{ secrets.DOCKER_USERNAME }}/dalla-data-processing:${{ steps.version.outputs.VERSION }}
+  #       cache-from: type=gha
+  #       cache-to: type=gha,mode=max

dalla_data_processing-0.0.3/.gitignore

@@ -0,0 +1,69 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# setuptools-scm version file
+dalla_data_processing/_version.py
+
+# Virtual environments
+venv/
+env/
+ENV/
+.venv
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+
+# Type checking
+.mypy_cache/
+.pytype/
+.pyre/
+
+# Distribution
+*.whl
+
+# Native binaries
+*.o
+*.a
+*.dylib
+*.dll
+
+# Data
+*.arrow
+*.parquet
+
+# Logs
+*.log
+logs/
+
+# OS
+.DS_Store
+Thumbs.db

dalla_data_processing-0.0.3/.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+# Pre-commit hooks for dalla-data-processing
+# Install: pre-commit install
+# Run manually: pre-commit run --all-files
+
+repos:
+  # Ruff - Fast Python linter and formatter
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.15
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format
+
+# Configuration
+default_language_version:
+  python: python3.12

dalla_data_processing-0.0.3/MANIFEST.in

@@ -0,0 +1,8 @@
+include README.md
+include LICENSE
+recursive-include dalla_data_processing *.py
+recursive-include dalla_data_processing/stemming/data *.txt
+recursive-include dalla_data_processing/deduplication/bin *
+recursive-include dalla_data_processing/deduplication/onion *.c *.h Makefile*
+global-exclude __pycache__
+global-exclude *.py[co]

dalla_data_processing-0.0.3/PKG-INFO

@@ -0,0 +1,147 @@
+Metadata-Version: 2.4
+Name: dalla-data-processing
+Version: 0.0.3
+Summary: data processing pipeline with deduplication, stemming, quality checking, and readability scoring, used for the DALLA Models
+Author-email: Hadi Hamoud <hhamoud@dohainstitute.edu.qa>, Digital Research Unit - Arab Center <dru@dohainstitute.edu.qa>
+Project-URL: Homepage, https://github.com/U4RASD/dalla-data-processing
+Project-URL: Documentation, https://github.com/U4RASD/dalla-data-processing#readme
+Project-URL: Repository, https://github.com/U4RASD/dalla-data-processing
+Project-URL: Bug Tracker, https://github.com/U4RASD/dalla-data-processing/issues
+Keywords: arabic,nlp,data-processing,deduplication,stemming,readability,quality
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: <3.13,>=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: datasets>=2.14.0
+Requires-Dist: transformers>=4.30.0
+Requires-Dist: click>=8.0.0
+Requires-Dist: tqdm>=4.65.0
+Requires-Dist: pyarrow>=12.0.0
+Requires-Dist: structlog>=24.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: pre-commit>=3.0.0; extra == "dev"
+Provides-Extra: dedup
+Requires-Dist: camel-tools==1.5.7; extra == "dedup"
+Provides-Extra: dedup-native
+Requires-Dist: cffi>=1.15.0; extra == "dedup-native"
+Provides-Extra: stem
+Requires-Dist: camel-tools==1.5.7; extra == "stem"
+Provides-Extra: quality
+Requires-Dist: camel-tools==1.5.7; extra == "quality"
+Provides-Extra: readability
+Requires-Dist: textstat>=0.7.0; extra == "readability"
+Provides-Extra: pack
+Requires-Dist: sentencepiece>=0.2.0; extra == "pack"
+Requires-Dist: pyyaml; extra == "pack"
+Provides-Extra: all
+Requires-Dist: dalla-data-processing[dedup,dedup-native,dev,pack,quality,readability,stem]; extra == "all"
+
+# Dalla Data Processing (dalla-dp)
+
+A comprehensive Arabic data processing pipeline with deduplication, stemming, quality checking, and readability scoring, used for the DALLA Models.
+
+## Compatibility
+
+- **Linux**: Fully supported
+- **macOS**: Fully supported (Intel or through rosetta)
+- **Windows**: Supported through WSL (Windows Subsystem for Linux) only, for native windows: manual build from source works for deduplication.
+
+## Installation
+
+### Quick Start (All Features)
+
+For most users, install with all features enabled:
+
+<b>Using uv</b>
+
+```bash
+uv pip install "dalla-data-processing[all]"
+```
+
+<b>Using pip</b>
+
+```bash
+pip install "dalla-data-processing[all]"
+```
+
+### Modular Installation (Advanced)
+
+Install only the components you need to keep dependencies minimal:
+
+```bash
+# Base installation (no processing features, only core dependencies)
+pip install dalla-data-processing
+
+# Install specific features
+pip install "dalla-data-processing[dedup]"        # Deduplication only
+pip install "dalla-data-processing[stem]"         # Stemming only
+pip install "dalla-data-processing[quality]"      # Quality checking only
+pip install "dalla-data-processing[readability]"  # Readability scoring only
+pip install "dalla-data-processing[pack]"         # Dataset packing only
+
+# Combine multiple features
+pip install "dalla-data-processing[dedup,stem,quality]"
+```
+
+### Development Installation
+
+<b>From Source (with uv - recommended)</b>
+
+```bash
+git clone https://github.com/U4RASD/dalla-data-processing.git
+cd dalla-data-processing
+
+# Install all features and dev dependencies
+uv sync --all-extras
+
+# Or install with specific extras only
+uv sync --extra dedup --extra stem
+```
+
+<b>From Source (with pip)</b>
+
+```bash
+git clone https://github.com/U4RASD/dalla-data-processing.git
+cd dalla-data-processing
+
+# Install with all features for development
+pip install -e ".[all,dev]"
+```
+
+## Components
+
+> **Note:** Each component requires its corresponding extra to be installed. Install with `[all]` to enable all features, or see [Modular Installation](#modular-installation-advanced) to install only what you need.
+
+### 1. [Deduplication](dalla_data_processing/deduplication/README.md)
+Detect and remove duplicate or near-duplicate documents from your datasets using the Onion algorithm.
+- **Requires:** `[dedup]` extra
+
+### 2. [Stemming](dalla_data_processing/stemming/README.md)
+Apply morphological analysis and stemming using CAMeL Tools.
+- **Requires:** `[stem]` extra
+
+### 3. [Quality Checking](dalla_data_processing/quality/README.md)
+Check text quality using morphological analysis to detect errors and foreign words.
+- **Requires:** `[quality]` extra
+
+### 4. [Readability Scoring](dalla_data_processing/readability/README.md)
+Calculate readability scores using Flesch Reading Ease and Osman methods.
+Contains also ranking according to both scores
+- **Requires:** `[readability]` extra
+
+### 5. [Dataset Packing](dalla_data_processing/packing/README.md)
+Pack and prepare datasets for training.
+- **Requires:** `[pack]` extra
+
+## Links
+
+- Homepage: https://github.com/U4RASD/dalla-data-processing
+- Issues: https://github.com/U4RASD/dalla-data-processing/issues
+- Documentation: https://github.com/U4RASD/dalla-data-processing#readme