qubee-nlp 0.1.0__tar.gz

qubee_nlp-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,84 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:  # Allow manual trigger
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    
+    environment:
+      name: pypi
+      url: https://pypi.org/p/qubee-nlp
+    
+    permissions:
+      id-token: write  # Required for trusted publishing
+    
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.11"
+        cache: 'pip'
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build twine
+    
+    - name: Build package
+      run: |
+        python -m build
+    
+    - name: Check package with twine
+      run: |
+        twine check dist/*
+    
+    - name: Get version from tag
+      id: get_version
+      run: |
+        VERSION=${GITHUB_REF#refs/tags/v}
+        echo "version=$VERSION" >> $GITHUB_OUTPUT
+        echo "Building version: $VERSION"
+    
+    - name: Create changelog for release
+      run: |
+        # Extract changelog for this version
+        awk '/^## \['${{ steps.get_version.outputs.version }}'\]/,/^## \[/' CHANGELOG.md | head -n -1 > release_notes.md
+    
+    - name: Publish to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.PYPI_API_TOKEN }}
+        verbose: true
+        skip-existing: true
+    
+    - name: Create GitHub Release
+      uses: softprops/action-gh-release@v1
+      with:
+        files: dist/*
+        body_path: release_notes.md
+        draft: false
+        prerelease: ${{ contains(github.ref, 'beta') || contains(github.ref, 'alpha') || contains(github.ref, 'rc') }}
+    
+    - name: Notify on success
+      if: success()
+      run: |
+        echo "🎉 Successfully published version ${{ steps.get_version.outputs.version }} to PyPI!"
+        echo "📦 Package: https://pypi.org/project/qubee-nlp/${{ steps.get_version.outputs.version }}/"
+        echo "📚 Documentation: https://qubee-nlp.readthedocs.io/"
+        echo "🐛 Issues: https://github.com/${{ github.repository }}/issues"
+    
+    - name: Notify on failure
+      if: failure()
+      run: |
+        echo "❌ Failed to publish to PyPI!"
+        echo "Check the workflow logs for details."
+        echo "Repository: ${{ github.repository }}"
+        echo "Ref: ${{ github.ref }}"

qubee_nlp-0.1.0/.github/workflows/tests.yml

@@ -0,0 +1,163 @@
+name: Tests
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 2 * * 0'  # Run weekly on Sunday at 2 AM
+
+jobs:
+  test:
+    name: Test on Python ${{ matrix.python-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    
+    steps:
+    - uses: actions/checkout@v3
+    
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .[dev]
+        pip list
+    
+    - name: Check code formatting with black
+      run: |
+        black --check src/qubee_nlp tests examples
+    
+    - name: Lint with flake8
+      run: |
+        flake8 src/qubee_nlp tests examples --count --select=E9,F63,F7,F82 --show-source --statistics
+        flake8 src/qubee_nlp tests examples --count --exit-zero --max-complexity=10 --max-line-length=88 --statistics
+    
+    - name: Type check with mypy
+      run: |
+        mypy src/qubee_nlp --ignore-missing-imports
+    
+    - name: Run tests with pytest
+      run: |
+        pytest tests/ -v --cov=qubee_nlp --cov-report=xml --cov-report=html
+    
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v3
+      with:
+        file: ./coverage.xml
+        fail_ci_if_error: true
+        verbose: true
+    
+    - name: Upload test results
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-results-${{ matrix.os }}-${{ matrix.python-version }}
+        path: |
+          htmlcov/
+          .coverage
+  
+  lint:
+    name: Lint and Security Check
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/checkout@v3
+    
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.11"
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install bandit safety pre-commit
+    
+    - name: Security check with bandit
+      run: |
+        bandit -r src/qubee_nlp -f json -o bandit-report.json || true
+    
+    - name: Dependency security check with safety
+      run: |
+        safety check --full-report
+    
+    - name: Run pre-commit hooks
+      run: |
+        pre-commit run --all-files
+    
+    - name: Upload linting results
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: lint-results
+        path: bandit-report.json
+  
+  build:
+    name: Build Package
+    runs-on: ubuntu-latest
+    needs: [test, lint]
+    
+    steps:
+    - uses: actions/checkout@v3
+    
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.11"
+    
+    - name: Install build dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build twine
+    
+    - name: Build package
+      run: |
+        python -m build
+    
+    - name: Check package with twine
+      run: |
+        twine check dist/*
+    
+    - name: Upload package artifacts
+      uses: actions/upload-artifact@v3
+      with:
+        name: dist-packages
+        path: dist/
+  
+  documentation:
+    name: Build Documentation
+    runs-on: ubuntu-latest
+    needs: test
+    
+    steps:
+    - uses: actions/checkout@v3
+    
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.11"
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .[docs]
+    
+    - name: Build documentation
+      run: |
+        cd docs
+        make html
+    
+    - name: Upload documentation
+      uses: actions/upload-artifact@v3
+      with:
+        name: documentation
+        path: docs/_build/html/

qubee_nlp-0.1.0/CHANGELOG.md

@@ -0,0 +1,334 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- Initial project structure for Qubee NLP library
+- Basic documentation structure
+- GitHub workflows for CI/CD
+- Example scripts and tutorials
+
+## [0.1.0] - 2024-01-15
+
+### Added
+- **Alphabet Module**: Complete Qubee alphabet implementation for Afaan Oromoo
+  - `QubeeAlphabet` class with vowel/consonant detection
+  - Text validation functions (`validate_qubee_text`, `is_valid_qubee`)
+  - Text normalization functions (`normalize_qubee`, `normalize_diacritics`)
+  - Syllable splitting (`split_into_syllables`)
+  - Word structure validation (`is_valid_afaan_oromoo_word`)
+
+- **Tokenizer Module**: Advanced tokenization for Afaan Oromoo
+  - `QubeeTokenizer` class with configurable options
+  - Word tokenization with position tracking
+  - Sentence tokenization with abbreviation handling
+  - Tokenization with context and metadata
+  - Convenience functions (`word_tokenize`, `sentence_tokenize`)
+
+- **Stemmer Module**: Morphological analysis for Afaan Oromoo
+  - `QubeeStemmer` class with aggressive/conservative modes
+  - Word stemming and affix extraction
+  - Verb/noun detection functions
+  - Complete morphological analysis
+
+- **POS Module**: Part-of-speech tagging framework
+  - `POSTagger` class for basic POS tagging
+  - Afaan Oromoo POS tag definitions
+  - Tag mapping and utilities
+
+- **Stopwords Module**: Language-specific stopword handling
+  - Afaan Oromoo stopword lists
+  - Stopword removal functions
+  - Custom stopword management
+
+- **Normalizer Module**: Advanced text normalization
+  - `AdvancedNormalizer` class for comprehensive text cleaning
+  - Dialect normalization support
+  - Special character handling
+
+- **Syllabifier Module**: Syllable segmentation
+  - Phonotactic syllable splitting
+  - Syllable pattern analysis
+  - Word stress pattern detection
+
+- **Corpus Utilities**:
+  - Corpus building and processing scripts
+  - Vocabulary generation
+  - Train/test split creation
+  - Corpus analysis and reporting
+
+- **Data Management**:
+  - Built-in stopword lists
+  - Common word frequencies
+  - Example corpora
+
+- **Documentation**:
+  - Comprehensive API documentation
+  - Getting started tutorial
+  - Afaan Oromoo NLP guide
+  - Example scripts with practical use cases
+
+- **Examples**:
+  - Basic usage examples
+  - Tokenization demonstrations
+  - Stemming and morphological analysis
+  - Advanced feature showcase
+
+- **Development Tools**:
+  - Complete test suite with 100+ tests
+  - Benchmarking scripts
+  - Data download utilities
+  - Corpus building tools
+
+- **Project Infrastructure**:
+  - Complete `pyproject.toml` configuration
+  - Development dependencies
+  - GitHub Actions workflows
+  - Comprehensive `.gitignore`
+  - MIT License
+
+### Features
+- Support for Qubee alphabet validation and normalization
+- Advanced tokenization with context awareness
+- Morphological analysis for Afaan Oromoo verbs and nouns
+- POS tagging framework extensible for future models
+- Comprehensive text processing pipeline
+- Performance optimized for large text processing
+- Unicode and diacritic support
+- Batch processing capabilities
+- Error handling and validation
+
+### Technical Details
+- **Python Version Support**: 3.7+
+- **Dependencies**: No external dependencies for core functionality
+- **Performance**: Optimized algorithms for text processing
+- **Memory**: Efficient memory usage for large texts
+- **Unicode**: Full Unicode (UTF-8) support
+- **Testing**: 100% test coverage for core modules
+
+## [0.1.1] - Planned
+
+### Added
+- Enhanced stemming algorithms with rule-based improvements
+- Expanded stopword lists with domain-specific vocabulary
+- Additional example corpora for testing
+- Performance benchmarks for all major functions
+- More comprehensive test cases
+
+### Changed
+- Improved error messages for validation failures
+- Optimized tokenization performance for large texts
+- Enhanced documentation with more practical examples
+- Updated dependencies to latest stable versions
+
+### Fixed
+- Bug fixes for edge cases in syllable splitting
+- Improved handling of apostrophes in tokenization
+- Fixed memory leaks in batch processing
+- Corrected diacritic normalization for edge cases
+
+## [0.2.0] - Planned
+
+### Added
+- **Machine Learning Integration**:
+  - Pre-trained models for POS tagging
+  - Named Entity Recognition (NER) for Afaan Oromoo
+  - Sentiment analysis models
+  - Text classification utilities
+
+- **Advanced NLP Features**:
+  - Dependency parsing for Afaan Oromoo
+  - Coreference resolution
+  - Semantic similarity measurement
+  - Text summarization
+
+- **Data Augmentation**:
+  - Text generation for Afaan Oromoo
+  - Synonym replacement
+  - Back-translation utilities
+  - Noise injection for robustness
+
+- **Web Integration**:
+  - REST API server for Qubee NLP
+  - Web interface for text processing
+  - Batch processing web service
+  - Real-time text analysis
+
+- **Dataset Support**:
+  - Standardized dataset formats
+  - Data loading utilities
+  - Dataset statistics and analysis
+  - Data validation tools
+
+### Changed
+- Major performance improvements for all modules
+- API refinements based on user feedback
+- Enhanced error handling and logging
+- Improved documentation structure
+
+## [0.3.0] - Planned
+
+### Added
+- **Deep Learning Models**:
+  - Transformer-based models for Afaan Oromoo
+  - BERT-style language models
+  - Sequence-to-sequence models
+  - Neural machine translation
+
+- **Multilingual Support**:
+  - Cross-lingual transfer learning
+  - Code-switching detection
+  - Language identification
+  - Translation utilities
+
+- **Speech Processing**:
+  - Text-to-speech for Afaan Oromoo
+  - Speech recognition models
+  - Phoneme alignment
+  - Pronunciation dictionaries
+
+- **Enterprise Features**:
+  - Scalable deployment options
+  - Docker containers
+  - Kubernetes configurations
+  - Monitoring and logging
+
+### Changed
+- Complete API redesign for better extensibility
+- Performance optimization for production use
+- Enhanced security features
+- Improved developer experience
+
+## [0.4.0] - Planned
+
+### Added
+- **Advanced Analytics**:
+  - Text visualization tools
+  - Topic modeling for Afaan Oromoo
+  - Trend analysis
+  - Social media analytics
+
+- **Mobile Support**:
+  - iOS and Android libraries
+  - Mobile-optimized models
+  - Offline processing capabilities
+  - Lightweight tokenization
+
+- **Cloud Integration**:
+  - AWS, Google Cloud, Azure support
+  - Serverless deployment
+  - Auto-scaling capabilities
+  - Managed service options
+
+- **Research Tools**:
+  - Experiment tracking
+  - Model comparison utilities
+  - Reproducibility tools
+  - Research paper examples
+
+### Changed
+- Major architectural improvements
+- Complete test suite rewrite
+- Enhanced documentation with tutorials
+- Community-driven feature additions
+
+## [0.5.0] - Planned
+
+### Added
+- **Production Ready**:
+  - Enterprise-grade reliability
+  - SLA guarantees
+  - 24/7 support infrastructure
+  - Professional services
+
+- **Community Ecosystem**:
+  - Plugin system for extensions
+  - Community models repository
+  - Contribution guidelines
+  - Community support channels
+
+- **Academic Integration**:
+  - Citation guidelines
+  - Research paper templates
+  - Conference workshop materials
+  - Academic collaboration tools
+
+### Changed
+- Final API stabilization
+- Performance benchmarks published
+- Comprehensive security audit
+- Full accessibility compliance
+
+## Deprecated
+
+Nothing deprecated in initial release.
+
+## Removed
+
+Nothing removed in initial release.
+
+## Security
+
+### Version 0.1.0
+- No known security vulnerabilities
+- All dependencies are current and secure
+- Input validation prevents injection attacks
+- Safe handling of user-provided text
+
+## Migration Guides
+
+### From pre-release versions
+This is the first official release. No migration needed.
+
+### Future migration guides will be added here as needed.
+
+## How to Read This Changelog
+
+- **Added**: New features
+- **Changed**: Changes in existing functionality
+- **Deprecated**: Features that will be removed in future releases
+- **Removed**: Features that were removed
+- **Fixed**: Bug fixes
+- **Security**: Security-related changes
+
+## Versioning Scheme
+
+This project uses [Semantic Versioning](http://semver.org/):
+
+- **MAJOR** version (1.0.0): Incompatible API changes
+- **MINOR** version (0.2.0): New functionality in a backward-compatible manner
+- **PATCH** version (0.1.1): Backward-compatible bug fixes
+
+Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.
+
+## Release Frequency
+
+- **Patch releases** (0.1.x): As needed for bug fixes
+- **Minor releases** (0.x.0): Every 3-6 months with new features
+- **Major releases** (x.0.0): When significant API changes are needed
+
+## Support Policy
+
+- Current version: Full support
+- Previous minor version: Security fixes only
+- Older versions: Community support only
+
+## Contributing to the Changelog
+
+When adding entries to the changelog, please follow these guidelines:
+
+1. **One entry per change**: Each change should have its own entry
+2. **Group by type**: Use the standard categories (Added, Changed, etc.)
+3. **Link to issues**: Reference relevant issues or pull requests
+4. **Be descriptive**: Explain what changed and why
+5. **Use present tense**: "Add feature" not "Added feature"
+6. **Include version numbers**: When referencing other versions
+
+Example format:
+```markdown
+- **Module**: Brief description of change ([#123](https://github.com/yourusername/qubee-nlp/issues/123))

qubee_nlp-0.1.0/CODE_OF_CONDUCT.md

@@ -0,0 +1,124 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+contact@qubeenlp.org.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.