rust-crate-pipeline 1.2.6__py3-none-any.whl → 1.3.1__py3-none-any.whl

rust_crate_pipeline/utils/logging_utils.py

@@ -4,63 +4,80 @@ import time
 import psutil
 import logging
 from functools import wraps
-from typing import Optional
+from typing import Any, Callable, Dict, Optional, Union
 
-def configure_logging(log_dir: Optional[str] = None, log_level: int = logging.INFO) -> logging.Logger:
+
+def configure_logging(
+    log_dir: Optional[str] = None, log_level: int = logging.INFO
+) -> logging.Logger:
     """
     Configure global logging with file and console handlers
-    
+
     Args:
         log_dir: Directory for log files (defaults to current directory)
         log_level: Logging level (default: INFO)
-    
+
     Returns:
         Root logger instance
     """
     logger = logging.getLogger()
     logger.setLevel(log_level)
-    
+
     # Console handler
     console_handler = logging.StreamHandler()
     console_handler.setLevel(log_level)
     console_format = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
     console_handler.setFormatter(console_format)
     logger.addHandler(console_handler)
-    
+
     # File handler
     if log_dir:
-        log_file = os.path.join(log_dir, f"pipeline_{time.strftime('%Y%m%d-%H%M%S')}.log")
+        log_file = os.path.join(
+            log_dir,
+            f"pipeline_{
+                time.strftime('%Y%m%d-%H%M%S')}.log",
+        )
         file_handler = logging.FileHandler(log_file)
         file_handler.setLevel(log_level)
-        file_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s")
+        file_format = logging.Formatter(
+            "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
+        )
         file_handler.setFormatter(file_format)
         logger.addHandler(file_handler)
-    
+
     return logger
 
-def log_execution_time(func):
+
+def log_execution_time(func: Callable[..., Any]) -> Callable[..., Any]:
     """Decorator to log function execution time"""
+
     @wraps(func)
-    def wrapper(*args, **kwargs):
+    def wrapper(*args, **kwargs) -> None:
         start_time = time.time()
         result = func(*args, **kwargs)
         end_time = time.time()
         logging.info(f"{func.__name__} executed in {end_time - start_time:.2f} seconds")
         return result
+
     return wrapper
 
-def log_resource_usage():
+
+def log_resource_usage() -> Dict[str, Any]:
     """Log current resource utilization (CPU, memory, disk)"""
     cpu_percent = psutil.cpu_percent()
     mem = psutil.virtual_memory()
-    disk = psutil.disk_usage('.')
-    
-    logging.info(f"Resource Usage - CPU: {cpu_percent}%, Memory: {mem.percent}%, Disk: {disk.percent}%")
-    
+    disk = psutil.disk_usage(".")
+
+    logging.info(
+        f"Resource Usage - CPU: {cpu_percent}%, Memory: {
+            mem.percent}%, Disk: {
+            disk.percent}%"
+    )
+
     return {
         "cpu_percent": cpu_percent,
         "memory_percent": mem.percent,
         "disk_percent": disk.percent,
         "memory_available": mem.available,
-        "disk_free": disk.free
+        "disk_free": disk.free,
     }

rust_crate_pipeline/version.py

@@ -1,9 +1,60 @@
+from typing import Dict, List, Tuple, Optional, Any
 """Version information for rust-crate-pipeline."""
 
-__version__ = "1.2.6"
-__version_info__ = tuple(int(x) for x in __version__.split("."))
+__version__ = "1.3.1"
+__version_info__ = tuple(int(x) for x in __version__.split("-")[0].split("."))
 
 # Version history
+# 1.2.5-dev.20250621 - Dev branch: experimental, not a formal
+# release. Originated from v1.2.5.
+# 1.2.5 - Last official release.
+# 1.5.1 - Configuration Standardization Release: Model Path Consistency
+#         - Standardized all configuration to use GGUF model paths
+#         - Updated CLI defaults for --crawl4ai-model to
+#           ~/models/deepseek/deepseek-coder-6.7b-instruct.Q4_K_M.gguf
+#         - Enhanced Rule Zero alignment with transparent configuration practices
+#         - Updated all test files to use consistent GGUF model path references
+#         - Comprehensive documentation updates for proper model configuration
+#         - Removed inconsistent Ollama references in favor of llama-cpp-python
+#         - Ensured CLI help text and JSON examples reflect correct model paths
+# 1.5.0 - Major Release: Enhanced Web Scraping with Crawl4AI Integration
+#         - Integrated Crawl4AI for advanced web scraping capabilities
+#         - Added JavaScript-rendered content extraction via Playwright
+#         - Enhanced README parsing with LLM-powered content analysis
+#         - New CLI options: --enable-crawl4ai, --disable-crawl4ai, --crawl4ai-model
+#         - Enhanced configuration with local GGUF model paths and crawl4ai_timeout
+#         - Comprehensive test coverage for all Crawl4AI features
+#         - Rule Zero compliant with full transparency and audit trails
+# 1.4.0 - Major Release: Rule Zero Compliance Audit Complete
+#         - Completed comprehensive Rule Zero alignment audit
+#         - Eliminated all code redundancy and dead code
+#         - Achieved 100% test coverage (22/22 tests passing)
+#         - Refactored to pure asyncio architecture (thread-free)
+#         - Suppressed Pydantic deprecation warnings
+#         - Full production readiness with Docker support
+#         - Enhanced documentation with PyPI cross-references
+#         - Certified Rule Zero compliance across all four principles
+# 1.3.1 - Bug Fix Release: Crawl4AI Integration Cleanup
+#         - Fixed CSS selector syntax errors in Crawl4AI integration
+#         - Cleaned up duplicate and obsolete test files
+#         - Resolved import conflicts between workspace and integration configs
+#         - Improved error handling in enhanced scraping module
+#         - Standardized on direct llama.cpp approach (removed Ollama dependencies)
+#         - Enhanced Rule Zero compliance with transparent cleanup process
+#         - Fixed type annotation compatibility issues
+#         - Fixed Python 3.9 compatibility for type annotations
+#         - Updated dict[str, Any] to "dict[str, Any]" format
+#         - Fixed Union type expressions in conditional imports
+#         - Resolved IDE linter errors in network.py, pipeline.py, and production_config.py
+#         - Improved code quality and maintainability
+# 1.3.0 - Quality & Integration Release: Comprehensive code quality improvements
+#         - Fixed all critical PEP 8 violations (F821, F811, E114)
+#         - Enhanced error handling with graceful dependency fallbacks
+#         - Improved module integration and import path resolution
+#         - Added comprehensive test validation (21/21 tests passing)
+#         - Enhanced async support and Unicode handling
+#         - Production-ready CLI interfaces with robust error handling
+#         - Full Rule Zero compliance validation
 # 1.2.0 - Major release: Production-ready, cleaned codebase
 #         - Unified documentation into single comprehensive README
 #         - Removed all non-essential development and test files

rust_crate_pipeline-1.3.1.dist-info/METADATA

@@ -0,0 +1,357 @@
+Metadata-Version: 2.4
+Name: rust-crate-pipeline
+Version: 1.3.1
+Summary: A comprehensive system for gathering, enriching, and analyzing metadata for Rust crates using AI-powered insights
+Home-page: https://github.com/Superuser666-Sigil/SigilDERG-Data_Production
+Author: SuperUser666-Sigil
+Author-email: SuperUser666-Sigil <miragemodularframework@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/Superuser666-Sigil/SigilDERG-Data_Production
+Project-URL: Documentation, https://github.com/Superuser666-Sigil/SigilDERG-Data_Production#readme
+Project-URL: Repository, https://github.com/Superuser666-Sigil/SigilDERG-Data_Production
+Project-URL: Bug Tracker, https://github.com/Superuser666-Sigil/SigilDERG-Data_Production/issues
+Keywords: rust,crates,metadata,ai,analysis,pipeline,dependencies
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.28.0
+Requires-Dist: requests-cache>=1.0.0
+Requires-Dist: beautifulsoup4>=4.11.0
+Requires-Dist: tqdm>=4.64.0
+Requires-Dist: llama-cpp-python>=0.2.0
+Requires-Dist: tiktoken>=0.5.0
+Requires-Dist: psutil>=5.9.0
+Requires-Dist: python-dateutil>=2.8.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: black>=22.0.0; extra == "dev"
+Requires-Dist: isort>=5.10.0; extra == "dev"
+Provides-Extra: advanced
+Requires-Dist: radon>=6.0.0; extra == "advanced"
+Requires-Dist: rustworkx>=0.13.0; extra == "advanced"
+Dynamic: author
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+
+# Rust Crate Pipeline
+
+A comprehensive system for gathering, enriching, and analyzing metadata for Rust crates using AI-powered insights, web scraping, and dependency analysis.
+
+## Overview
+
+The Rust Crate Pipeline is designed to collect, process, and enrich metadata from Rust crates available on crates.io. It combines web scraping, AI-powered analysis, and cargo testing to provide comprehensive insights into Rust ecosystem packages.
+
+## Features
+
+- **Web Scraping**: Automated collection of crate metadata from crates.io using Crawl4AI
+- **AI Enrichment**: Local and Azure OpenAI-powered analysis of crate descriptions, features, and documentation
+- **Multi-Provider LLM Support**: Unified LLM processor supporting OpenAI, Azure OpenAI, Ollama, LM Studio, and LiteLLM
+- **Cargo Testing**: Automated cargo build, test, and audit execution for comprehensive crate analysis
+- **Dependency Analysis**: Deep analysis of crate dependencies and their relationships
+- **Batch Processing**: Efficient processing of multiple crates with configurable batch sizes
+- **Data Export**: Structured output in JSON format for further analysis
+- **RAG Cache**: Intelligent caching with Rule Zero policies and architectural patterns
+- **Docker Support**: Containerized deployment with optimized Docker configurations
+
+## Installation
+
+```bash
+# Clone the repository
+git clone https://github.com/Superuser666-Sigil/SigilDERG-Data_Production.git
+cd SigilDERG-Data_Production
+
+# Install in development mode
+pip install -e .
+
+# Install additional dependencies for AI processing
+pip install -r requirements-crawl4ai.txt
+```
+
+## Configuration
+
+### Environment Variables
+
+Set the following environment variables for full functionality:
+
+```bash
+# GitHub Personal Access Token (required for API access)
+export GITHUB_TOKEN="your_github_token_here"
+
+# Azure OpenAI (optional, for cloud AI processing)
+export AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/"
+export AZURE_OPENAI_API_KEY="your_azure_openai_key"
+export AZURE_OPENAI_DEPLOYMENT_NAME="your_deployment_name"
+export AZURE_OPENAI_API_VERSION="2024-02-15-preview"
+
+# PyPI API Token (optional, for publishing)
+export PYPI_API_TOKEN="your_pypi_token"
+
+# LiteLLM Configuration (optional, for multi-provider LLM support)
+export LITELLM_MODEL="deepseek-coder:33b"
+export LITELLM_BASE_URL="http://localhost:11434"  # For Ollama
+```
+
+### Configuration File
+
+Create a `config.json` file for custom settings:
+
+```json
+{
+    "batch_size": 10,
+    "n_workers": 4,
+    "max_retries": 3,
+    "checkpoint_interval": 10,
+    "use_azure_openai": true,
+    "crawl4ai_config": {
+        "max_pages": 5,
+        "concurrency": 2
+    }
+}
+```
+
+## Usage
+
+### Command Line Interface
+
+#### Basic Usage
+
+```bash
+# Run with default settings
+python -m rust_crate_pipeline
+
+# Run with custom batch size
+python -m rust_crate_pipeline --batch-size 20
+
+# Run with specific workers
+python -m rust_crate_pipeline --workers 8
+
+# Use configuration file
+python -m rust_crate_pipeline --config-file config.json
+```
+
+#### Advanced Options
+
+```bash
+# Enable Azure OpenAI processing
+python -m rust_crate_pipeline --enable-azure-openai
+
+# Set custom model path for local AI
+python -m rust_crate_pipeline --model-path /path/to/model.gguf
+
+# Configure token limits
+python -m rust_crate_pipeline --max-tokens 2048
+
+# Set checkpoint interval
+python -m rust_crate_pipeline --checkpoint-interval 5
+
+# Enable verbose logging
+python -m rust_crate_pipeline --log-level DEBUG
+```
+
+#### Multi-Provider LLM Support
+
+```bash
+# Use OpenAI
+python -m rust_crate_pipeline.unified_llm_processor --provider openai --model-name gpt-4
+
+# Use Azure OpenAI
+python -m rust_crate_pipeline.unified_llm_processor --provider azure --model-name gpt-4
+
+# Use Ollama (local)
+python -m rust_crate_pipeline.unified_llm_processor --provider ollama --model-name deepseek-coder:33b
+
+# Use LM Studio
+python -m rust_crate_pipeline.unified_llm_processor --provider openai --base-url http://localhost:1234/v1 --model-name local-model
+
+# Use LiteLLM
+python -m rust_crate_pipeline.unified_llm_processor --provider litellm --model-name deepseek-coder:33b
+```
+
+#### Production Mode
+
+```bash
+# Run production pipeline with optimizations
+python run_production.py
+
+# Run with Sigil Protocol integration
+python -m rust_crate_pipeline --enable-sigil-protocol
+```
+
+### Programmatic Usage
+
+```python
+from rust_crate_pipeline import CrateDataPipeline
+from rust_crate_pipeline.config import PipelineConfig
+
+# Create configuration
+config = PipelineConfig(
+    batch_size=10,
+    n_workers=4,
+    use_azure_openai=True
+)
+
+# Initialize pipeline
+pipeline = CrateDataPipeline(config)
+
+# Run pipeline
+import asyncio
+result = asyncio.run(pipeline.run())
+```
+
+## Sample Data
+
+### Input: Crate List
+
+The pipeline processes crates from `rust_crate_pipeline/crate_list.txt`:
+
+```
+tokio
+serde
+reqwest
+actix-web
+clap
+```
+
+### Output: Enriched Crate Data
+
+```json
+{
+    "name": "tokio",
+    "version": "1.35.1",
+    "description": "An asynchronous runtime for Rust",
+    "downloads": 125000000,
+    "github_stars": 21500,
+    "keywords": ["async", "runtime", "tokio", "futures"],
+    "categories": ["asynchronous", "network-programming"],
+    "features": {
+        "full": ["all features enabled"],
+        "rt": ["runtime features"],
+        "macros": ["macro support"]
+    },
+    "readme_summary": "Tokio is an asynchronous runtime for Rust that provides the building blocks for writing network applications.",
+    "use_case": "Networking",
+    "factual_counterfactual": "✅ Factual: Tokio provides async I/O primitives\n❌ Counterfactual: Tokio is a synchronous runtime",
+    "score": 9.5,
+    "cargo_test_results": {
+        "build_success": true,
+        "test_success": true,
+        "audit_clean": true,
+        "dependencies": 45
+    },
+    "ai_insights": {
+        "complexity": "High",
+        "maturity": "Production Ready",
+        "community_health": "Excellent"
+    }
+}
+```
+
+## Architecture
+
+### Core Components
+
+- **Pipeline Orchestrator**: Manages the overall data processing workflow
+- **Web Scraper**: Collects crate metadata using Crawl4AI
+- **AI Enricher**: Enhances data with local or cloud AI analysis
+- **Cargo Analyzer**: Executes cargo commands for comprehensive testing
+- **Data Exporter**: Outputs structured results in various formats
+
+### Data Flow
+
+1. **Input**: Crate names from `crate_list.txt`
+2. **Scraping**: Web scraping of crates.io for metadata
+3. **Enrichment**: AI-powered analysis and insights
+4. **Testing**: Cargo build, test, and audit execution
+5. **Output**: Structured JSON with comprehensive crate analysis
+
+## Development
+
+### Running Tests
+
+```bash
+# Run all tests
+pytest tests/
+
+# Run specific test module
+pytest tests/test_main_integration.py
+
+# Run with coverage
+pytest --cov=rust_crate_pipeline tests/
+```
+
+### Code Quality
+
+```bash
+# Format code
+black rust_crate_pipeline/
+
+# Sort imports
+isort rust_crate_pipeline/
+
+# Type checking
+pyright rust_crate_pipeline/
+```
+
+## Requirements
+
+- Python 3.12+
+- Rust toolchain (for cargo testing)
+- Git (for GitHub API access)
+- Internet connection (for web scraping and API calls)
+
+## License
+
+MIT License - see LICENSE file for details.
+
+## Contributing
+
+1. Fork the repository
+2. Create a feature branch
+3. Make your changes
+4. Add tests for new functionality
+5. Submit a pull request
+
+## Support
+
+For issues and questions:
+- GitHub Issues: https://github.com/Superuser666-Sigil/SigilDERG-Data_Production/issues
+- Documentation: https://github.com/Superuser666-Sigil/SigilDERG-Data_Production#readme 
+
+## API Compliance & Attribution
+
+### crates.io and GitHub API Usage
+- This project accesses crates.io and GitHub APIs for data gathering and verification.
+- **User-Agent:** All requests use:
+  
+  `SigilDERG-Data-Production (Superuser666-Sigil; miragemodularframework@gmail.com; https://github.com/Superuser666-Sigil/SigilDERG-Data_Production)`
+- **Contact:** miragemodularframework@gmail.com
+- **GitHub:** [Superuser666-Sigil/SigilDERG-Data_Production](https://github.com/Superuser666-Sigil/SigilDERG-Data_Production)
+- The project respects all rate limits and crawler policies. If you have questions or concerns, please contact us.
+
+### Crawl4AI Attribution
+This project uses [Crawl4AI](https://github.com/unclecode/crawl4ai) for web data extraction.
+
+<!-- Badge Attribution (Disco Theme) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-disco.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+Or, text attribution:
+
+```
+This project uses Crawl4AI (https://github.com/unclecode/crawl4ai) for web data extraction.
+``` 

rust_crate_pipeline-1.3.1.dist-info/RECORD

@@ -0,0 +1,30 @@
+rust_crate_pipeline/__init__.py,sha256=ZJCApGu8h2Rn5-dkoBLXOpdoeD6b36w76--o0fEismQ,1749
+rust_crate_pipeline/__main__.py,sha256=PexSWQYtbFQg5P36WEnJ0X-oAtT8WDej3bIJoSAcCCQ,157
+rust_crate_pipeline/ai_processing.py,sha256=MP6VcvV3Jw2Pjof3NrewjTmO8ruVyJKcJGa9zhS_2eY,24140
+rust_crate_pipeline/analysis.py,sha256=9-WpGCOwto7mQre_AEYB6MA6bELekUMlGJzs-C5jg5g,15359
+rust_crate_pipeline/azure_ai_processing.py,sha256=kxbHGNSRSD_5KNkL2ihqCASJq8kdnb_N9u1-ogXbneE,16449
+rust_crate_pipeline/config.py,sha256=Fw3fRKCZawKaLQi7YqsmNNku4whZi89mWzr8BVRNS5E,3009
+rust_crate_pipeline/crate_analysis.py,sha256=GsoXemJ9VFyAbb4Sm5gY5ToTqNtOA4pI38AtngAQONk,2090
+rust_crate_pipeline/crate_list.txt,sha256=W3NxDtxvihyKp9SN85FYXX6p8Hh49IFih1M4-c-CynM,4334
+rust_crate_pipeline/github_token_checker.py,sha256=COXXS9uoLV9WYIcT02C-bV5uH3fa9D9HJImc07vMjLs,3766
+rust_crate_pipeline/main.py,sha256=Pfh-rQNQfCe5st_dcIZOALXAy14_IDJaMNK89DLJydc,18880
+rust_crate_pipeline/network.py,sha256=FtEQRYC-Ynep99zKYvuI7_Jd6I2poMIPlxoJFlxgAg8,13308
+rust_crate_pipeline/pipeline.py,sha256=YN6PEhg0Si_oo6-Wtm_PviytzWzpQupTPC2e4L1F7XE,16349
+rust_crate_pipeline/production_config.py,sha256=uWylP9AIZZx7-9aT4sFmAKEEW9miJDxaiek8VE6WP-0,2372
+rust_crate_pipeline/unified_llm_processor.py,sha256=eo7KotNuqwc7_hgpFm18QLokFoufFslnvi8TnDsSYEg,25064
+rust_crate_pipeline/unified_pipeline.py,sha256=2yglmXVlQfSkVq0HVTPonDee6VxWaQWZw0X2l4lLBGw,23704
+rust_crate_pipeline/version.py,sha256=2EZ0sMLJDvFLO6arTvjiXhlCztJ1xWetEH4Oh6IDHtY,4489
+rust_crate_pipeline/core/__init__.py,sha256=Sq4HWdANGqoYln7JdCog7m3BsGeR3tHdseeflvNetoQ,509
+rust_crate_pipeline/core/canon_registry.py,sha256=36tmt_wU6-kSyZnGfh53N64C7E3G-QR7GFbr9epj4zg,4700
+rust_crate_pipeline/core/irl_engine.py,sha256=QRZUdkN24W9XutLkj8JDplEz6FmnquUrwKsl0s2zRr4,10491
+rust_crate_pipeline/core/sacred_chain.py,sha256=6s4gFLDT6KUwuu0Fpxu6h_YHlsEvHZb3CQw4tRHGyDU,3773
+rust_crate_pipeline/scraping/__init__.py,sha256=ySkTRg7nIxgcbHJQ3L1XzcrOo281NZu07-XtiGi-558,307
+rust_crate_pipeline/scraping/unified_scraper.py,sha256=ZE2gkc0vQ3BOLdSX_IV-kMe8QAm2Av4M7VqpkxEKyT4,9965
+rust_crate_pipeline/utils/file_utils.py,sha256=tMaCPy7ghs9x4Hxu_sviX8MXU2sBjNvohUrvt4MejoM,2853
+rust_crate_pipeline/utils/logging_utils.py,sha256=e5jG0Yd6k3exgAdbVca46kWADJ_Qz8UJ3yEJzwTqPyI,2452
+rust_crate_pipeline-1.3.1.dist-info/licenses/LICENSE,sha256=tpd4XNpbssrSx9-iErATOLrOh0ivNPfO2I5MAPUpats,1088
+rust_crate_pipeline-1.3.1.dist-info/METADATA,sha256=XEkcmRsrlOToAUchpzjvMyD81aUwgoZgcBrh3_-Js3g,11254
+rust_crate_pipeline-1.3.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rust_crate_pipeline-1.3.1.dist-info/entry_points.txt,sha256=9Rr_IRuFRIridXxUSdEJbB3ba0NnpEfKmknZXFdYRC0,70
+rust_crate_pipeline-1.3.1.dist-info/top_level.txt,sha256=GUdB7RyxHLhijQxui_KTy3B8p_L2APui9C6RYa0FuaE,20
+rust_crate_pipeline-1.3.1.dist-info/RECORD,,