rust-crate-pipeline 1.3.5__py3-none-any.whl → 1.3.6__py3-none-any.whl

rust_crate_pipeline/analysis.py

@@ -1,70 +1,112 @@
 # analysis.py
+from __future__ import annotations
+
 import io
 import re
 import tarfile
 import requests
 import logging
 import tempfile
-from typing import Any
+from typing import Any, Dict, List, Optional, Union
 import os
 import sys
 import time
 import subprocess
+from dataclasses import dataclass
 
 from .config import EnrichedCrate
 
-# Add the project root to the path to ensure utils can be imported
-# This is a common pattern in scripts to handle execution from different directories
-project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-if project_root not in sys.path:
-    sys.path.insert(0, project_root)
-
-try:
-    from utils.rust_code_analyzer import RustCodeAnalyzer  # type: ignore
-except ImportError as e:
-    logging.error(
-        f"Failed to import RustCodeAnalyzer: {e}. "
-        f"Ensure the utils directory is in the Python path."
-    )
-    # Provide a non-functional fallback to avoid crashing the entire application
-    # if the import fails, but ensure it logs the error.
-
-    class RustCodeAnalyzer:  # type: ignore
-        def __init__(self, code_content: str) -> None:
-            logging.error(
-                "Using fallback RustCodeAnalyzer. Analysis will be incomplete."
-            )
-            self.code_content = code_content
-
-        def analyze(self) -> dict[str, Any]:
-            return {
-                "functions": [],
-                "structs": [],
-                "enums": [],
-                "traits": [],
-                "complexity": 0,
-                "lines_of_code": len(self.code_content.split("\n")),
-            }
-
-        @staticmethod
-        def create_empty_metrics() -> dict[str, Any]:
-            return {}
-
-        @staticmethod
-        def detect_project_structure(files: list[str]) -> dict[str, bool]:
-            return {}
-
-        @staticmethod
-        def analyze_rust_content(content: str) -> dict[str, Any]:
-            return {}
-
-        @staticmethod
-        def aggregate_metrics(
-            metrics: dict[str, Any],
-            content_analysis: dict[str, Any],
-            structure: dict[str, bool],
-        ) -> dict[str, Any]:
-            return metrics
+# Create a fallback RustCodeAnalyzer that doesn't depend on external utils
+class RustCodeAnalyzer:
+    """Fallback Rust code analyzer for when the full analyzer is not available."""
+    
+    def __init__(self, code_content: str) -> None:
+        self.code_content = code_content
+
+    def analyze(self) -> dict[str, Any]:
+        """Basic analysis of Rust code content."""
+        lines = self.code_content.split('\n')
+        return {
+            "functions": self._count_functions(),
+            "structs": self._count_structs(),
+            "enums": self._count_enums(),
+            "traits": self._count_traits(),
+            "complexity": self._calculate_complexity(),
+            "lines_of_code": len(lines),
+        }
+
+    def _count_functions(self) -> int:
+        """Count function definitions."""
+        return len(re.findall(r'fn\s+\w+\s*\(', self.code_content))
+
+    def _count_structs(self) -> int:
+        """Count struct definitions."""
+        return len(re.findall(r'struct\s+\w+', self.code_content))
+
+    def _count_enums(self) -> int:
+        """Count enum definitions."""
+        return len(re.findall(r'enum\s+\w+', self.code_content))
+
+    def _count_traits(self) -> int:
+        """Count trait definitions."""
+        return len(re.findall(r'trait\s+\w+', self.code_content))
+
+    def _calculate_complexity(self) -> int:
+        """Calculate basic cyclomatic complexity."""
+        complexity = 0
+        complexity += len(re.findall(r'\bif\b', self.code_content))
+        complexity += len(re.findall(r'\bfor\b', self.code_content))
+        complexity += len(re.findall(r'\bwhile\b', self.code_content))
+        complexity += len(re.findall(r'\bmatch\b', self.code_content))
+        return complexity
+
+    @staticmethod
+    def create_empty_metrics() -> dict[str, Any]:
+        """Create empty metrics structure."""
+        return {
+            "functions": 0,
+            "structs": 0,
+            "enums": 0,
+            "traits": 0,
+            "complexity": 0,
+            "lines_of_code": 0,
+            "file_count": 0,
+        }
+
+    @staticmethod
+    def detect_project_structure(files: list[str]) -> dict[str, bool]:
+        """Detect basic project structure."""
+        return {
+            "has_cargo_toml": any("Cargo.toml" in f for f in files),
+            "has_src": any("/src/" in f for f in files),
+            "has_tests": any("/tests/" in f for f in files),
+            "has_examples": any("/examples/" in f for f in files),
+        }
+
+    @staticmethod
+    def analyze_rust_content(content: str) -> dict[str, Any]:
+        """Analyze Rust content."""
+        analyzer = RustCodeAnalyzer(content)
+        return analyzer.analyze()
+
+    @staticmethod
+    def aggregate_metrics(
+        metrics: dict[str, Any],
+        content_analysis: dict[str, Any],
+        structure: dict[str, bool],
+    ) -> dict[str, Any]:
+        """Aggregate metrics from multiple sources."""
+        for key, value in content_analysis.items():
+            if isinstance(value, (int, float)):
+                metrics[key] = metrics.get(key, 0) + value
+            elif isinstance(value, list):
+                if key not in metrics:
+                    metrics[key] = []
+                metrics[key].extend(value)
+        
+        # Add structure information
+        metrics.update(structure)
+        return metrics
 
 
 # Constants for URLs and paths

rust_crate_pipeline/network.py

@@ -1,18 +1,14 @@
 # network.py
 import os
-import sys
 import re
+import sys
 import time
 import logging
 import requests
+from typing import Any, Dict, List, Optional, Union
 from bs4 import BeautifulSoup, Tag
-from typing import Any, Union
 from .config import PipelineConfig
 
-# Import utilities
-# Add the parent directory to the path to import utils
-sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
-
 
 class GitHubBatchClient:
     def __init__(self, config: PipelineConfig) -> None:

rust_crate_pipeline/version.py

@@ -1,7 +1,7 @@
 from typing import Dict, List, Tuple, Optional, Any
 """Version information for rust-crate-pipeline."""
 
-__version__ = "1.3.5"
+__version__ = "1.3.6"
 __version_info__ = tuple(int(x) for x in __version__.split("-")[0].split("."))
 __author__ = "SigilDERG Team"
 __email__ = "sigilderg@example.com"

{rust_crate_pipeline-1.3.5.dist-info → rust_crate_pipeline-1.3.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rust-crate-pipeline
-Version: 1.3.5
+Version: 1.3.6
 Summary: A comprehensive pipeline for analyzing Rust crates with AI enrichment and enhanced scraping
 Home-page: https://github.com/SigilDERG/rust-crate-pipeline
 Author: SigilDERG Team
@@ -15,24 +15,24 @@ Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Software Development :: Quality Assurance
-Requires-Python: >=3.8
+Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: requests>=2.28.0
 Requires-Dist: requests-cache>=1.0.0
 Requires-Dist: beautifulsoup4>=4.11.0
+Requires-Dist: crawl4ai>=0.6.0
+Requires-Dist: playwright>=1.49.0
 Requires-Dist: tqdm>=4.64.0
 Requires-Dist: llama-cpp-python>=0.2.0
 Requires-Dist: tiktoken>=0.5.0
 Requires-Dist: psutil>=5.9.0
 Requires-Dist: python-dateutil>=2.8.0
+Requires-Dist: litellm>=1.0.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0.0; extra == "dev"
 Requires-Dist: black>=22.0.0; extra == "dev"
@@ -55,7 +55,7 @@ The Rust Crate Pipeline is designed to collect, process, and enrich metadata fro
 
 ## Features
 
-- **Web Scraping**: Automated collection of crate metadata from crates.io using Crawl4AI
+- **Enhanced Web Scraping**: Automated collection of crate metadata from crates.io using Crawl4AI with Playwright
 - **AI Enrichment**: Local and Azure OpenAI-powered analysis of crate descriptions, features, and documentation
 - **Multi-Provider LLM Support**: Unified LLM processor supporting OpenAI, Azure OpenAI, Ollama, LM Studio, and LiteLLM
 - **Cargo Testing**: Automated cargo build, test, and audit execution for comprehensive crate analysis
@@ -64,6 +64,15 @@ The Rust Crate Pipeline is designed to collect, process, and enrich metadata fro
 - **Data Export**: Structured output in JSON format for further analysis
 - **RAG Cache**: Intelligent caching with Rule Zero policies and architectural patterns
 - **Docker Support**: Containerized deployment with optimized Docker configurations
+- **Real-time Progress Monitoring**: CLI-based progress tracking with ASCII status indicators
+- **Cross-platform Compatibility**: Full Unicode symbol replacement for better encoding support
+
+## Requirements
+
+- **Python 3.12+**: Required for modern type annotations and language features
+- **Git**: For cloning repositories during analysis
+- **Cargo**: For Rust crate testing and analysis
+- **Playwright**: Automatically installed for enhanced web scraping
 
 ## Installation
 
@@ -72,13 +81,22 @@ The Rust Crate Pipeline is designed to collect, process, and enrich metadata fro
 git clone https://github.com/Superuser666-Sigil/SigilDERG-Data_Production.git
 cd SigilDERG-Data_Production
 
-# Install in development mode
+# Install in development mode (includes all dependencies)
 pip install -e .
 
-# Install additional dependencies for AI processing
-pip install -r requirements-crawl4ai.txt
+# Install Playwright browsers for enhanced scraping
+playwright install
 ```
 
+### Automatic Dependency Installation
+
+The package automatically installs all required dependencies including:
+- `crawl4ai` for web scraping
+- `playwright` for enhanced browser automation
+- `requests` for HTTP requests
+- `aiohttp` for async operations
+- And all other required packages
+
 ## Configuration
 
 ### Environment Variables
@@ -158,6 +176,27 @@ python -m rust_crate_pipeline --checkpoint-interval 5
 
 # Enable verbose logging
 python -m rust_crate_pipeline --log-level DEBUG
+
+# Enable enhanced scraping with Playwright
+python -m rust_crate_pipeline --enable-enhanced-scraping
+
+# Set output directory for results
+python -m rust_crate_pipeline --output-path ./results
+```
+
+#### Enhanced Scraping
+
+The pipeline now supports enhanced web scraping using Playwright for better data extraction:
+
+```bash
+# Enable enhanced scraping (default)
+python -m rust_crate_pipeline --enable-enhanced-scraping
+
+# Use basic scraping only
+python -m rust_crate_pipeline --disable-enhanced-scraping
+
+# Configure scraping options
+python -m rust_crate_pipeline --scraping-config '{"max_pages": 10, "concurrency": 3}'
 ```
 
 #### Multi-Provider LLM Support
@@ -278,6 +317,12 @@ clap
 
 ## Development
 
+### Prerequisites
+
+- Python 3.12+ (required for modern type annotations)
+- Git for version control
+- Cargo for Rust crate testing
+
 ### Running Tests
 
 ```bash
@@ -289,6 +334,12 @@ pytest tests/test_main_integration.py
 
 # Run with coverage
 pytest --cov=rust_crate_pipeline tests/
+
+# Run type checking
+pyright rust_crate_pipeline/
+
+# Run linting
+flake8 rust_crate_pipeline/
 ```
 
 ### Code Quality
@@ -302,14 +353,64 @@ isort rust_crate_pipeline/
 
 # Type checking
 pyright rust_crate_pipeline/
+
+# Lint code
+flake8 rust_crate_pipeline/
 ```
 
-## Requirements
+### Building and Publishing
+
+```bash
+# Build package
+python -m build
+
+# Upload to PyPI (requires PYPI_API_TOKEN)
+python -m twine upload dist/*
+
+# Create release
+python scripts/create_release.py
+```
+
+### Docker Development
+
+```bash
+# Build Docker image
+docker build -t rust-crate-pipeline .
+
+# Run in Docker
+docker run -it rust-crate-pipeline
+
+# Run with volume mount for development
+docker run -it -v $(pwd):/app rust-crate-pipeline
+```
 
-- Python 3.12+
-- Rust toolchain (for cargo testing)
-- Git (for GitHub API access)
-- Internet connection (for web scraping and API calls)
+## Recent Improvements
+
+### Version 1.3.6
+- **Python 3.12+ Requirement**: Updated to use modern type annotations and language features
+- **Type Safety**: Enhanced type annotations throughout the codebase with modern syntax
+- **Build System**: Updated pyproject.toml and setup.py for better compatibility
+
+### Version 1.3.5
+- **Enhanced Web Scraping**: Added Playwright-based scraping for better data extraction
+- **Unicode Compatibility**: Replaced all Unicode symbols with ASCII equivalents for better cross-platform support
+- **Automatic Dependencies**: All required packages are now automatically installed
+- **Real-time Progress**: Added CLI-based progress monitoring with ASCII status indicators
+- **Docker Optimization**: Updated Dockerfile to include Playwright browser installation
+
+### Version 1.3.4
+- **PEP8 Compliance**: Fixed all Unicode emoji and symbols for better encoding support
+- **Cross-platform Compatibility**: Improved compatibility across different operating systems
+- **Type Safety**: Enhanced type annotations throughout the codebase
+
+### Version 1.3.3
+- **Real-time Progress Monitoring**: Added CLI-only progress tracking feature
+- **Enhanced Logging**: Improved status reporting and error handling
+
+### Version 1.3.2
+- **Multi-Provider LLM Support**: Added support for OpenAI, Azure OpenAI, Ollama, LM Studio, and LiteLLM
+- **Unified LLM Processor**: Centralized LLM processing with provider abstraction
+- **Enhanced Error Handling**: Better error recovery and retry mechanisms
 
 ## License
 

{rust_crate_pipeline-1.3.5.dist-info → rust_crate_pipeline-1.3.6.dist-info}/RECORD

@@ -1,20 +1,20 @@
 rust_crate_pipeline/__init__.py,sha256=ZJCApGu8h2Rn5-dkoBLXOpdoeD6b36w76--o0fEismQ,1749
 rust_crate_pipeline/__main__.py,sha256=PexSWQYtbFQg5P36WEnJ0X-oAtT8WDej3bIJoSAcCCQ,157
 rust_crate_pipeline/ai_processing.py,sha256=MP6VcvV3Jw2Pjof3NrewjTmO8ruVyJKcJGa9zhS_2eY,24140
-rust_crate_pipeline/analysis.py,sha256=9-WpGCOwto7mQre_AEYB6MA6bELekUMlGJzs-C5jg5g,15359
+rust_crate_pipeline/analysis.py,sha256=_cmjynLWaQbGIdLQHU3P3rfqHB3gcNNgCdzStbsKrdw,17021
 rust_crate_pipeline/azure_ai_processing.py,sha256=kxbHGNSRSD_5KNkL2ihqCASJq8kdnb_N9u1-ogXbneE,16449
 rust_crate_pipeline/config.py,sha256=Fw3fRKCZawKaLQi7YqsmNNku4whZi89mWzr8BVRNS5E,3009
 rust_crate_pipeline/crate_analysis.py,sha256=GsoXemJ9VFyAbb4Sm5gY5ToTqNtOA4pI38AtngAQONk,2090
 rust_crate_pipeline/crate_list.txt,sha256=W3NxDtxvihyKp9SN85FYXX6p8Hh49IFih1M4-c-CynM,4334
 rust_crate_pipeline/github_token_checker.py,sha256=COXXS9uoLV9WYIcT02C-bV5uH3fa9D9HJImc07vMjLs,3766
 rust_crate_pipeline/main.py,sha256=iGYEAYvXkoFFvaA6DIVGiUL3wLhiCzatB6Fvf-Yrj2A,18858
-rust_crate_pipeline/network.py,sha256=khyjfOplaDvMxLWGB-JbPQnc27ZfozKGYBFw2b3BScM,12834
+rust_crate_pipeline/network.py,sha256=mWjiRvOX31piBZ2QiJ-F75DBD4l6cqzTXcQdJvHxe90,12718
 rust_crate_pipeline/pipeline.py,sha256=CqPHLLRvMOpy-3ONL6hnPahV6Vh6S4M8oDsHd_lDrPc,16203
 rust_crate_pipeline/production_config.py,sha256=uWylP9AIZZx7-9aT4sFmAKEEW9miJDxaiek8VE6WP-0,2372
 rust_crate_pipeline/progress_monitor.py,sha256=5K9KP-Xggi1JEINfRmq2W-wGUHtNIBTcocpDtB1t8iM,13743
 rust_crate_pipeline/unified_llm_processor.py,sha256=eo7KotNuqwc7_hgpFm18QLokFoufFslnvi8TnDsSYEg,25064
 rust_crate_pipeline/unified_pipeline.py,sha256=2yglmXVlQfSkVq0HVTPonDee6VxWaQWZw0X2l4lLBGw,23704
-rust_crate_pipeline/version.py,sha256=1OVfgKIllkCMfu2zCazTqINgUwkozqsQSa2z_MDz5yY,4481
+rust_crate_pipeline/version.py,sha256=izXdwKOkBxecVcCuMmOVbZnu5y-hHZZkEg39LmBPnis,4481
 rust_crate_pipeline/core/__init__.py,sha256=Sq4HWdANGqoYln7JdCog7m3BsGeR3tHdseeflvNetoQ,509
 rust_crate_pipeline/core/canon_registry.py,sha256=36tmt_wU6-kSyZnGfh53N64C7E3G-QR7GFbr9epj4zg,4700
 rust_crate_pipeline/core/irl_engine.py,sha256=QRZUdkN24W9XutLkj8JDplEz6FmnquUrwKsl0s2zRr4,10491
@@ -23,9 +23,9 @@ rust_crate_pipeline/scraping/__init__.py,sha256=ySkTRg7nIxgcbHJQ3L1XzcrOo281NZu0
 rust_crate_pipeline/scraping/unified_scraper.py,sha256=ZE2gkc0vQ3BOLdSX_IV-kMe8QAm2Av4M7VqpkxEKyT4,9965
 rust_crate_pipeline/utils/file_utils.py,sha256=tMaCPy7ghs9x4Hxu_sviX8MXU2sBjNvohUrvt4MejoM,2853
 rust_crate_pipeline/utils/logging_utils.py,sha256=e5jG0Yd6k3exgAdbVca46kWADJ_Qz8UJ3yEJzwTqPyI,2452
-rust_crate_pipeline-1.3.5.dist-info/licenses/LICENSE,sha256=tpd4XNpbssrSx9-iErATOLrOh0ivNPfO2I5MAPUpats,1088
-rust_crate_pipeline-1.3.5.dist-info/METADATA,sha256=CXaKKIGRNDIkeaJvDcsslH7aM9Bu0zFzsNKwG_P2i10,11048
-rust_crate_pipeline-1.3.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rust_crate_pipeline-1.3.5.dist-info/entry_points.txt,sha256=9Rr_IRuFRIridXxUSdEJbB3ba0NnpEfKmknZXFdYRC0,70
-rust_crate_pipeline-1.3.5.dist-info/top_level.txt,sha256=GUdB7RyxHLhijQxui_KTy3B8p_L2APui9C6RYa0FuaE,20
-rust_crate_pipeline-1.3.5.dist-info/RECORD,,
+rust_crate_pipeline-1.3.6.dist-info/licenses/LICENSE,sha256=tpd4XNpbssrSx9-iErATOLrOh0ivNPfO2I5MAPUpats,1088
+rust_crate_pipeline-1.3.6.dist-info/METADATA,sha256=BWUkQKtJCbXt1KZbmdofzy0eC4LpoeMBLKVXu3H3hD0,14539
+rust_crate_pipeline-1.3.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rust_crate_pipeline-1.3.6.dist-info/entry_points.txt,sha256=9Rr_IRuFRIridXxUSdEJbB3ba0NnpEfKmknZXFdYRC0,70
+rust_crate_pipeline-1.3.6.dist-info/top_level.txt,sha256=GUdB7RyxHLhijQxui_KTy3B8p_L2APui9C6RYa0FuaE,20
+rust_crate_pipeline-1.3.6.dist-info/RECORD,,