rust-crate-pipeline 1.2.3__tar.gz → 1.2.5__tar.gz

{rust_crate_pipeline-1.2.3 → rust_crate_pipeline-1.2.5}/CHANGELOG.md

@@ -2,6 +2,43 @@
 
 All notable changes to the Rust Crate Pipeline project.
 
+## [1.2.5] - 2025-06-18
+
+### 🎯 Balanced Dataset & Clean Build
+
+#### ✨ Enhanced
+- **Balanced Dataset**: Expanded crate list from 105 to ~425 crates with balanced category distribution
+- **Reduced ML/AI Bias**: Decreased from 52% ML/AI crates to ~13% for more representative ecosystem analysis
+- **Comprehensive Coverage**: Added 19 well-distributed categories covering the full Rust ecosystem
+- **Category Expansion**: Significantly expanded web frameworks, async runtimes, databases, cryptography, gaming, and system programming categories
+- **Clean Build Environment**: Cleaned up build artifacts and temporary scripts
+
+#### 🔧 Technical
+- **Duplicate Removal**: Eliminated duplicate crates across categories
+- **Build Process**: Clean package build and validation
+- **Version Alignment**: Updated all version references across all files
+
+## [1.2.4] - 2025-06-18
+
+### 🐛 Critical Logging Fix
+
+#### ✨ Fixed
+- **Critical Logging Issue**: Fixed 0-byte log file problem caused by conflicting `logging.basicConfig()` calls
+- **Enhanced File Logging**: Improved logging setup with proper handler management and UTF-8 encoding
+- **Better Error Tracking**: Now properly logs all processing steps, errors, and skipped crates to file
+- **Console + File Output**: Maintains both console output and detailed file logging
+
+#### 🔧 Improved
+- **Logging Conflicts**: Resolved production config vs main config logging conflicts
+- **File Handler**: Added proper error handling for log file creation
+- **Encoding Issues**: Fixed Unicode handling in log files
+- **Debug Information**: Always captures DEBUG+ level info to log files while respecting console log level
+
+#### 📊 Monitoring
+- **Better Tracking**: Now you can properly see which crates were skipped and why
+- **Detailed Logs**: Each processing step is properly logged with timestamps
+- **Error Analysis**: Failed crates and reasons are now captured in log files
+
 ## [1.2.3] - 2025-06-18
 
 ### 🚀 L4 GPU Optimization Release

{rust_crate_pipeline-1.2.3/rust_crate_pipeline.egg-info → rust_crate_pipeline-1.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rust-crate-pipeline
-Version: 1.2.3
+Version: 1.2.5
 Summary: A comprehensive system for gathering, enriching, and analyzing metadata for Rust crates using AI-powered insights
 Home-page: https://github.com/DaveTmire85/SigilDERG-Data_Production
 Author: SuperUser666-Sigil

{rust_crate_pipeline-1.2.3 → rust_crate_pipeline-1.2.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rust-crate-pipeline"
-version = "1.2.3"
+version = "1.2.5"
 authors = [
     {name = "SuperUser666-Sigil", email = "miragemodularframework@gmail.com"},
 ]

rust_crate_pipeline-1.2.5/requirements-dev.txt

@@ -0,0 +1,64 @@
+# Development Dependencies for Rust Crate Pipeline
+# Install with: pip install -r requirements-dev.txt
+
+# Testing frameworks and utilities
+pytest>=7.4.0
+pytest-cov>=4.1.0
+pytest-mock>=3.11.0
+pytest-asyncio>=0.21.0
+pytest-timeout>=2.1.0
+pytest-xdist>=3.3.0  # Parallel test execution
+coverage>=7.2.0
+
+# Code formatting and linting
+black>=23.0.0
+isort>=5.12.0
+flake8>=6.0.0
+mypy>=1.5.0
+pylint>=2.17.0
+bandit>=1.7.5  # Security linting
+
+# Documentation
+sphinx>=7.0.0
+sphinx-rtd-theme>=1.3.0
+sphinx-autodoc-typehints>=1.24.0
+
+# Type checking and development tools
+pre-commit>=3.3.0
+twine>=4.0.0  # For PyPI publishing
+build>=0.10.0  # Modern Python build tool
+wheel>=0.40.0
+
+# Jupyter notebook support (for data analysis)
+jupyter>=1.0.0
+ipykernel>=6.25.0
+matplotlib>=3.7.0
+pandas>=2.0.0
+seaborn>=0.12.0
+
+# Database development and debugging
+sqlite-utils>=3.34.0
+
+# Performance profiling
+py-spy>=0.3.14
+memory-profiler>=0.61.0
+
+# Mock and testing utilities
+responses>=0.23.0  # HTTP request mocking
+factory-boy>=3.3.0  # Test data generation
+faker>=19.0.0  # Fake data generation
+
+# API development and testing
+httpx>=0.24.0  # Modern HTTP client for testing
+respx>=0.20.0  # HTTPX request mocking
+
+# Environment and configuration management
+python-dotenv>=1.0.0
+pydantic>=2.0.0  # Data validation
+
+# Git hooks and workflow
+gitpython>=3.1.0
+
+# Optional: Static analysis tools
+vulture>=2.9.0  # Dead code detection
+radon>=6.0.0  # Code complexity analysis

{rust_crate_pipeline-1.2.3 → rust_crate_pipeline-1.2.5}/requirements.txt

@@ -1,4 +1,4 @@
-# Core dependencies
+# Core dependencies for Rust Crate Pipeline
 requests>=2.28.0
 requests-cache>=1.0.0
 beautifulsoup4>=4.11.0
@@ -8,11 +8,6 @@ tiktoken>=0.5.0
 psutil>=5.9.0
 python-dateutil>=2.8.0
 
-# Optional dependencies for advanced features
+# Analysis and data processing
 radon>=6.0.0
 rustworkx>=0.13.0
-
-# Development dependencies (optional)
-pytest>=7.0.0
-black>=22.0.0
-isort>=5.10.0

{rust_crate_pipeline-1.2.3 → rust_crate_pipeline-1.2.5}/rust_crate_pipeline/main.py

@@ -111,15 +111,52 @@ Examples:
     return parser.parse_args()
 
 def configure_logging(log_level: str = 'INFO'):
+    """Configure logging with both console and file output"""
     level = getattr(logging, log_level.upper())
-    logging.basicConfig(
-        level=level,
-        format="%(asctime)s [%(levelname)s] %(message)s",
-        handlers=[
-            logging.StreamHandler(),
-            logging.FileHandler(f"crate_enrichment_{time.strftime('%Y%m%d-%H%M%S')}.log")
-        ]
+    
+    # Clear any existing handlers to avoid conflicts
+    root_logger = logging.getLogger()
+    for handler in root_logger.handlers[:]:
+        root_logger.removeHandler(handler)
+    
+    # Set root logger level
+    root_logger.setLevel(level)
+    
+    # Create formatters
+    detailed_formatter = logging.Formatter(
+        "%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+        datefmt='%Y-%m-%d %H:%M:%S'
     )
+    simple_formatter = logging.Formatter(
+        "%(asctime)s [%(levelname)s] %(message)s"
+    )
+    
+    # Console handler
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(level)
+    console_handler.setFormatter(simple_formatter)
+    root_logger.addHandler(console_handler)
+    
+    # File handler with unique timestamp
+    log_filename = f"crate_enrichment_{time.strftime('%Y%m%d-%H%M%S')}.log"
+    try:
+        file_handler = logging.FileHandler(log_filename, mode='w', encoding='utf-8')
+        file_handler.setLevel(logging.DEBUG)  # Always capture DEBUG+ to file
+        file_handler.setFormatter(detailed_formatter)
+        root_logger.addHandler(file_handler)
+        
+        # Log a test message to verify file handler works
+        logging.info(f"Logging initialized - file: {log_filename}")
+        
+    except Exception as e:
+        logging.error(f"Failed to create log file {log_filename}: {e}")
+        print(f"Warning: Could not create log file: {e}")
+    
+    # Set library loggers to less verbose levels
+    logging.getLogger('requests').setLevel(logging.WARNING)
+    logging.getLogger('urllib3').setLevel(logging.WARNING)
+    logging.getLogger('requests_cache').setLevel(logging.WARNING)
+    logging.getLogger('llama_cpp').setLevel(logging.WARNING)
 
 def check_disk_space():
     if shutil.disk_usage(".").free < 1_000_000_000:  # 1GB

{rust_crate_pipeline-1.2.3 → rust_crate_pipeline-1.2.5}/rust_crate_pipeline/pipeline.py

@@ -29,37 +29,98 @@ class CrateDataPipeline:
     def get_crate_list(self, limit: Optional[int] = None) -> List[str]:
         """Return a comprehensive list of all high-value crates to process"""
         crates = [
-            # Web frameworks
-            "actix-web", "rocket", "axum", "warp", "tower",
+            # Web frameworks and servers
+            "actix-web", "rocket", "axum", "warp", "tower", "tide", "gotham", "iron", 
+            "nickel", "rouille", "thruster", "poem", "salvo", "viz", "ntex", "may-minihttp",
+            "tiny_http", "httptest", "mockito", "wiremock",
             
             # Async runtimes and utilities
-            "tokio", "tokio-stream", "async-trait", "futures",
+            "tokio", "tokio-stream", "async-trait", "futures", "async-std", "smol", 
+            "embassy", "embassy-executor", "embassy-time", "embassy-sync", "async-channel",
+            "async-broadcast", "async-lock", "async-once", "async-recursion", "futures-util",
+            "futures-channel", "futures-timer", "futures-test", "pin-project", "pin-project-lite",
             
             # Serialization/deserialization
-            "serde", "serde_json", "serde_yaml", "bincode",
+            "serde", "serde_json", "serde_yaml", "bincode", "toml", "ron", "postcard",
+            "ciborium", "rmp-serde", "quick-xml", "roxmltree", "serde_cbor", "serde_derive",
+            "serde_repr", "serde_with", "serde_bytes", "flexbuffers", "bson", "avro-rs",
             
-            # Error handling
-            "anyhow", "thiserror",
+            # Error handling and debugging
+            "anyhow", "thiserror", "eyre", "color-eyre", "miette", "fehler", "snafu",
+            "failure", "quick-error", "derive_more", "displaydoc", "backtrace", "better-panic",
+              # Command line and terminal
+            "clap", "structopt", "argh", "gumdrop", "docopt", "getopts", "pico-args",
+            "crossterm", "termion", "console", "indicatif", "dialoguer", "termcolor",
+            "colored", "yansi", "owo-colors", "nu-ansi-term", "terminal_size",
+              # Utilities and general purpose
+            "rand", "uuid", "itertools", "num", "cfg-if", "bytes", "mime", 
+            "form_urlencoded", "csv", "once_cell", "base64", "flate2", "tar", "dirs",
+            "walkdir", "glob", "bitflags", "indexmap", "smallvec", "arrayvec", "tinyvec", 
+            "ahash", "fxhash", "rustc-hash", "seahash", "siphasher", "wyhash", "xxhash-rust", 
+            "getrandom", "fastrand", "nanorand", "url", "percent-encoding", "unicode-segmentation",
+            "unicode-normalization", "unicode-width", "memchr", "aho-corasick", "bstr",
+              # HTTP clients and servers
+            "reqwest", "hyper", "surf", "ureq", "attohttpc", "isahc", "curl", "libcurl-sys",
+            "http", "http-body", "httparse", "hyper-tls", "hyper-rustls", "native-tls",
+            "webpki", "webpki-roots",
             
-            # Utilities
-            "rand", "uuid", "chrono", "regex", "log", "env_logger", "clap", "crossterm", 
-            "itertools", "num", "cfg-if", "bytes", "mime", "form_urlencoded", "parking_lot", 
-            "csv", "lazy_static", "once_cell", "tracing", "base64", "sha2", "flate2", "tar",
+            # Database and storage
+            "sqlx", "diesel", "postgres", "rusqlite", "mysql", "mongodb", "redis",
+            "tokio-postgres", "deadpool-postgres", "bb8", "r2d2", "sea-orm", "rbatis",
+            "sled", "rocksdb", "lmdb", "redb", "pickledb", "persy", "heed", "fjall",
+              # Concurrency and parallelism
+            "rayon", "crossbeam", "crossbeam-channel", "crossbeam-utils", "crossbeam-epoch",
+            "crossbeam-deque", "parking_lot", "spin", "atomic", "arc-swap", "dashmap",
+            "flume", "kanal", "tokio-util", "futures-concurrency",
+              # Protocol buffers, gRPC, and messaging
+            "prost", "tonic", "protobuf", "grpcio", "tarpc", "capnp", "rmp",
+            "zmq", "nanomsg", "nats", "rdkafka", "pulsar", "lapin", "amqp", "rumqttc",
+              # Procedural macros and metaprogramming
+            "syn", "quote", "proc-macro2", "proc-macro-crate", "proc-macro-error",
+            "darling", "derive_builder", "strum", "strum_macros",
+            "enum-iterator", "num-derive", "num-traits", "paste", "lazy_static",
             
-            # HTTP clients and servers
-            "reqwest", "hyper",
+            # Cryptography and security
+            "ring", "rustls", "openssl", "sha2", "sha3", "blake2", "blake3", "md5",
+            "hmac", "pbkdf2", "scrypt", "argon2", "bcrypt", "chacha20poly1305",
+            "aes-gcm", "rsa", "ed25519-dalek", "x25519-dalek", "curve25519-dalek",
+            "secp256k1", "k256", "p256", "ecdsa", "signature", "rand_core",
             
-            # Database
-            "sqlx", "diesel", "postgres", "rusqlite",
+            # Game development and graphics
+            "bevy", "macroquad", "ggez", "piston", "winit", "wgpu", "vulkano", "glium",
+            "three-d", "kiss3d", "nalgebra", "cgmath", "glam", "ultraviolet", "mint",
+            "image", "imageproc", "resvg", "tiny-skia", "lyon", "femtovg", "skulpin",
+              # Networking and protocols
+            "socket2", "mio", "polling", "async-io", "calloop", "quinn",
+            "rustls-pemfile", "trust-dns", "hickory-dns", "async-h1", "h2", "h3",
+            "websocket", "tokio-tungstenite", "tungstenite", "ws", "warp-ws",
             
-            # Concurrency
-            "rayon",
+            # Text processing and parsing
+            "regex", "regex-syntax", "pest", "pest_derive", "nom", "combine", "winnow",
+            "lalrpop", "chumsky", "logos", "lex", "yacc", "tree-sitter", "syntect",
+            "pulldown-cmark", "comrak", "markdown", "ammonia", "scraper", "kuchiki",
             
-            # Protocol buffers and gRPC
-            "prost", "tonic",
+            # System programming and OS interfaces
+            "libc", "winapi", "windows", "nix", "users", "sysinfo", "procfs", "psutil",
+            "notify", "inotify", "hotwatch", "signal-hook", "ctrlc", "daemonize",
+            "fork", "shared_memory", "memmap2", "mlock", "caps", "uzers",
+              # Testing and development tools
+            "criterion", "proptest", "quickcheck", "rstest", "serial_test", "mockall",
+            "httpmock", "assert_cmd", "assert_fs", "predicates", "tempfile",
+            "insta", "goldenfile", "similar", "difference", "pretty_assertions",
             
-            # Procedural macros
-            "syn", "quote", "proc-macro2",
+            # Configuration and environment
+            "config", "figment", "envy", "dotenv", "confy", "directories", "app_dirs",
+            "etcetera", "platform-dirs", "home", "which", "dunce", "normpath",
+            
+            # Logging and observability
+            "log", "env_logger", "tracing", "tracing-subscriber", "tracing-futures",
+            "tracing-actix-web", "tracing-log", "slog", "fern", "flexi_logger",
+            "log4rs", "simplelog", "stderrlog", "pretty_env_logger", "fast_log",
+            
+            # Time and date
+            "chrono", "time", "humantime", "chrono-tz", "chrono-english", "ical",
+            "cron", "tokio-cron-scheduler", "job_scheduler", "delay_timer",
             
             # Machine Learning & AI
             "tokenizers", "safetensors", "linfa", "ndarray", "smartcore", "burn", 

{rust_crate_pipeline-1.2.3 → rust_crate_pipeline-1.2.5}/rust_crate_pipeline/production_config.py

@@ -12,14 +12,8 @@ import os
 def configure_production_logging():
     """Configure logging for production to reduce verbose warnings"""
     
-    # Set up logging format
-    logging.basicConfig(
-        level=logging.INFO,  # Default to INFO level
-        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-        datefmt='%Y-%m-%d %H:%M:%S'
-    )
-    
-    # Set specific loggers to less verbose levels
+    # Don't use basicConfig here - let main.py handle it
+    # Just set specific loggers to less verbose levels
     logging.getLogger('requests').setLevel(logging.WARNING)
     logging.getLogger('urllib3').setLevel(logging.WARNING)
     logging.getLogger('requests_cache').setLevel(logging.WARNING)

{rust_crate_pipeline-1.2.3 → rust_crate_pipeline-1.2.5}/rust_crate_pipeline/version.py

@@ -1,6 +1,6 @@
 """Version information for rust-crate-pipeline."""
 
-__version__ = "1.2.3"
+__version__ = "1.2.5"
 __version_info__ = tuple(int(x) for x in __version__.split("."))
 
 # Version history

{rust_crate_pipeline-1.2.3 → rust_crate_pipeline-1.2.5/rust_crate_pipeline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rust-crate-pipeline
-Version: 1.2.3
+Version: 1.2.5
 Summary: A comprehensive system for gathering, enriching, and analyzing metadata for Rust crates using AI-powered insights
 Home-page: https://github.com/DaveTmire85/SigilDERG-Data_Production
 Author: SuperUser666-Sigil

{rust_crate_pipeline-1.2.3 → rust_crate_pipeline-1.2.5}/rust_crate_pipeline.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@ LICENSE
 MANIFEST.in
 README.md
 pyproject.toml
+requirements-dev.txt
 requirements.txt
 setup.py
 rust_crate_pipeline/__init__.py

{rust_crate_pipeline-1.2.3 → rust_crate_pipeline-1.2.5}/setup.py

@@ -8,7 +8,7 @@ with open("requirements.txt", "r", encoding="utf-8") as fh:
 
 setup(
     name="rust-crate-pipeline",
-    version="1.2.3",
+    version="1.2.5",
     author="SuperUser666-Sigil",
     author_email="miragemodularframework@gmail.com",
     description="A comprehensive system for gathering, enriching, and analyzing metadata for Rust crates using AI-powered insights",