rust-crate-pipeline 1.2.3__py3-none-any.whl → 1.2.5__py3-none-any.whl

rust_crate_pipeline/main.py

@@ -111,15 +111,52 @@ Examples:
     return parser.parse_args()
 
 def configure_logging(log_level: str = 'INFO'):
+    """Configure logging with both console and file output"""
     level = getattr(logging, log_level.upper())
-    logging.basicConfig(
-        level=level,
-        format="%(asctime)s [%(levelname)s] %(message)s",
-        handlers=[
-            logging.StreamHandler(),
-            logging.FileHandler(f"crate_enrichment_{time.strftime('%Y%m%d-%H%M%S')}.log")
-        ]
+    
+    # Clear any existing handlers to avoid conflicts
+    root_logger = logging.getLogger()
+    for handler in root_logger.handlers[:]:
+        root_logger.removeHandler(handler)
+    
+    # Set root logger level
+    root_logger.setLevel(level)
+    
+    # Create formatters
+    detailed_formatter = logging.Formatter(
+        "%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+        datefmt='%Y-%m-%d %H:%M:%S'
     )
+    simple_formatter = logging.Formatter(
+        "%(asctime)s [%(levelname)s] %(message)s"
+    )
+    
+    # Console handler
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(level)
+    console_handler.setFormatter(simple_formatter)
+    root_logger.addHandler(console_handler)
+    
+    # File handler with unique timestamp
+    log_filename = f"crate_enrichment_{time.strftime('%Y%m%d-%H%M%S')}.log"
+    try:
+        file_handler = logging.FileHandler(log_filename, mode='w', encoding='utf-8')
+        file_handler.setLevel(logging.DEBUG)  # Always capture DEBUG+ to file
+        file_handler.setFormatter(detailed_formatter)
+        root_logger.addHandler(file_handler)
+        
+        # Log a test message to verify file handler works
+        logging.info(f"Logging initialized - file: {log_filename}")
+        
+    except Exception as e:
+        logging.error(f"Failed to create log file {log_filename}: {e}")
+        print(f"Warning: Could not create log file: {e}")
+    
+    # Set library loggers to less verbose levels
+    logging.getLogger('requests').setLevel(logging.WARNING)
+    logging.getLogger('urllib3').setLevel(logging.WARNING)
+    logging.getLogger('requests_cache').setLevel(logging.WARNING)
+    logging.getLogger('llama_cpp').setLevel(logging.WARNING)
 
 def check_disk_space():
     if shutil.disk_usage(".").free < 1_000_000_000:  # 1GB

rust_crate_pipeline/pipeline.py

@@ -29,37 +29,98 @@ class CrateDataPipeline:
     def get_crate_list(self, limit: Optional[int] = None) -> List[str]:
         """Return a comprehensive list of all high-value crates to process"""
         crates = [
-            # Web frameworks
-            "actix-web", "rocket", "axum", "warp", "tower",
+            # Web frameworks and servers
+            "actix-web", "rocket", "axum", "warp", "tower", "tide", "gotham", "iron", 
+            "nickel", "rouille", "thruster", "poem", "salvo", "viz", "ntex", "may-minihttp",
+            "tiny_http", "httptest", "mockito", "wiremock",
             
             # Async runtimes and utilities
-            "tokio", "tokio-stream", "async-trait", "futures",
+            "tokio", "tokio-stream", "async-trait", "futures", "async-std", "smol", 
+            "embassy", "embassy-executor", "embassy-time", "embassy-sync", "async-channel",
+            "async-broadcast", "async-lock", "async-once", "async-recursion", "futures-util",
+            "futures-channel", "futures-timer", "futures-test", "pin-project", "pin-project-lite",
             
             # Serialization/deserialization
-            "serde", "serde_json", "serde_yaml", "bincode",
+            "serde", "serde_json", "serde_yaml", "bincode", "toml", "ron", "postcard",
+            "ciborium", "rmp-serde", "quick-xml", "roxmltree", "serde_cbor", "serde_derive",
+            "serde_repr", "serde_with", "serde_bytes", "flexbuffers", "bson", "avro-rs",
             
-            # Error handling
-            "anyhow", "thiserror",
+            # Error handling and debugging
+            "anyhow", "thiserror", "eyre", "color-eyre", "miette", "fehler", "snafu",
+            "failure", "quick-error", "derive_more", "displaydoc", "backtrace", "better-panic",
+              # Command line and terminal
+            "clap", "structopt", "argh", "gumdrop", "docopt", "getopts", "pico-args",
+            "crossterm", "termion", "console", "indicatif", "dialoguer", "termcolor",
+            "colored", "yansi", "owo-colors", "nu-ansi-term", "terminal_size",
+              # Utilities and general purpose
+            "rand", "uuid", "itertools", "num", "cfg-if", "bytes", "mime", 
+            "form_urlencoded", "csv", "once_cell", "base64", "flate2", "tar", "dirs",
+            "walkdir", "glob", "bitflags", "indexmap", "smallvec", "arrayvec", "tinyvec", 
+            "ahash", "fxhash", "rustc-hash", "seahash", "siphasher", "wyhash", "xxhash-rust", 
+            "getrandom", "fastrand", "nanorand", "url", "percent-encoding", "unicode-segmentation",
+            "unicode-normalization", "unicode-width", "memchr", "aho-corasick", "bstr",
+              # HTTP clients and servers
+            "reqwest", "hyper", "surf", "ureq", "attohttpc", "isahc", "curl", "libcurl-sys",
+            "http", "http-body", "httparse", "hyper-tls", "hyper-rustls", "native-tls",
+            "webpki", "webpki-roots",
             
-            # Utilities
-            "rand", "uuid", "chrono", "regex", "log", "env_logger", "clap", "crossterm", 
-            "itertools", "num", "cfg-if", "bytes", "mime", "form_urlencoded", "parking_lot", 
-            "csv", "lazy_static", "once_cell", "tracing", "base64", "sha2", "flate2", "tar",
+            # Database and storage
+            "sqlx", "diesel", "postgres", "rusqlite", "mysql", "mongodb", "redis",
+            "tokio-postgres", "deadpool-postgres", "bb8", "r2d2", "sea-orm", "rbatis",
+            "sled", "rocksdb", "lmdb", "redb", "pickledb", "persy", "heed", "fjall",
+              # Concurrency and parallelism
+            "rayon", "crossbeam", "crossbeam-channel", "crossbeam-utils", "crossbeam-epoch",
+            "crossbeam-deque", "parking_lot", "spin", "atomic", "arc-swap", "dashmap",
+            "flume", "kanal", "tokio-util", "futures-concurrency",
+              # Protocol buffers, gRPC, and messaging
+            "prost", "tonic", "protobuf", "grpcio", "tarpc", "capnp", "rmp",
+            "zmq", "nanomsg", "nats", "rdkafka", "pulsar", "lapin", "amqp", "rumqttc",
+              # Procedural macros and metaprogramming
+            "syn", "quote", "proc-macro2", "proc-macro-crate", "proc-macro-error",
+            "darling", "derive_builder", "strum", "strum_macros",
+            "enum-iterator", "num-derive", "num-traits", "paste", "lazy_static",
             
-            # HTTP clients and servers
-            "reqwest", "hyper",
+            # Cryptography and security
+            "ring", "rustls", "openssl", "sha2", "sha3", "blake2", "blake3", "md5",
+            "hmac", "pbkdf2", "scrypt", "argon2", "bcrypt", "chacha20poly1305",
+            "aes-gcm", "rsa", "ed25519-dalek", "x25519-dalek", "curve25519-dalek",
+            "secp256k1", "k256", "p256", "ecdsa", "signature", "rand_core",
             
-            # Database
-            "sqlx", "diesel", "postgres", "rusqlite",
+            # Game development and graphics
+            "bevy", "macroquad", "ggez", "piston", "winit", "wgpu", "vulkano", "glium",
+            "three-d", "kiss3d", "nalgebra", "cgmath", "glam", "ultraviolet", "mint",
+            "image", "imageproc", "resvg", "tiny-skia", "lyon", "femtovg", "skulpin",
+              # Networking and protocols
+            "socket2", "mio", "polling", "async-io", "calloop", "quinn",
+            "rustls-pemfile", "trust-dns", "hickory-dns", "async-h1", "h2", "h3",
+            "websocket", "tokio-tungstenite", "tungstenite", "ws", "warp-ws",
             
-            # Concurrency
-            "rayon",
+            # Text processing and parsing
+            "regex", "regex-syntax", "pest", "pest_derive", "nom", "combine", "winnow",
+            "lalrpop", "chumsky", "logos", "lex", "yacc", "tree-sitter", "syntect",
+            "pulldown-cmark", "comrak", "markdown", "ammonia", "scraper", "kuchiki",
             
-            # Protocol buffers and gRPC
-            "prost", "tonic",
+            # System programming and OS interfaces
+            "libc", "winapi", "windows", "nix", "users", "sysinfo", "procfs", "psutil",
+            "notify", "inotify", "hotwatch", "signal-hook", "ctrlc", "daemonize",
+            "fork", "shared_memory", "memmap2", "mlock", "caps", "uzers",
+              # Testing and development tools
+            "criterion", "proptest", "quickcheck", "rstest", "serial_test", "mockall",
+            "httpmock", "assert_cmd", "assert_fs", "predicates", "tempfile",
+            "insta", "goldenfile", "similar", "difference", "pretty_assertions",
             
-            # Procedural macros
-            "syn", "quote", "proc-macro2",
+            # Configuration and environment
+            "config", "figment", "envy", "dotenv", "confy", "directories", "app_dirs",
+            "etcetera", "platform-dirs", "home", "which", "dunce", "normpath",
+            
+            # Logging and observability
+            "log", "env_logger", "tracing", "tracing-subscriber", "tracing-futures",
+            "tracing-actix-web", "tracing-log", "slog", "fern", "flexi_logger",
+            "log4rs", "simplelog", "stderrlog", "pretty_env_logger", "fast_log",
+            
+            # Time and date
+            "chrono", "time", "humantime", "chrono-tz", "chrono-english", "ical",
+            "cron", "tokio-cron-scheduler", "job_scheduler", "delay_timer",
             
             # Machine Learning & AI
             "tokenizers", "safetensors", "linfa", "ndarray", "smartcore", "burn", 

rust_crate_pipeline/production_config.py

@@ -12,14 +12,8 @@ import os
 def configure_production_logging():
     """Configure logging for production to reduce verbose warnings"""
     
-    # Set up logging format
-    logging.basicConfig(
-        level=logging.INFO,  # Default to INFO level
-        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-        datefmt='%Y-%m-%d %H:%M:%S'
-    )
-    
-    # Set specific loggers to less verbose levels
+    # Don't use basicConfig here - let main.py handle it
+    # Just set specific loggers to less verbose levels
     logging.getLogger('requests').setLevel(logging.WARNING)
     logging.getLogger('urllib3').setLevel(logging.WARNING)
     logging.getLogger('requests_cache').setLevel(logging.WARNING)

rust_crate_pipeline/version.py

@@ -1,6 +1,6 @@
 """Version information for rust-crate-pipeline."""
 
-__version__ = "1.2.3"
+__version__ = "1.2.5"
 __version_info__ = tuple(int(x) for x in __version__.split("."))
 
 # Version history

{rust_crate_pipeline-1.2.3.dist-info → rust_crate_pipeline-1.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rust-crate-pipeline
-Version: 1.2.3
+Version: 1.2.5
 Summary: A comprehensive system for gathering, enriching, and analyzing metadata for Rust crates using AI-powered insights
 Home-page: https://github.com/DaveTmire85/SigilDERG-Data_Production
 Author: SuperUser666-Sigil

{rust_crate_pipeline-1.2.3.dist-info → rust_crate_pipeline-1.2.5.dist-info}/RECORD

@@ -4,16 +4,16 @@ rust_crate_pipeline/ai_processing.py,sha256=B93rCDdxE-UkYMjmT0UotQTahx9-Lgzec7_b
 rust_crate_pipeline/analysis.py,sha256=ijP4zp3cFnN09nZkeCluyAvbyAtAW_M2YSxALpQX8LY,18615
 rust_crate_pipeline/config.py,sha256=r4Y_5SD-lfrM1112edk9T0S0MiVxaNSSHk4q2yDrM88,1528
 rust_crate_pipeline/github_token_checker.py,sha256=MJqHP8J84NEZ6nzdutpC7iRnsP0kyqscjLUosvmI4MI,3768
-rust_crate_pipeline/main.py,sha256=J8ORQA6s3wyWw2R3oB_IEm2J5tx1CFdspw5kb5Ep8zQ,6323
+rust_crate_pipeline/main.py,sha256=Wz4Q4TX-G7qvLNMyYT6cHbgRCeMJoWILCvXcJr1FYAc,7876
 rust_crate_pipeline/network.py,sha256=t_G8eh_WHNugm_laMftcWVbHsmP0bOlTPnVW9DqF6SU,13375
-rust_crate_pipeline/pipeline.py,sha256=Uwfw4uLL3aN1gJl5xSwvvyaY9ceeP7LVr02IzNx0tPM,12033
-rust_crate_pipeline/production_config.py,sha256=2GT8bxytcrMRrcfjzpay5RTtATE3rbmDvNUBvVhrYSQ,2472
-rust_crate_pipeline/version.py,sha256=r_w4Eokm27opXYKcOCTKax8TO7pFI5E3TkB0L9c62yY,1022
+rust_crate_pipeline/pipeline.py,sha256=fcWgqKC0teGeVyNbwayFwngoZLJGWwWZAlWtMqwtdyY,17074
+rust_crate_pipeline/production_config.py,sha256=TdvmO1SIRpex1xZ0AymTKXpLfkkvOG44Jyy7S5M-u7k,2304
+rust_crate_pipeline/version.py,sha256=coYkryx3UFGAYS5obh7aBC1n8hhQIMhIeFhlnGfggeg,1022
 rust_crate_pipeline/utils/file_utils.py,sha256=lnHeLrt1JYaQhRDKtA1TWR2HIyRO8zwOyWb-KmAmWgk,2126
 rust_crate_pipeline/utils/logging_utils.py,sha256=O4Jnr_k9dBchrVqXf-vqtDKgizDtL_ljh8g7G2VCX_c,2241
-rust_crate_pipeline-1.2.3.dist-info/licenses/LICENSE,sha256=tpd4XNpbssrSx9-iErATOLrOh0ivNPfO2I5MAPUpats,1088
-rust_crate_pipeline-1.2.3.dist-info/METADATA,sha256=1bU7P1g6veyD0hJ78cjGJcVWRTujAF6Q6RL_CV_MVIY,16741
-rust_crate_pipeline-1.2.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rust_crate_pipeline-1.2.3.dist-info/entry_points.txt,sha256=9Rr_IRuFRIridXxUSdEJbB3ba0NnpEfKmknZXFdYRC0,70
-rust_crate_pipeline-1.2.3.dist-info/top_level.txt,sha256=GUdB7RyxHLhijQxui_KTy3B8p_L2APui9C6RYa0FuaE,20
-rust_crate_pipeline-1.2.3.dist-info/RECORD,,
+rust_crate_pipeline-1.2.5.dist-info/licenses/LICENSE,sha256=tpd4XNpbssrSx9-iErATOLrOh0ivNPfO2I5MAPUpats,1088
+rust_crate_pipeline-1.2.5.dist-info/METADATA,sha256=tCIAwMZ41r7K7BesgCHH27f8S_5UMdqsujdu9MPDePk,16741
+rust_crate_pipeline-1.2.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rust_crate_pipeline-1.2.5.dist-info/entry_points.txt,sha256=9Rr_IRuFRIridXxUSdEJbB3ba0NnpEfKmknZXFdYRC0,70
+rust_crate_pipeline-1.2.5.dist-info/top_level.txt,sha256=GUdB7RyxHLhijQxui_KTy3B8p_L2APui9C6RYa0FuaE,20
+rust_crate_pipeline-1.2.5.dist-info/RECORD,,