rust-crate-pipeline 1.3.6__py3-none-any.whl → 1.4.1__py3-none-any.whl

rust_crate_pipeline/core/canon_registry.py

@@ -26,7 +26,7 @@ class CanonRegistry:
     def __init__(self) -> None:
         self.canon_entries: Dict[str, CanonEntry] = {}
         self.authority_chain: List[str] = []
-        self.version = "1.3.0"
+        self.version = "1.4.0"
         self.logger = logging.getLogger(__name__)
         
         self._initialize_default_canon()
@@ -36,7 +36,7 @@ class CanonRegistry:
             "crates.io": {
                 "authority_level": 10,
                 "base_url": "https://crates.io/api/v1/",
-                "version": "1.3.0",
+                "version": "1.4.0",
                 "last_validated": datetime.now(timezone.utc).isoformat(),
             },
             "github.com": {

rust_crate_pipeline/github_token_checker.py

@@ -1,108 +1,108 @@
-from typing import Dict, List, Tuple, Optional, Any
-# github_token_checker.py
-"""
-GitHub Token Checker Module
-Lightweight version of the token checker for integration into the main pipeline.
-"""
-
-import os
-import sys
-import requests
-import logging
-
-
-def check_github_token_quick() -> None:
-    """Quick check if GitHub token is available and valid"""
-    token = os.getenv("GITHUB_TOKEN")
-
-    if not token:
-        return False, "GITHUB_TOKEN environment variable not set"
-
-    if len(token) < 20:
-        return False, "GITHUB_TOKEN seems too short - may be invalid"
-
-    try:
-        # Quick API check
-        headers = {
-            "Accept": "application/vnd.github.v3+json",
-            "Authorization": f"token {token}",
-        }
-
-        response = requests.get(
-            "https://api.github.com/rate_limit", headers=headers, timeout=10
-        )
-
-        if response.status_code == 200:
-            data = response.json()
-            remaining = data["resources"]["core"]["remaining"]
-            return True, f"Token valid, {remaining} API calls remaining"
-        elif response.status_code == 401:
-            return False, "GitHub token is invalid or expired"
-        else:
-            return (
-                False,
-                f"GitHub API returned status code: {response.status_code}",
-            )
-    except requests.RequestException as e:
-        return False, f"API request failed: {e}"
-    except Exception as e:
-        return False, f"Error checking token: {str(e)}"
-
-
-def prompt_for_token_setup() -> None:
-    """Prompt user to set up GitHub token"""
-    print("\n" + "=" * 60)
-    print("[KEY] GitHub Token Required")
-    print("=" * 60)
-    print("\nThe Rust Crate Pipeline requires a GitHub Personal Access Token")
-    print("to access repository information and avoid rate limits.")
-    print("\n[GUIDE] Quick Setup:")
-    print("1. Get token: https://github.com/settings/tokens")
-    print("2. Required scopes: public_repo, read:user")
-    print("3. Set in environment:")
-    print('   export GITHUB_TOKEN="your_token_here"')
-    print("\n[TOOLS] Setup Scripts Available:")
-    print("   ./setup_github_token.sh    (Interactive setup)")
-    print("   python3 check_github_token.py    (Full verification)")
-    print("\n" + "=" * 60)
-
-    # Ask if user wants to continue without token (limited functionality)
-    response = input("\nContinue without GitHub token? (y/N): ").strip().lower()
-
-    if response in ["y", "yes"]:
-        print("[WARNING] Running with limited GitHub API access (60 requests/hour)")
-        print("   You may encounter rate limit warnings.")
-        return True
-    else:
-        print("\n[STOP] Please set up your GitHub token and try again.")
-        return False
-
-
-def check_and_setup_github_token() -> None:
-    """Checks and sets up the GitHub token."""
-    is_valid, message = check_github_token_quick()
-
-    if is_valid:
-        logging.debug(f"GitHub token check: {message}")
-        return True
-
-    # Token is missing or invalid
-    logging.warning(f"GitHub token issue: {message}")
-
-    # Check if we're in a non-interactive environment
-    if not sys.stdin.isatty():
-        logging.error("GitHub token not configured and running in non-interactive mode")
-        logging.error("Set GITHUB_TOKEN environment variable before running")
-        return False
-
-    # Interactive prompt
-    return prompt_for_token_setup()
-
-
-if __name__ == "__main__":
-    # Allow running this module directly for testing
-    is_valid, message = check_github_token_quick()
-    print(f"Token check: {'[OK]' if is_valid else '[FAIL]'} {message}")
-
-    if not is_valid:
-        check_and_setup_github_token()
+from typing import Dict, List, Tuple, Optional, Any
+# github_token_checker.py
+"""
+GitHub Token Checker Module
+Lightweight version of the token checker for integration into the main pipeline.
+"""
+
+import os
+import sys
+import requests
+import logging
+
+
+def check_github_token_quick() -> tuple[bool, str]:
+    """Quick check if GitHub token is available and valid"""
+    token = os.getenv("GITHUB_TOKEN")
+
+    if not token:
+        return False, "GITHUB_TOKEN environment variable not set"
+
+    if len(token) < 20:
+        return False, "GITHUB_TOKEN seems too short - may be invalid"
+
+    try:
+        # Quick API check
+        headers = {
+            "Accept": "application/vnd.github.v3+json",
+            "Authorization": f"token {token}",
+        }
+
+        response = requests.get(
+            "https://api.github.com/rate_limit", headers=headers, timeout=10
+        )
+
+        if response.status_code == 200:
+            data = response.json()
+            remaining = data["resources"]["core"]["remaining"]
+            return True, f"Token valid, {remaining} API calls remaining"
+        elif response.status_code == 401:
+            return False, "GitHub token is invalid or expired"
+        else:
+            return (
+                False,
+                f"GitHub API returned status code: {response.status_code}",
+            )
+    except requests.RequestException as e:
+        return False, f"API request failed: {e}"
+    except Exception as e:
+        return False, f"Error checking token: {str(e)}"
+
+
+def prompt_for_token_setup() -> bool:
+    """Prompt user to set up GitHub token"""
+    print("\n" + "=" * 60)
+    print("[KEY] GitHub Token Required")
+    print("=" * 60)
+    print("\nThe Rust Crate Pipeline requires a GitHub Personal Access Token")
+    print("to access repository information and avoid rate limits.")
+    print("\n[GUIDE] Quick Setup:")
+    print("1. Get token: https://github.com/settings/tokens")
+    print("2. Required scopes: public_repo, read:user")
+    print("3. Set in environment:")
+    print('   export GITHUB_TOKEN="your_token_here"')
+    print("\n[TOOLS] Setup Scripts Available:")
+    print("   ./setup_github_token.sh    (Interactive setup)")
+    print("   python3 check_github_token.py    (Full verification)")
+    print("\n" + "=" * 60)
+
+    # Ask if user wants to continue without token (limited functionality)
+    response = input("\nContinue without GitHub token? (y/N): ").strip().lower()
+
+    if response in ["y", "yes"]:
+        print("[WARNING] Running with limited GitHub API access (60 requests/hour)")
+        print("   You may encounter rate limit warnings.")
+        return True
+    else:
+        print("\n[STOP] Please set up your GitHub token and try again.")
+        return False
+
+
+def check_and_setup_github_token() -> bool:
+    """Checks and sets up the GitHub token."""
+    is_valid, message = check_github_token_quick()
+
+    if is_valid:
+        logging.debug(f"GitHub token check: {message}")
+        return True
+
+    # Token is missing or invalid
+    logging.warning(f"GitHub token issue: {message}")
+
+    # Check if we're in a non-interactive environment
+    if not sys.stdin.isatty():
+        logging.error("GitHub token not configured and running in non-interactive mode")
+        logging.error("Set GITHUB_TOKEN environment variable before running")
+        return False
+
+    # Interactive prompt
+    return prompt_for_token_setup()
+
+
+if __name__ == "__main__":
+    # Allow running this module directly for testing
+    is_valid, message = check_github_token_quick()
+    print(f"Token check: {'[OK]' if is_valid else '[FAIL]'} {message}")
+
+    if not is_valid:
+        check_and_setup_github_token()

rust_crate_pipeline/version.py

@@ -1,7 +1,7 @@
 from typing import Dict, List, Tuple, Optional, Any
 """Version information for rust-crate-pipeline."""
 
-__version__ = "1.3.6"
+__version__ = "1.4.1"
 __version_info__ = tuple(int(x) for x in __version__.split("-")[0].split("."))
 __author__ = "SigilDERG Team"
 __email__ = "sigilderg@example.com"

{rust_crate_pipeline-1.3.6.dist-info → rust_crate_pipeline-1.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rust-crate-pipeline
-Version: 1.3.6
+Version: 1.4.1
 Summary: A comprehensive pipeline for analyzing Rust crates with AI enrichment and enhanced scraping
 Home-page: https://github.com/SigilDERG/rust-crate-pipeline
 Author: SigilDERG Team
@@ -386,6 +386,13 @@ docker run -it -v $(pwd):/app rust-crate-pipeline
 
 ## Recent Improvements
 
+### Version 1.4.0
+- **Security**: Robust Ed25519/RSA cryptographic signing and provenance
+- **Automation**: Automated RAG and provenance workflows
+- **CI/CD**: Improved GitHub Actions for validation and publishing
+- **Docker**: Updated Docker image and compose for new version
+- **Bug Fixes**: Workflow and validation fixes for Ed25519
+
 ### Version 1.3.6
 - **Python 3.12+ Requirement**: Updated to use modern type annotations and language features
 - **Type Safety**: Enhanced type annotations throughout the codebase with modern syntax
@@ -453,4 +460,56 @@ Or, text attribution:
 
 ```
 This project uses Crawl4AI (https://github.com/unclecode/crawl4ai) for web data extraction.
-``` 
+```
+
+## 🚀 Unified, Cross-Platform, Multi-Provider LLM Support
+
+This project supports **all major LLM providers** (cloud and local) on **Mac, Linux, and Windows** using a single, unified interface. All LLM calls are routed through the `UnifiedLLMProcessor` and `LLMConfig` abstractions, ensuring:
+
+- **One code path for all providers:** Azure OpenAI, OpenAI, Anthropic, Google, Cohere, HuggingFace, Ollama, LM Studio, and any OpenAI-compatible endpoint.
+- **Cross-platform compatibility:** Works out of the box on Mac, Linux, and Windows.
+- **Configurable via CLI and config files:** Select provider, model, API key, endpoint, and provider-specific options at runtime.
+- **Easy extensibility:** Add new providers by updating your config or CLI arguments—no code changes needed.
+
+### 📖 Provider Setup & Usage
+- See [`README_LLM_PROVIDERS.md`](./README_LLM_PROVIDERS.md) for full details, setup instructions, and usage examples for every supported provider.
+- Run `python run_pipeline_with_llm.py --help` for CLI options and provider-specific arguments.
+
+### 🧩 Example Usage
+```bash
+# Azure OpenAI
+python run_pipeline_with_llm.py --llm-provider azure --llm-model gpt-4o --crates tokio
+
+# Ollama (local)
+python run_pipeline_with_llm.py --llm-provider ollama --llm-model llama2 --crates serde
+
+# OpenAI API
+python run_pipeline_with_llm.py --llm-provider openai --llm-model gpt-4 --llm-api-key YOUR_KEY --crates tokio
+
+# Anthropic Claude
+python run_pipeline_with_llm.py --llm-provider anthropic --llm-model claude-3-sonnet --llm-api-key YOUR_KEY --crates serde
+```
+
+### 🔒 Security & Best Practices
+- Store API keys as environment variables.
+- Use local providers (Ollama, LM Studio) for full privacy—no data leaves your machine.
+- All LLM calls are routed through a single, auditable interface for maximum maintainability and security.
+
+### 🧪 Testing
+- Run `python test_unified_llm.py` to verify provider support and configuration.
+
+For more, see [`README_LLM_PROVIDERS.md`](./README_LLM_PROVIDERS.md) and the CLI help output. 
+
+## Public RAG Database Hash Verification
+
+The canonical hash of the RAG SQLite database (`sigil_rag_cache.db`) is stored in the public file `sigil_rag_cache.hash`.
+
+- **Purpose:** Anyone can verify the integrity of the RAG database by comparing its SHA256 hash to the value in `sigil_rag_cache.hash`.
+- **How to verify:**
+
+```sh
+python audits/validate_db_hash.py --db sigil_rag_cache.db --expected-hash "$(cat sigil_rag_cache.hash)"
+```
+
+- **CI/CD:** The GitHub Actions workflow `.github/workflows/validate-db-hash.yml` automatically checks this on every push.
+- **No secrets required:** The hash is public and verifiable by anyone.

{rust_crate_pipeline-1.3.6.dist-info → rust_crate_pipeline-1.4.1.dist-info}/RECORD

@@ -1,12 +1,12 @@
 rust_crate_pipeline/__init__.py,sha256=ZJCApGu8h2Rn5-dkoBLXOpdoeD6b36w76--o0fEismQ,1749
 rust_crate_pipeline/__main__.py,sha256=PexSWQYtbFQg5P36WEnJ0X-oAtT8WDej3bIJoSAcCCQ,157
-rust_crate_pipeline/ai_processing.py,sha256=MP6VcvV3Jw2Pjof3NrewjTmO8ruVyJKcJGa9zhS_2eY,24140
+rust_crate_pipeline/ai_processing.py,sha256=Q_jmIL0OzFcP6zSKTgrIikUTHuUB3Py4MqwLXmB7-KQ,29057
 rust_crate_pipeline/analysis.py,sha256=_cmjynLWaQbGIdLQHU3P3rfqHB3gcNNgCdzStbsKrdw,17021
-rust_crate_pipeline/azure_ai_processing.py,sha256=kxbHGNSRSD_5KNkL2ihqCASJq8kdnb_N9u1-ogXbneE,16449
+rust_crate_pipeline/azure_ai_processing.py,sha256=h2ZUaFPt5LmTH--5CXfXBdbKnoJA4Ha8zCfbLawhDz8,16409
 rust_crate_pipeline/config.py,sha256=Fw3fRKCZawKaLQi7YqsmNNku4whZi89mWzr8BVRNS5E,3009
 rust_crate_pipeline/crate_analysis.py,sha256=GsoXemJ9VFyAbb4Sm5gY5ToTqNtOA4pI38AtngAQONk,2090
 rust_crate_pipeline/crate_list.txt,sha256=W3NxDtxvihyKp9SN85FYXX6p8Hh49IFih1M4-c-CynM,4334
-rust_crate_pipeline/github_token_checker.py,sha256=COXXS9uoLV9WYIcT02C-bV5uH3fa9D9HJImc07vMjLs,3766
+rust_crate_pipeline/github_token_checker.py,sha256=0IpTh78DSaw4znaed031cSVSZDsi92eDManPzRIIN3Y,3670
 rust_crate_pipeline/main.py,sha256=iGYEAYvXkoFFvaA6DIVGiUL3wLhiCzatB6Fvf-Yrj2A,18858
 rust_crate_pipeline/network.py,sha256=mWjiRvOX31piBZ2QiJ-F75DBD4l6cqzTXcQdJvHxe90,12718
 rust_crate_pipeline/pipeline.py,sha256=CqPHLLRvMOpy-3ONL6hnPahV6Vh6S4M8oDsHd_lDrPc,16203
@@ -14,18 +14,18 @@ rust_crate_pipeline/production_config.py,sha256=uWylP9AIZZx7-9aT4sFmAKEEW9miJDxa
 rust_crate_pipeline/progress_monitor.py,sha256=5K9KP-Xggi1JEINfRmq2W-wGUHtNIBTcocpDtB1t8iM,13743
 rust_crate_pipeline/unified_llm_processor.py,sha256=eo7KotNuqwc7_hgpFm18QLokFoufFslnvi8TnDsSYEg,25064
 rust_crate_pipeline/unified_pipeline.py,sha256=2yglmXVlQfSkVq0HVTPonDee6VxWaQWZw0X2l4lLBGw,23704
-rust_crate_pipeline/version.py,sha256=izXdwKOkBxecVcCuMmOVbZnu5y-hHZZkEg39LmBPnis,4481
+rust_crate_pipeline/version.py,sha256=whkmTDquEVytez4svUFUBfbfK0EOvDTPA8K5TuZffbE,4481
 rust_crate_pipeline/core/__init__.py,sha256=Sq4HWdANGqoYln7JdCog7m3BsGeR3tHdseeflvNetoQ,509
-rust_crate_pipeline/core/canon_registry.py,sha256=36tmt_wU6-kSyZnGfh53N64C7E3G-QR7GFbr9epj4zg,4700
+rust_crate_pipeline/core/canon_registry.py,sha256=_3cu0akJvLc7ZnomMaLeMa8adOBYn1dtjpB0yE3vGL8,4700
 rust_crate_pipeline/core/irl_engine.py,sha256=QRZUdkN24W9XutLkj8JDplEz6FmnquUrwKsl0s2zRr4,10491
 rust_crate_pipeline/core/sacred_chain.py,sha256=6s4gFLDT6KUwuu0Fpxu6h_YHlsEvHZb3CQw4tRHGyDU,3773
 rust_crate_pipeline/scraping/__init__.py,sha256=ySkTRg7nIxgcbHJQ3L1XzcrOo281NZu07-XtiGi-558,307
 rust_crate_pipeline/scraping/unified_scraper.py,sha256=ZE2gkc0vQ3BOLdSX_IV-kMe8QAm2Av4M7VqpkxEKyT4,9965
 rust_crate_pipeline/utils/file_utils.py,sha256=tMaCPy7ghs9x4Hxu_sviX8MXU2sBjNvohUrvt4MejoM,2853
 rust_crate_pipeline/utils/logging_utils.py,sha256=e5jG0Yd6k3exgAdbVca46kWADJ_Qz8UJ3yEJzwTqPyI,2452
-rust_crate_pipeline-1.3.6.dist-info/licenses/LICENSE,sha256=tpd4XNpbssrSx9-iErATOLrOh0ivNPfO2I5MAPUpats,1088
-rust_crate_pipeline-1.3.6.dist-info/METADATA,sha256=BWUkQKtJCbXt1KZbmdofzy0eC4LpoeMBLKVXu3H3hD0,14539
-rust_crate_pipeline-1.3.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rust_crate_pipeline-1.3.6.dist-info/entry_points.txt,sha256=9Rr_IRuFRIridXxUSdEJbB3ba0NnpEfKmknZXFdYRC0,70
-rust_crate_pipeline-1.3.6.dist-info/top_level.txt,sha256=GUdB7RyxHLhijQxui_KTy3B8p_L2APui9C6RYa0FuaE,20
-rust_crate_pipeline-1.3.6.dist-info/RECORD,,
+rust_crate_pipeline-1.4.1.dist-info/licenses/LICENSE,sha256=tpd4XNpbssrSx9-iErATOLrOh0ivNPfO2I5MAPUpats,1088
+rust_crate_pipeline-1.4.1.dist-info/METADATA,sha256=OY5aKfWvpdRnLr9oKJ0SyX1N6evt1IYvu4J4GuFjwy0,17605
+rust_crate_pipeline-1.4.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rust_crate_pipeline-1.4.1.dist-info/entry_points.txt,sha256=9Rr_IRuFRIridXxUSdEJbB3ba0NnpEfKmknZXFdYRC0,70
+rust_crate_pipeline-1.4.1.dist-info/top_level.txt,sha256=GUdB7RyxHLhijQxui_KTy3B8p_L2APui9C6RYa0FuaE,20
+rust_crate_pipeline-1.4.1.dist-info/RECORD,,