promptforest 0.1.1__tar.gz → 0.5.0a0__tar.gz

{promptforest-0.1.1/promptforest.egg-info → promptforest-0.5.0a0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: promptforest
-Version: 0.1.1
+Version: 0.5.0a0
 Summary: Ensemble Prompt Injection Detection
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
@@ -36,6 +36,28 @@ This discrepancy score enables downstream workflows such as:
 - Adaptive throttling or alerting in production systems  
 - Continuous monitoring and model improvement  
 
+## Statistics
+PromptForest was evaluated against the SOTA model Qualifire Sentinel model (v2).
+
+| Metric                           | PromptForest | Sentinel v2 |
+| -------------------------------- | ------------ | ----------- |
+| Accuracy                         | 0.802        | 0.982       |
+| Avg Confidence on Wrong Answers  | 0.643        | 0.858       |
+| Expected Calibration Error (ECE) | 0.060        | 0.202       |
+| Approximate Model Size           | ~250M params  | 600M params |
+
+
+### Key Insights
+
+- Calibrated uncertainty: PromptForest is less confident on wrong predictions than Sentinel, resulting in a much lower ECE.
+
+- Parameter efficiency: Achieves competitive reliability with <50% of the parameters.
+
+- Interpretability: Confidence scores can be used to flag uncertain predictions for human review.
+
+Interpretation:
+While Sentinel has higher raw accuracy, PromptForest provides better-calibrated confidence. For systems where overconfidence on wrong answers is risky, PromptForest can reduce the chance of critical errors despite being smaller and faster.
+
 
 ## Supported Models
 
@@ -43,13 +65,12 @@ This discrepancy score enables downstream workflows such as:
 | ------------- | ----------------------------------------- |
 | **Meta**      | Llama Prompt Guard 86M (Built with Llama) |
 | **ProtectAI** | DebertaV3 Prompt Injection Finetune       |
-| **Deepset**   | DebertaV3-base Injection Finetune         |
-| **Katanemo**  | Arch-Guard                                |
+| **Vijil**     | Vijil Dome Prompt Injection Detection     |
 | **Appleroll** | PromptForest-XGBoost                      |
 
 ## Performance
-**Request Latency** \
-Best Case: 50ms \
+**E2E Request Latency** \
+Average Case: 100ms \
 Worst Case: 200ms
 
 **Accuracy** \

{promptforest-0.1.1 → promptforest-0.5.0a0}/README.md

@@ -11,6 +11,28 @@ This discrepancy score enables downstream workflows such as:
 - Adaptive throttling or alerting in production systems  
 - Continuous monitoring and model improvement  
 
+## Statistics
+PromptForest was evaluated against the SOTA model Qualifire Sentinel model (v2).
+
+| Metric                           | PromptForest | Sentinel v2 |
+| -------------------------------- | ------------ | ----------- |
+| Accuracy                         | 0.802        | 0.982       |
+| Avg Confidence on Wrong Answers  | 0.643        | 0.858       |
+| Expected Calibration Error (ECE) | 0.060        | 0.202       |
+| Approximate Model Size           | ~250M params  | 600M params |
+
+
+### Key Insights
+
+- Calibrated uncertainty: PromptForest is less confident on wrong predictions than Sentinel, resulting in a much lower ECE.
+
+- Parameter efficiency: Achieves competitive reliability with <50% of the parameters.
+
+- Interpretability: Confidence scores can be used to flag uncertain predictions for human review.
+
+Interpretation:
+While Sentinel has higher raw accuracy, PromptForest provides better-calibrated confidence. For systems where overconfidence on wrong answers is risky, PromptForest can reduce the chance of critical errors despite being smaller and faster.
+
 
 ## Supported Models
 
@@ -18,13 +40,12 @@ This discrepancy score enables downstream workflows such as:
 | ------------- | ----------------------------------------- |
 | **Meta**      | Llama Prompt Guard 86M (Built with Llama) |
 | **ProtectAI** | DebertaV3 Prompt Injection Finetune       |
-| **Deepset**   | DebertaV3-base Injection Finetune         |
-| **Katanemo**  | Arch-Guard                                |
+| **Vijil**     | Vijil Dome Prompt Injection Detection     |
 | **Appleroll** | PromptForest-XGBoost                      |
 
 ## Performance
-**Request Latency** \
-Best Case: 50ms \
+**E2E Request Latency** \
+Average Case: 100ms \
 Worst Case: 200ms
 
 **Accuracy** \

{promptforest-0.1.1 → promptforest-0.5.0a0}/promptforest/config.py

@@ -13,10 +13,9 @@ MODELS_DIR = USER_DATA_DIR / "models"
 DEFAULT_CONFIG = {
     "models": [
         {"name": "llama_guard", "path": "llama_guard", "type": "hf", "enabled": True},
-        {"name": "protectai", "path": "protectai_deberta", "type": "hf", "enabled": True},
-        {"name": "deepset", "path": "deepset_deberta", "type": "hf", "enabled": True},
-        {"name": "katanemo", "path": "katanemo_arch", "type": "hf", "enabled": True},
-        {"name": "xgboost", "type": "xgboost", "enabled": True}
+        {"name": "protectai", "path": "protectai", "type": "hf", "enabled": True},
+        {"name": "vijil", "path": "vijil_dome", "type": "hf", "enabled": True},
+        {"name": "xgboost", "type": "xgboost", "enabled": True, "threshold": 0.10}
     ],
     "settings": {
         "device": "auto",  # Options: auto, cuda, mps, cpu

{promptforest-0.1.1 → promptforest-0.5.0a0}/promptforest/download.py

@@ -13,9 +13,8 @@ from .llama_guard_86m_downloader import download_llama_guard
 
 # Configuration
 MODELS = {
-    "protectai_deberta": "protectai/deberta-v3-base-prompt-injection",
-    "deepset_deberta": "deepset/deberta-v3-base-injection",
-    "katanemo_arch": "katanemo/Arch-Guard"
+    "protectai": "protectai/deberta-v3-base-prompt-injection-v2",
+    "vijil_dome": "vijil/vijil_dome_prompt_injection_detection"
 }
 
 EMBEDDING_MODEL_NAME = 'all-MiniLM-L6-v2'
@@ -31,15 +30,20 @@ def _download_hf_model(name, model_id):
     try:
         if save_path.exists():
             return
+        
+        # Special handling for Vijil (ModernBERT tokenizer issue)
+        tokenizer_id = model_id
+        if "vijil" in name or "vijil" in model_id:
+            tokenizer_id = "answerdotai/ModernBERT-base"
 
-        tokenizer = AutoTokenizer.from_pretrained(model_id)
+        tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
         model = AutoModelForSequenceClassification.from_pretrained(model_id)
         
         tokenizer.save_pretrained(save_path)
         model.save_pretrained(save_path)
         
     except Exception as e:
-        print("Failed to download a model!")
+        print(f"Failed to download {model_id}: {e}")
 
 def _download_sentence_transformer():
     """Download and save the SentenceTransformer model."""

{promptforest-0.1.1 → promptforest-0.5.0a0}/promptforest/lib.py

@@ -68,7 +68,8 @@ class HFModel(ModelInference):
         id2label = self.model.config.id2label
         found = False
         for idx, label in id2label.items():
-            if any(kw in label.lower() for kw in MALICIOUS_KEYWORDS):
+            # Check if label is string before calling lower()
+            if isinstance(label, str) and any(kw in label.lower() for kw in MALICIOUS_KEYWORDS):
                 self.malicious_idx = idx
                 found = True
                 break
@@ -99,9 +100,11 @@ class HFModel(ModelInference):
 
 
 class XGBoostModel(ModelInference):
-    def __init__(self, settings):
+    def __init__(self, settings, config=None):
         self.name = "xgboost_custom"
         self.settings = settings
+        self.config = config or {}
+        self.threshold = self.config.get('threshold', 0.5)
         self.model = None
         self.embedder = None
         
@@ -140,7 +143,18 @@ class XGBoostModel(ModelInference):
         try:
             emb = self.embedder.encode([prompt])
             prob = self.model.predict_proba(emb)[0][1]
-            return float(prob)
+            prob = float(prob)
+
+            # Apply threshold adjustment if a custom threshold is set
+            if self.threshold != 0.5:
+                if prob < self.threshold:
+                    # Map [0, threshold) -> [0, 0.5)
+                    prob = 0.5 * (prob / self.threshold)
+                else:
+                    # Map [threshold, 1.0] -> [0.5, 1.0]
+                    prob = 0.5 + 0.5 * (prob - self.threshold) / (1.0 - self.threshold)
+                    
+            return prob
         except Exception:
             return 0.0
 
@@ -151,14 +165,14 @@ class EnsembleGuard:
         Initialize the EnsembleGuard.
         :param config: Dictionary containing configuration. If None, loads default/user config.
         """
-        # Check if models need to be downloaded
-        self._ensure_models_available()
-        
         if config is None:
             self.config = load_config()
         else:
             self.config = config
             
+        # Check if models need to be downloaded
+        self._ensure_models_available()
+            
         self.models = []
         self._init_models()
         self.device_used = get_device(self.config['settings'].get('device', 'auto'))
@@ -167,14 +181,27 @@ class EnsembleGuard:
         """Check if models are available, download if needed."""
         from .config import MODELS_DIR
         
-        # Check if models directory exists and has content
-        if MODELS_DIR.exists() and any(MODELS_DIR.iterdir()):
-            return
-        
-        # Models not found, download them
-        print("Models not found. Downloading...")
-        from .download import download_all
-        download_all()
+        missing = False
+        if not MODELS_DIR.exists():
+            missing = True
+        else:
+            # Check for each enabled HF model
+            for model_cfg in self.config.get('models', []):
+                if model_cfg.get('type') == 'hf' and model_cfg.get('enabled', True):
+                    path = MODELS_DIR / model_cfg['path']
+                    if not path.exists():
+                        missing = True
+                        break
+            
+            # Check for SentenceTransformer (needed for XGBoost)
+            st_path = MODELS_DIR / 'sentence_transformer'
+            if not st_path.exists():
+                missing = True
+
+        if missing:
+            print("Some models not found. Downloading required models...")
+            from .download import download_all
+            download_all()
 
     def _init_models(self):
         settings = self.config.get('settings', {})
@@ -189,7 +216,7 @@ class EnsembleGuard:
             if model_type == 'hf':
                 self.models.append(HFModel(model_cfg['name'], model_cfg['path'], settings))
             elif model_type == 'xgboost':
-                self.models.append(XGBoostModel(settings))
+                self.models.append(XGBoostModel(settings, model_cfg))
             else:
                 print(f"Unknown model type: {model_type}")
 

promptforest-0.5.0a0/promptforest/llama_guard_86m_downloader.py

@@ -0,0 +1,58 @@
+"""
+Script to download Llama Guard 2 86M from custom GitHub releases.
+Downloads files in parallel for speed.
+"""
+
+import os
+import requests
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
+from .config import MODELS_DIR
+
+BASE_URL = "https://github.com/appleroll-research/promptforest-model-ensemble/releases/download/v0.5.0-alpha"
+FILES_TO_DOWNLOAD = [
+    "config.json",
+    "model.safetensors",
+    "special_tokens_map.json",
+    "tokenizer.json",
+    "tokenizer_config.json"
+]
+
+def _download_file(url, save_path):
+    """Download a single file."""
+    if save_path.exists():
+        return
+        
+    try:
+        response = requests.get(url, stream=True)
+        response.raise_for_status()
+        
+        with open(save_path, 'wb') as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                f.write(chunk)
+    except Exception as e:
+        print(f"Failed to download {url}: {e}")
+        # Clean up partial file
+        if save_path.exists():
+            os.remove(save_path)
+
+def download_llama_guard():
+    """Download Llama Guard files in parallel."""
+    save_dir = MODELS_DIR / "llama_guard"
+    
+    # Check if all files exist
+    if save_dir.exists() and all((save_dir / f).exists() for f in FILES_TO_DOWNLOAD):
+        return
+
+    save_dir.mkdir(parents=True, exist_ok=True)
+    
+    with ThreadPoolExecutor(max_workers=5) as executor:
+        futures = []
+        for filename in FILES_TO_DOWNLOAD:
+            url = f"{BASE_URL}/{filename}"
+            save_path = save_dir / filename
+            futures.append(executor.submit(_download_file, url, save_path))
+            
+        for future in futures:
+            future.result()
+            

{promptforest-0.1.1 → promptforest-0.5.0a0/promptforest.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: promptforest
-Version: 0.1.1
+Version: 0.5.0a0
 Summary: Ensemble Prompt Injection Detection
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
@@ -36,6 +36,28 @@ This discrepancy score enables downstream workflows such as:
 - Adaptive throttling or alerting in production systems  
 - Continuous monitoring and model improvement  
 
+## Statistics
+PromptForest was evaluated against the SOTA model Qualifire Sentinel model (v2).
+
+| Metric                           | PromptForest | Sentinel v2 |
+| -------------------------------- | ------------ | ----------- |
+| Accuracy                         | 0.802        | 0.982       |
+| Avg Confidence on Wrong Answers  | 0.643        | 0.858       |
+| Expected Calibration Error (ECE) | 0.060        | 0.202       |
+| Approximate Model Size           | ~250M params  | 600M params |
+
+
+### Key Insights
+
+- Calibrated uncertainty: PromptForest is less confident on wrong predictions than Sentinel, resulting in a much lower ECE.
+
+- Parameter efficiency: Achieves competitive reliability with <50% of the parameters.
+
+- Interpretability: Confidence scores can be used to flag uncertain predictions for human review.
+
+Interpretation:
+While Sentinel has higher raw accuracy, PromptForest provides better-calibrated confidence. For systems where overconfidence on wrong answers is risky, PromptForest can reduce the chance of critical errors despite being smaller and faster.
+
 
 ## Supported Models
 
@@ -43,13 +65,12 @@ This discrepancy score enables downstream workflows such as:
 | ------------- | ----------------------------------------- |
 | **Meta**      | Llama Prompt Guard 86M (Built with Llama) |
 | **ProtectAI** | DebertaV3 Prompt Injection Finetune       |
-| **Deepset**   | DebertaV3-base Injection Finetune         |
-| **Katanemo**  | Arch-Guard                                |
+| **Vijil**     | Vijil Dome Prompt Injection Detection     |
 | **Appleroll** | PromptForest-XGBoost                      |
 
 ## Performance
-**Request Latency** \
-Best Case: 50ms \
+**E2E Request Latency** \
+Average Case: 100ms \
 Worst Case: 200ms
 
 **Accuracy** \

{promptforest-0.1.1 → promptforest-0.5.0a0}/setup.py

@@ -6,7 +6,7 @@ long_description = (this_directory / "README.md").read_text(encoding="utf-8")
 
 setup(
     name="promptforest",
-    version="0.1.1",
+    version="0.5.0-alpha",
     description="Ensemble Prompt Injection Detection",
     packages=find_packages(),
     install_requires=[

promptforest-0.1.1/promptforest/llama_guard_86m_downloader.py

@@ -1,67 +0,0 @@
-"""
-Script to download Llama Guard 2 86M from a custom GitHub repository.
-Handles split safetensor files and combines them locally.
-"""
-
-import os
-import shutil
-import subprocess
-import tempfile
-from pathlib import Path
-from .config import MODELS_DIR
-
-LLAMA_GUARD_REPO = "https://github.com/appleroll-research/promptforest-model-ensemble.git"
-
-def _download_llama_guard():
-    """Download Llama Guard from custom repository and combine split files."""
-    save_path = MODELS_DIR / "llama_guard"
-    
-    if save_path.exists():
-        return
-    
-    try:
-        # Use temporary directory for cloning
-        with tempfile.TemporaryDirectory() as temp_dir:
-            temp_path = Path(temp_dir)
-            
-            # Clone repository silently
-            subprocess.run(
-                ["git", "clone", "--depth", "1", LLAMA_GUARD_REPO, str(temp_path)],
-                stdout=subprocess.DEVNULL,
-                stderr=subprocess.DEVNULL,
-                check=True
-            )
-            
-            # Get the llama_guard folder from the cloned repo
-            source_dir = temp_path / "llama_guard"
-            if not source_dir.exists():
-                raise FileNotFoundError(f"llama_guard folder not found in repository")
-            
-            # Copy to models directory
-            save_path.parent.mkdir(parents=True, exist_ok=True)
-            shutil.copytree(source_dir, save_path)
-            
-            # Combine split safetensor files
-            model_file = save_path / "model.safetensors"
-            if not model_file.exists():
-                # Find and combine c_* files
-                split_files = sorted(save_path.glob("c_*"))
-                if split_files:
-                    with open(model_file, 'wb') as outfile:
-                        for split_file in split_files:
-                            with open(split_file, 'rb') as infile:
-                                outfile.write(infile.read())
-                    
-                    # Delete split files
-                    for split_file in split_files:
-                        split_file.unlink()
-        
-    except Exception as e:
-        # Clean up on failure
-        if save_path.exists():
-            shutil.rmtree(save_path)
-        raise Exception(f"Failed to download Llama Guard: {e}")
-
-def download_llama_guard():
-    """Public interface for downloading Llama Guard."""
-    _download_llama_guard()