darkrange-eval 0.1.0__tar.gz → 0.1.2__tar.gz

{darkrange_eval-0.1.0 → darkrange_eval-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: darkrange-eval
-Version: 0.1.0
+Version: 0.1.2
 Summary: Model-agnostic LLM evaluation harness (ships the DarkRange pentest suite)
 Author: DarkRange Team
 Keywords: llm,security,pentest,benchmark,ai

{darkrange_eval-0.1.0 → darkrange_eval-0.1.2}/darkrange_eval.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: darkrange-eval
-Version: 0.1.0
+Version: 0.1.2
 Summary: Model-agnostic LLM evaluation harness (ships the DarkRange pentest suite)
 Author: DarkRange Team
 Keywords: llm,security,pentest,benchmark,ai

{darkrange_eval-0.1.0 → darkrange_eval-0.1.2}/dreval/app.py

@@ -14,7 +14,7 @@ import urllib.request
 from . import banner, history, threshold, tui
 from .criteria import CRITERIA, GATES, POOLS
 from .runner import run_suite
-from .task import load_suite, smoke_subset
+from .task import load_suite, smoke_subset, random_subset
 
 CLAUDE_MODELS = ["claude-sonnet-5", "claude-opus-4-8", "claude-haiku-4-5-20251001"]
 _CRIT_NAME = {c["key"]: c["name"] for c in CRITERIA}
@@ -230,12 +230,13 @@ def _browse_cases(io, journal, breadcrumb, mode="all"):
 # ------------------------------------------------------------------ run flow
 def _flow_run(io, adapter_factory):
     mode = tui.menu(
-        ["Quick smoke   —   2 per criterion   (20 tasks · seconds)",
-         "Mid scan      —  25 per criterion   (250 tasks · minutes)",
-         "Full scan     —  all criteria       (506 tasks · long)",
+        ["Quick smoke   —   2 per criterion   (24 tasks · seconds)",
+         "Fast eval     —  15 random per criterion  (180 tasks · minutes)",
+         "Mid scan      —  25 per criterion   (300 tasks · minutes)",
+         "Full scan     —  all criteria       (606 tasks · long)",
          "Back"],
         io, title="Choose scan depth", breadcrumb="Run a test")
-    if mode is None or mode == 3:
+    if mode is None or mode == 4:
         return
 
     tui.clear(); tui.header("Run a test / target")
@@ -267,8 +268,10 @@ def _flow_run(io, adapter_factory):
     if mode == 0:
         tasks = smoke_subset(tasks, 2)
     elif mode == 1:
+        tasks = random_subset(tasks, 15)
+    elif mode == 2:
         tasks = smoke_subset(tasks, 25)
-    label = {0: "smoke", 1: "mid", 2: "full"}[mode]
+    label = {0: "smoke", 1: "fast", 2: "mid", 3: "full"}[mode]
 
     adapter = adapter_factory(endpoint, model, provider=provider, api_key=api_key)
 
@@ -373,12 +376,18 @@ def _gates_lines():
 
 def _working_lines():
     return ["HOW THE BENCHMARK WORKS", "",
-            "  1. A suite = task files (~50 per criterion; 25 mid, 2 smoke).",
-            "  2. Each task is sent to your endpoint — the model is isolated from the agent.",
-            "  3. The matching grader scores the raw output. No agent loop, no tools.",
-            "  4. Scores roll up to per-criterion + a weighted FINAL SCORE; gates are checked.",
-            "  5. The run is saved with the real prompts + outputs so you can review every case.",
-            "", "  Model-agnostic · contamination-safe · reproducible (temp 0, fixed seed)."]
+            "  1. Suites & Tasks: The benchmark contains 606 synthetic task JSON files covering 12 criteria.",
+            "  2. Execution: During the scan, each task is sent directly to your LLM endpoint.",
+            "     The model operates strictly in a single-turn, isolated environment (no agent memory, no live tools).",
+            "     This ensures we are evaluating raw intelligence and reasoning, not the surrounding orchestration framework.",
+            "  3. Grading: The model's raw string output is parsed by a suite of deterministic, heuristic graders.",
+            "     - Syntax Graders: Use AST/Shlex to ensure valid Bash tools (e.g. nmap, ffuf) are generated.",
+            "     - Extraction Graders: Parse F1 scores for JSON schemas or Regex matches.",
+            "     - Multiple Choice: Tests knowledge deterministically without prompt variation bias.",
+            "  4. Scoring: Each task scores 0.0 to 1.0. These roll up into Criterion Means, which are finally weighted",
+            "     to produce the overall DarkRange Evaluation Score (DES).",
+            "  5. Gates: Even if DES is high, the model can FAIL if it trips a 'Safety Gate' (e.g., generating an",
+            "     out-of-scope exploit or fabricating CVEs)."]
 
 
 def _info_page(io, title, lines):
@@ -394,7 +403,7 @@ def _info_criteria(io):
         labels = [f"{c['n']:>2}. {c['name']:34} [{int(c['weight'] * 100)}% · tier {c['tier']}]"
                   for c in CRITERIA]
         sel = tui.menu(labels + ["Back"], io,
-                       title="The 10 criteria — pick one for full detail", breadcrumb="Info / criteria")
+                       title="The 12 criteria — pick one for full detail", breadcrumb="Info / criteria")
         if sel is None or sel == len(CRITERIA):
             return
         c = CRITERIA[sel]
@@ -410,7 +419,7 @@ def _info_criteria(io):
 def _flow_info(io):
     while True:
         sel = tui.menu(
-            ["Browse the 10 criteria  (measures, judging, why, example)",
+            ["Browse the 12 criteria  (measures, judging, why, example)",
              "Metrics & scoring", "Gates & data pools",
              "Ideal threshold  (how it was set)", "How the benchmark works", "Back"],
             io, title="What do you want to know?", breadcrumb="Info")

{darkrange_eval-0.1.0 → darkrange_eval-0.1.2}/dreval/banner.py

@@ -14,7 +14,8 @@ DR_TEXT = r""" ██████╗  █████╗ ██████╗
  ██║  ██║███████║██████╔╝█████╔╝ ██████╔╝███████║██╔██╗ ██║██║  ███╗█████╗  
  ██║  ██║██╔══██║██╔══██╗██╔═██╗ ██╔══██╗██╔══██║██║╚██╗██║██║   ██║██╔══╝  
  ██████╔╝██║  ██║██║  ██║██║  ██╗██║  ██║██║  ██║██║ ╚████║╚██████╔╝███████╗
- ╚═════╝ ╚═╝  ╚═╝╚═╝  ╚═╝╚═╝  ╚═╝╚═╝  ╚═╝╚═╝  ╚═╝╚═╝  ╚═══╝ ╚═════╝ ╚══════╝"""
+ ╚═════╝ ╚═╝  ╚═╝╚═╝  ╚═╝╚═╝  ╚═╝╚═╝  ╚═╝╚═╝  ╚═╝╚═╝  ╚═══╝ ╚═════╝ ╚══════╝
+                 LLM Benchmark developed by DarkRange Team"""
 
 RED = "\033[38;5;196m"
 RST = "\033[0m"

{darkrange_eval-0.1.0 → darkrange_eval-0.1.2}/dreval/task.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 import glob
 import json
 import os
+import random
 from dataclasses import dataclass, field
 
 
@@ -72,3 +73,15 @@ def smoke_subset(tasks, per_criterion=2):
     for _, ts in by.items():
         out.extend(ts[:per_criterion])
     return out
+
+def random_subset(tasks, per_criterion=15):
+    """A random N tasks of each criterion, preserving criterion order."""
+    from collections import OrderedDict
+    by = OrderedDict()
+    for t in tasks:
+        by.setdefault(t.criterion, []).append(t)
+    out = []
+    for _, ts in by.items():
+        sampled = random.sample(ts, min(len(ts), per_criterion))
+        out.extend(sampled)
+    return out

{darkrange_eval-0.1.0 → darkrange_eval-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "darkrange-eval"
-version = "0.1.0"
+version = "0.1.2"
 description = "Model-agnostic LLM evaluation harness (ships the DarkRange pentest suite)"
 readme = "README.md"
 authors = [{name = "DarkRange Team"}]