adversarial-workflow 0.6.6__py3-none-any.whl → 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- adversarial_workflow/__init__.py +1 -1
- adversarial_workflow/cli.py +351 -5
- adversarial_workflow/evaluators/__init__.py +11 -2
- adversarial_workflow/evaluators/config.py +39 -2
- adversarial_workflow/evaluators/discovery.py +97 -9
- adversarial_workflow/evaluators/resolver.py +211 -0
- adversarial_workflow/evaluators/runner.py +36 -13
- adversarial_workflow/library/__init__.py +56 -0
- adversarial_workflow/library/cache.py +184 -0
- adversarial_workflow/library/client.py +224 -0
- adversarial_workflow/library/commands.py +849 -0
- adversarial_workflow/library/config.py +81 -0
- adversarial_workflow/library/models.py +129 -0
- adversarial_workflow/utils/citations.py +643 -0
- {adversarial_workflow-0.6.6.dist-info → adversarial_workflow-0.9.0.dist-info}/METADATA +160 -3
- {adversarial_workflow-0.6.6.dist-info → adversarial_workflow-0.9.0.dist-info}/RECORD +20 -12
- {adversarial_workflow-0.6.6.dist-info → adversarial_workflow-0.9.0.dist-info}/WHEEL +0 -0
- {adversarial_workflow-0.6.6.dist-info → adversarial_workflow-0.9.0.dist-info}/entry_points.txt +0 -0
- {adversarial_workflow-0.6.6.dist-info → adversarial_workflow-0.9.0.dist-info}/licenses/LICENSE +0 -0
- {adversarial_workflow-0.6.6.dist-info → adversarial_workflow-0.9.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Library configuration with env > file > defaults precedence."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
import yaml
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass
|
|
12
|
+
class LibraryConfig:
|
|
13
|
+
"""Configuration for the evaluator library.
|
|
14
|
+
|
|
15
|
+
Precedence: environment variables > config file > defaults.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
url: str = "https://raw.githubusercontent.com/movito/adversarial-evaluator-library/main"
|
|
19
|
+
ref: str = "main"
|
|
20
|
+
cache_ttl: int = 3600 # 1 hour
|
|
21
|
+
cache_dir: Path = field(default_factory=lambda: Path.home() / ".cache" / "adversarial-workflow")
|
|
22
|
+
enabled: bool = True
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def get_library_config(config_path: Optional[Path] = None) -> LibraryConfig:
|
|
26
|
+
"""
|
|
27
|
+
Load library configuration with precedence: env > file > defaults.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
config_path: Optional path to config file. Defaults to .adversarial/config.yml
|
|
31
|
+
|
|
32
|
+
Returns:
|
|
33
|
+
LibraryConfig with merged settings.
|
|
34
|
+
"""
|
|
35
|
+
config = LibraryConfig()
|
|
36
|
+
|
|
37
|
+
# Load from config file if exists
|
|
38
|
+
config_file = config_path or Path(".adversarial/config.yml")
|
|
39
|
+
if config_file.exists():
|
|
40
|
+
try:
|
|
41
|
+
with open(config_file, "r", encoding="utf-8") as f:
|
|
42
|
+
data = yaml.safe_load(f) or {}
|
|
43
|
+
# Handle non-dict YAML (list, scalar, etc.) gracefully
|
|
44
|
+
if not isinstance(data, dict):
|
|
45
|
+
data = {}
|
|
46
|
+
lib_config = data.get("library", {})
|
|
47
|
+
|
|
48
|
+
if "url" in lib_config:
|
|
49
|
+
config.url = lib_config["url"]
|
|
50
|
+
if "ref" in lib_config:
|
|
51
|
+
config.ref = lib_config["ref"]
|
|
52
|
+
if "cache_ttl" in lib_config:
|
|
53
|
+
config.cache_ttl = int(lib_config["cache_ttl"])
|
|
54
|
+
if "cache_dir" in lib_config:
|
|
55
|
+
# Expand ~ in path
|
|
56
|
+
config.cache_dir = Path(lib_config["cache_dir"]).expanduser()
|
|
57
|
+
if "enabled" in lib_config:
|
|
58
|
+
config.enabled = bool(lib_config["enabled"])
|
|
59
|
+
except (yaml.YAMLError, OSError, ValueError):
|
|
60
|
+
# Config file is invalid, use defaults
|
|
61
|
+
pass
|
|
62
|
+
|
|
63
|
+
# Apply environment variable overrides (highest precedence)
|
|
64
|
+
if url := os.environ.get("ADVERSARIAL_LIBRARY_URL"):
|
|
65
|
+
config.url = url
|
|
66
|
+
|
|
67
|
+
# Process TTL first, then NO_CACHE (so NO_CACHE always wins)
|
|
68
|
+
if ttl := os.environ.get("ADVERSARIAL_LIBRARY_CACHE_TTL"):
|
|
69
|
+
try:
|
|
70
|
+
config.cache_ttl = int(ttl)
|
|
71
|
+
except ValueError:
|
|
72
|
+
pass # Invalid TTL, keep current value
|
|
73
|
+
|
|
74
|
+
# NO_CACHE takes precedence over CACHE_TTL - check it last
|
|
75
|
+
if os.environ.get("ADVERSARIAL_LIBRARY_NO_CACHE"):
|
|
76
|
+
config.cache_ttl = 0
|
|
77
|
+
|
|
78
|
+
if ref := os.environ.get("ADVERSARIAL_LIBRARY_REF"):
|
|
79
|
+
config.ref = ref
|
|
80
|
+
|
|
81
|
+
return config
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""Data models for the evaluator library client."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from datetime import datetime, timezone
|
|
5
|
+
from typing import Dict, List, Optional
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class EvaluatorEntry:
|
|
10
|
+
"""An evaluator entry from the library index."""
|
|
11
|
+
|
|
12
|
+
name: str
|
|
13
|
+
provider: str
|
|
14
|
+
path: str
|
|
15
|
+
model: str
|
|
16
|
+
category: str
|
|
17
|
+
description: str
|
|
18
|
+
|
|
19
|
+
@classmethod
|
|
20
|
+
def from_dict(cls, data: Dict) -> "EvaluatorEntry":
|
|
21
|
+
"""Create an EvaluatorEntry from a dictionary."""
|
|
22
|
+
return cls(
|
|
23
|
+
name=data["name"],
|
|
24
|
+
provider=data["provider"],
|
|
25
|
+
path=data["path"],
|
|
26
|
+
model=data["model"],
|
|
27
|
+
category=data["category"],
|
|
28
|
+
description=data["description"],
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
@property
|
|
32
|
+
def full_name(self) -> str:
|
|
33
|
+
"""Return provider/name format."""
|
|
34
|
+
return f"{self.provider}/{self.name}"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class IndexData:
|
|
39
|
+
"""Parsed library index data."""
|
|
40
|
+
|
|
41
|
+
version: str
|
|
42
|
+
evaluators: List[EvaluatorEntry]
|
|
43
|
+
categories: Dict[str, str]
|
|
44
|
+
fetched_at: Optional[datetime] = None
|
|
45
|
+
|
|
46
|
+
@classmethod
|
|
47
|
+
def from_dict(cls, data: Dict) -> "IndexData":
|
|
48
|
+
"""Create an IndexData from a dictionary."""
|
|
49
|
+
evaluators = [EvaluatorEntry.from_dict(e) for e in data.get("evaluators", [])]
|
|
50
|
+
return cls(
|
|
51
|
+
version=data.get("version", "unknown"),
|
|
52
|
+
evaluators=evaluators,
|
|
53
|
+
categories=data.get("categories", {}),
|
|
54
|
+
fetched_at=datetime.now(timezone.utc),
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
def get_evaluator(self, provider: str, name: str) -> Optional[EvaluatorEntry]:
|
|
58
|
+
"""Find an evaluator by provider and name."""
|
|
59
|
+
for e in self.evaluators:
|
|
60
|
+
if e.provider == provider and e.name == name:
|
|
61
|
+
return e
|
|
62
|
+
return None
|
|
63
|
+
|
|
64
|
+
def filter_by_provider(self, provider: str) -> List[EvaluatorEntry]:
|
|
65
|
+
"""Filter evaluators by provider."""
|
|
66
|
+
return [e for e in self.evaluators if e.provider == provider]
|
|
67
|
+
|
|
68
|
+
def filter_by_category(self, category: str) -> List[EvaluatorEntry]:
|
|
69
|
+
"""Filter evaluators by category."""
|
|
70
|
+
return [e for e in self.evaluators if e.category == category]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass
|
|
74
|
+
class InstalledEvaluatorMeta:
|
|
75
|
+
"""Metadata for an installed evaluator (from _meta block)."""
|
|
76
|
+
|
|
77
|
+
source: str
|
|
78
|
+
source_path: str
|
|
79
|
+
version: str
|
|
80
|
+
installed: str
|
|
81
|
+
file_path: Optional[str] = None # Path to the installed file
|
|
82
|
+
|
|
83
|
+
@classmethod
|
|
84
|
+
def from_dict(cls, data: Dict) -> Optional["InstalledEvaluatorMeta"]:
|
|
85
|
+
"""Create from _meta dictionary, returns None if invalid."""
|
|
86
|
+
if not data:
|
|
87
|
+
return None
|
|
88
|
+
try:
|
|
89
|
+
return cls(
|
|
90
|
+
source=data.get("source", ""),
|
|
91
|
+
source_path=data.get("source_path", ""),
|
|
92
|
+
version=data.get("version", ""),
|
|
93
|
+
installed=data.get("installed", ""),
|
|
94
|
+
)
|
|
95
|
+
except (KeyError, TypeError):
|
|
96
|
+
return None
|
|
97
|
+
|
|
98
|
+
@property
|
|
99
|
+
def provider(self) -> str:
|
|
100
|
+
"""Extract provider from source_path."""
|
|
101
|
+
parts = self.source_path.split("/")
|
|
102
|
+
return parts[0] if parts else ""
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def name(self) -> str:
|
|
106
|
+
"""Extract name from source_path."""
|
|
107
|
+
parts = self.source_path.split("/")
|
|
108
|
+
return parts[1] if len(parts) > 1 else ""
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@dataclass
|
|
112
|
+
class UpdateInfo:
|
|
113
|
+
"""Information about an available update."""
|
|
114
|
+
|
|
115
|
+
name: str
|
|
116
|
+
installed_version: str
|
|
117
|
+
available_version: str
|
|
118
|
+
is_outdated: bool
|
|
119
|
+
is_local_only: bool = False
|
|
120
|
+
|
|
121
|
+
@property
|
|
122
|
+
def status(self) -> str:
|
|
123
|
+
"""Human-readable status."""
|
|
124
|
+
if self.is_local_only:
|
|
125
|
+
return "Local only"
|
|
126
|
+
elif self.is_outdated:
|
|
127
|
+
return "Update available"
|
|
128
|
+
else:
|
|
129
|
+
return "Up to date"
|