aicert 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aicert/__init__.py +3 -0
- aicert/__main__.py +6 -0
- aicert/artifacts.py +104 -0
- aicert/cli.py +1423 -0
- aicert/config.py +193 -0
- aicert/doctor.py +366 -0
- aicert/hashing.py +28 -0
- aicert/metrics.py +305 -0
- aicert/providers/__init__.py +13 -0
- aicert/providers/anthropic.py +182 -0
- aicert/providers/base.py +36 -0
- aicert/providers/openai.py +153 -0
- aicert/providers/openai_compatible.py +152 -0
- aicert/runner.py +620 -0
- aicert/templating.py +83 -0
- aicert/validation.py +322 -0
- aicert-0.1.0.dist-info/METADATA +306 -0
- aicert-0.1.0.dist-info/RECORD +22 -0
- aicert-0.1.0.dist-info/WHEEL +5 -0
- aicert-0.1.0.dist-info/entry_points.txt +2 -0
- aicert-0.1.0.dist-info/licenses/LICENSE +21 -0
- aicert-0.1.0.dist-info/top_level.txt +1 -0
aicert/__init__.py
ADDED
aicert/__main__.py
ADDED
aicert/artifacts.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Artifact management for aicert runs."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import random
|
|
5
|
+
import string
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Dict, Optional
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def generate_run_id() -> str:
|
|
12
|
+
"""Generate a unique run ID with ISO-like timestamp and random suffix."""
|
|
13
|
+
# Get current UTC time in ISO format, replacing colons and periods for safety
|
|
14
|
+
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ")
|
|
15
|
+
# Add 6 character random suffix
|
|
16
|
+
random_suffix = ''.join(random.choices(string.ascii_lowercase + string.digits, k=6))
|
|
17
|
+
return f"{timestamp}-{random_suffix}"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def create_run_dir(out_dir: Optional[str] = None) -> Path:
|
|
21
|
+
"""Create a run directory.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
out_dir: Optional custom output directory. If None, uses default
|
|
25
|
+
`.aicert/runs/<run_id>/` under current working directory.
|
|
26
|
+
|
|
27
|
+
Returns:
|
|
28
|
+
Path to the created run directory.
|
|
29
|
+
"""
|
|
30
|
+
if out_dir is None:
|
|
31
|
+
run_id = generate_run_id()
|
|
32
|
+
run_dir = Path.cwd() / ".aicert" / "runs" / run_id
|
|
33
|
+
else:
|
|
34
|
+
run_dir = Path(out_dir)
|
|
35
|
+
|
|
36
|
+
run_dir.mkdir(parents=True, exist_ok=True)
|
|
37
|
+
return run_dir
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def write_config(run_dir: Path, config: Dict[str, Any]) -> Path:
|
|
41
|
+
"""Write the resolved config to config.resolved.yaml.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
run_dir: Path to the run directory.
|
|
45
|
+
config: Configuration dictionary to write.
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
Path to the written file.
|
|
49
|
+
"""
|
|
50
|
+
import yaml
|
|
51
|
+
|
|
52
|
+
config_path = run_dir / "config.resolved.yaml"
|
|
53
|
+
with open(config_path, "w", encoding="utf-8") as f:
|
|
54
|
+
yaml.safe_dump(config, f, allow_unicode=True, sort_keys=False)
|
|
55
|
+
return config_path
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def append_result(run_dir: Path, result_dict: Dict[str, Any]) -> Path:
|
|
59
|
+
"""Append a result to results.jsonl.
|
|
60
|
+
|
|
61
|
+
Args:
|
|
62
|
+
run_dir: Path to the run directory.
|
|
63
|
+
result_dict: Result dictionary to append.
|
|
64
|
+
|
|
65
|
+
Returns:
|
|
66
|
+
Path to the results file.
|
|
67
|
+
"""
|
|
68
|
+
results_path = run_dir / "results.jsonl"
|
|
69
|
+
with open(results_path, "a", encoding="utf-8") as f:
|
|
70
|
+
json_line = json.dumps(result_dict, ensure_ascii=False)
|
|
71
|
+
f.write(json_line + "\n")
|
|
72
|
+
return results_path
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def write_summary(run_dir: Path, summary_dict: Dict[str, Any]) -> Path:
|
|
76
|
+
"""Write summary to summary.json.
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
run_dir: Path to the run directory.
|
|
80
|
+
summary_dict: Summary dictionary to write.
|
|
81
|
+
|
|
82
|
+
Returns:
|
|
83
|
+
Path to the written file.
|
|
84
|
+
"""
|
|
85
|
+
summary_path = run_dir / "summary.json"
|
|
86
|
+
with open(summary_path, "w", encoding="utf-8") as f:
|
|
87
|
+
json.dump(summary_dict, f, indent=2, ensure_ascii=False)
|
|
88
|
+
return summary_path
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def write_report(run_dir: Path, report_text: str) -> Path:
|
|
92
|
+
"""Write report to report.txt.
|
|
93
|
+
|
|
94
|
+
Args:
|
|
95
|
+
run_dir: Path to the run directory.
|
|
96
|
+
report_text: Report text to write.
|
|
97
|
+
|
|
98
|
+
Returns:
|
|
99
|
+
Path to the written file.
|
|
100
|
+
"""
|
|
101
|
+
report_path = run_dir / "report.txt"
|
|
102
|
+
with open(report_path, "w", encoding="utf-8") as f:
|
|
103
|
+
f.write(report_text)
|
|
104
|
+
return report_path
|