release-gate 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- release_gate-0.3.0/LICENSE +21 -0
- release_gate-0.3.0/PKG-INFO +18 -0
- release_gate-0.3.0/README.md +535 -0
- release_gate-0.3.0/release_gate.egg-info/PKG-INFO +18 -0
- release_gate-0.3.0/release_gate.egg-info/SOURCES.txt +10 -0
- release_gate-0.3.0/release_gate.egg-info/dependency_links.txt +1 -0
- release_gate-0.3.0/release_gate.egg-info/entry_points.txt +2 -0
- release_gate-0.3.0/release_gate.egg-info/requires.txt +2 -0
- release_gate-0.3.0/release_gate.egg-info/top_level.txt +1 -0
- release_gate-0.3.0/setup.cfg +4 -0
- release_gate-0.3.0/setup.py +21 -0
- release_gate-0.3.0/tests/test_release_gate.py +444 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 VamsiSudhakaran1
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: release-gate
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Governance enforcement for AI agents
|
|
5
|
+
Home-page: https://github.com/VamsiSudhakaran1/release-gate
|
|
6
|
+
Author: Vamsi Sudhakaran
|
|
7
|
+
Author-email: vamsi.sudhakaran@gmail.com
|
|
8
|
+
Requires-Python: >=3.8
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: pyyaml>=6.0
|
|
11
|
+
Requires-Dist: jsonschema>=4.0
|
|
12
|
+
Dynamic: author
|
|
13
|
+
Dynamic: author-email
|
|
14
|
+
Dynamic: home-page
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
Dynamic: requires-dist
|
|
17
|
+
Dynamic: requires-python
|
|
18
|
+
Dynamic: summary
|
|
@@ -0,0 +1,535 @@
|
|
|
1
|
+
# release-gate
|
|
2
|
+
|
|
3
|
+
🚪 **Governance enforcement for AI agents** — Prevent cost explosions, ensure safety measures, and enforce access control before deployment.
|
|
4
|
+
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
[](https://github.com/VamsiSudhakaran1/release-gate/actions)
|
|
7
|
+
[](https://pypi.org/project/release-gate/)
|
|
8
|
+
[](https://www.python.org/downloads/)
|
|
9
|
+
|
|
10
|
+
## The Problem It Solves
|
|
11
|
+
|
|
12
|
+
**Your AI agent costs you $50,000 in a single day.** No warning. No limit. No questions.
|
|
13
|
+
|
|
14
|
+
This happens because:
|
|
15
|
+
- ❌ No cost limits are set
|
|
16
|
+
- ❌ No one validates agent configuration
|
|
17
|
+
- ❌ Request volumes spiral unexpectedly
|
|
18
|
+
- ❌ Token usage balloons with complex prompts
|
|
19
|
+
- ❌ One retry loop = 10x cost multiplier
|
|
20
|
+
|
|
21
|
+
**release-gate stops this before it happens.**
|
|
22
|
+
|
|
23
|
+
---
|
|
24
|
+
|
|
25
|
+
## What It Does (The 4 Checks)
|
|
26
|
+
|
|
27
|
+
release-gate sits between testing and deployment, validating agents against 4 critical checks:
|
|
28
|
+
|
|
29
|
+
### 🎯 PRIMARY CHECK: ACTION_BUDGET (NEW v0.3)
|
|
30
|
+
|
|
31
|
+
**Prevents cost explosions** — The hero feature that stops $50K mistakes.
|
|
32
|
+
|
|
33
|
+
```yaml
|
|
34
|
+
checks:
|
|
35
|
+
action_budget:
|
|
36
|
+
enabled: true
|
|
37
|
+
max_daily_cost: 100 # Sets the limit
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
What happens:
|
|
41
|
+
|
|
42
|
+
```
|
|
43
|
+
Agent estimated to cost $250/day
|
|
44
|
+
Budget set to $100/day
|
|
45
|
+
Status: ❌ FAIL - Deployment blocked
|
|
46
|
+
|
|
47
|
+
Remediation:
|
|
48
|
+
Option 1: Use cheaper model (gpt-4-turbo saves 70%)
|
|
49
|
+
Option 2: Reduce daily request volume
|
|
50
|
+
Option 3: Increase budget to $250/day
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
**Cost Calculation (Full Transparency):**
|
|
54
|
+
```
|
|
55
|
+
Model: GPT-4-Turbo
|
|
56
|
+
Daily requests: 500
|
|
57
|
+
Input tokens/request: 800
|
|
58
|
+
Output tokens/request: 400
|
|
59
|
+
Estimated daily cost: $12.50
|
|
60
|
+
Monthly cost: $375.00
|
|
61
|
+
Safety margin: 8x (well under $100 budget)
|
|
62
|
+
Status: ✅ PASS
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
### Supporting Checks (Existing v0.2)
|
|
66
|
+
|
|
67
|
+
#### 📋 INPUT_CONTRACT
|
|
68
|
+
Ensures request schemas are defined and tested.
|
|
69
|
+
- ✅ Schema explicitly defined
|
|
70
|
+
- ✅ Valid inputs pass validation
|
|
71
|
+
- ✅ Invalid inputs fail validation
|
|
72
|
+
- Prevents input-based failures
|
|
73
|
+
|
|
74
|
+
#### ⏹ FALLBACK_DECLARED
|
|
75
|
+
Ensures agent can be stopped if something goes wrong.
|
|
76
|
+
- ✅ Kill switch defined (feature flag)
|
|
77
|
+
- ✅ Fallback mode exists (escalate to human)
|
|
78
|
+
- ✅ Team ownership clear
|
|
79
|
+
- ✅ Runbook provided
|
|
80
|
+
|
|
81
|
+
#### 🔐 IDENTITY_BOUNDARY
|
|
82
|
+
Enforces access control and rate limiting.
|
|
83
|
+
- ✅ Authentication required
|
|
84
|
+
- ✅ Rate limits configured
|
|
85
|
+
- ✅ Data isolation defined
|
|
86
|
+
- Prevents unauthorized access
|
|
87
|
+
|
|
88
|
+
---
|
|
89
|
+
|
|
90
|
+
## Quick Start (5 Minutes)
|
|
91
|
+
|
|
92
|
+
### 1. Install
|
|
93
|
+
```bash
|
|
94
|
+
pip install release-gate
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
### 2. Create governance.yaml
|
|
98
|
+
```yaml
|
|
99
|
+
project:
|
|
100
|
+
name: my-agent
|
|
101
|
+
|
|
102
|
+
agent:
|
|
103
|
+
model: gpt-4-turbo
|
|
104
|
+
daily_requests: 500
|
|
105
|
+
avg_input_tokens: 800
|
|
106
|
+
avg_output_tokens: 400
|
|
107
|
+
retry_rate: 1.1
|
|
108
|
+
|
|
109
|
+
checks:
|
|
110
|
+
action_budget:
|
|
111
|
+
enabled: true
|
|
112
|
+
max_daily_cost: 100
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
### 3. Run Validation
|
|
116
|
+
```bash
|
|
117
|
+
release-gate check --config governance.yaml
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
### 4. See Decision
|
|
121
|
+
```
|
|
122
|
+
┌──────────────────────────────────────────┐
|
|
123
|
+
│ 🚪 release-gate: All 4 Checks │
|
|
124
|
+
├──────────────────────────────────────────┤
|
|
125
|
+
|
|
126
|
+
💰 ACTION_BUDGET: ✓ PASS
|
|
127
|
+
Daily Cost: $12.50
|
|
128
|
+
Budget: $100.00
|
|
129
|
+
Safety Margin: 8.0x
|
|
130
|
+
|
|
131
|
+
📋 INPUT_CONTRACT: ✓ PASS
|
|
132
|
+
Schema defined
|
|
133
|
+
|
|
134
|
+
⏹ FALLBACK_DECLARED: ✓ PASS
|
|
135
|
+
Kill switch configured
|
|
136
|
+
|
|
137
|
+
🔐 IDENTITY_BOUNDARY: ✓ PASS
|
|
138
|
+
Authentication required
|
|
139
|
+
|
|
140
|
+
✅ FINAL DECISION: PASS (Safe to deploy)
|
|
141
|
+
└──────────────────────────────────────────┘
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
---
|
|
145
|
+
|
|
146
|
+
## Real-World Example: The $50K Mistake
|
|
147
|
+
|
|
148
|
+
### Scenario
|
|
149
|
+
You deploy a customer support agent without checking costs:
|
|
150
|
+
|
|
151
|
+
```yaml
|
|
152
|
+
agent:
|
|
153
|
+
model: gpt-4 # Expensive model
|
|
154
|
+
daily_requests: 5000 # High volume
|
|
155
|
+
avg_input_tokens: 2000 # Long context
|
|
156
|
+
avg_output_tokens: 1000
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
**Without release-gate:**
|
|
160
|
+
```
|
|
161
|
+
Day 1: Agent costs $250
|
|
162
|
+
Day 2: No one notices
|
|
163
|
+
Day 3: Cost spike warning
|
|
164
|
+
...
|
|
165
|
+
Week 2: You've spent $50,000
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
**With release-gate:**
|
|
169
|
+
```
|
|
170
|
+
$ release-gate check --config governance.yaml
|
|
171
|
+
|
|
172
|
+
❌ FAIL - Cost Control: Budget Exceeded
|
|
173
|
+
|
|
174
|
+
Daily cost: $250.00
|
|
175
|
+
Budget: $100.00
|
|
176
|
+
Daily overage: $150.00
|
|
177
|
+
Monthly overage: $4,500.00
|
|
178
|
+
|
|
179
|
+
❌ BLOCKED: Cannot deploy without fixing cost configuration
|
|
180
|
+
|
|
181
|
+
Remediation Options:
|
|
182
|
+
Option 1: Use gpt-4-turbo (3.3x cheaper)
|
|
183
|
+
New cost: $75.88/day → PASS ✓
|
|
184
|
+
|
|
185
|
+
Option 2: Reduce daily requests to 1000
|
|
186
|
+
New cost: $50/day → PASS ✓
|
|
187
|
+
|
|
188
|
+
Option 3: Increase budget to $250/day
|
|
189
|
+
Then re-run validation
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
**Result:** Deployment blocked. Problem caught before it costs you $50K.
|
|
193
|
+
|
|
194
|
+
---
|
|
195
|
+
|
|
196
|
+
## Key Features
|
|
197
|
+
|
|
198
|
+
### 💰 ACTION_BUDGET: Cost Control (v0.3 New)
|
|
199
|
+
|
|
200
|
+
#### Automatic Cost Estimation
|
|
201
|
+
```python
|
|
202
|
+
# Reads agent config
|
|
203
|
+
agent:
|
|
204
|
+
model: gpt-4-turbo
|
|
205
|
+
daily_requests: 500
|
|
206
|
+
avg_input_tokens: 800
|
|
207
|
+
avg_output_tokens: 400
|
|
208
|
+
retry_rate: 1.1
|
|
209
|
+
|
|
210
|
+
# Automatically calculates:
|
|
211
|
+
# Input cost: $0.00001 × 800 ÷ 1000 × 500 × 1.1 = $4.40/day
|
|
212
|
+
# Output cost: $0.00003 × 400 ÷ 1000 × 500 × 1.1 = $6.60/day
|
|
213
|
+
# Total: $11.00/day
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
#### Smart Thresholds
|
|
217
|
+
```yaml
|
|
218
|
+
checks:
|
|
219
|
+
action_budget:
|
|
220
|
+
max_daily_cost: 100
|
|
221
|
+
auto_approve_threshold: 10 # < $10 = instant PASS
|
|
222
|
+
manual_approval_threshold: 50 # $10-$50 = needs review
|
|
223
|
+
# $50+ = approval routing
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
#### Approval Routing (Future)
|
|
227
|
+
```yaml
|
|
228
|
+
approval_routes:
|
|
229
|
+
- type: slack
|
|
230
|
+
channel: "#ai-governance"
|
|
231
|
+
mentions: ["@platform-leads"]
|
|
232
|
+
- type: email
|
|
233
|
+
to: ["ai-team@company.com"]
|
|
234
|
+
cc: ["security@company.com"]
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
### 🔄 Dynamic Pricing
|
|
238
|
+
|
|
239
|
+
#### No Hardcoding
|
|
240
|
+
```python
|
|
241
|
+
# Instead of hardcoded enums:
|
|
242
|
+
# - Supports ANY model (past, present, future)
|
|
243
|
+
# - Auto-detects from code
|
|
244
|
+
# - User-extensible via JSON
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
#### Auto-Detection
|
|
248
|
+
```python
|
|
249
|
+
# Code:
|
|
250
|
+
client = OpenAI(model="gpt-4o")
|
|
251
|
+
response = client.chat.completions.create(model="gpt-4o")
|
|
252
|
+
|
|
253
|
+
# release-gate automatically detects: gpt-4o
|
|
254
|
+
# Looks up pricing
|
|
255
|
+
# Estimates cost
|
|
256
|
+
# All automatic
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
#### Custom Models
|
|
260
|
+
```json
|
|
261
|
+
{
|
|
262
|
+
"models": {
|
|
263
|
+
"my-internal-llama": {
|
|
264
|
+
"input": 0.0001,
|
|
265
|
+
"output": 0.0002,
|
|
266
|
+
"provider": "Internal"
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
Add a custom model. No code changes. Instant support.
|
|
273
|
+
|
|
274
|
+
---
|
|
275
|
+
|
|
276
|
+
## Installation
|
|
277
|
+
|
|
278
|
+
### Via pip
|
|
279
|
+
```bash
|
|
280
|
+
pip install release-gate
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
### From source
|
|
284
|
+
```bash
|
|
285
|
+
git clone https://github.com/VamsiSudhakaran1/release-gate.git
|
|
286
|
+
cd release-gate
|
|
287
|
+
pip install -e .
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
### Requirements
|
|
291
|
+
- Python 3.8+
|
|
292
|
+
- PyYAML >= 6.0
|
|
293
|
+
- jsonschema >= 4.0
|
|
294
|
+
|
|
295
|
+
---
|
|
296
|
+
|
|
297
|
+
## Configuration
|
|
298
|
+
|
|
299
|
+
### Minimal (Cost Control Only)
|
|
300
|
+
```yaml
|
|
301
|
+
project:
|
|
302
|
+
name: my-agent
|
|
303
|
+
|
|
304
|
+
agent:
|
|
305
|
+
model: gpt-4-turbo
|
|
306
|
+
daily_requests: 100
|
|
307
|
+
avg_input_tokens: 500
|
|
308
|
+
avg_output_tokens: 300
|
|
309
|
+
|
|
310
|
+
checks:
|
|
311
|
+
action_budget:
|
|
312
|
+
enabled: true
|
|
313
|
+
max_daily_cost: 50
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
### Complete (All 4 Checks)
|
|
317
|
+
```yaml
|
|
318
|
+
project:
|
|
319
|
+
name: customer-support-agent
|
|
320
|
+
version: 1.0.0
|
|
321
|
+
|
|
322
|
+
agent:
|
|
323
|
+
model: gpt-4-turbo
|
|
324
|
+
daily_requests: 500
|
|
325
|
+
avg_input_tokens: 800
|
|
326
|
+
avg_output_tokens: 400
|
|
327
|
+
retry_rate: 1.1
|
|
328
|
+
|
|
329
|
+
checks:
|
|
330
|
+
action_budget:
|
|
331
|
+
enabled: true
|
|
332
|
+
max_daily_cost: 100
|
|
333
|
+
auto_approve_threshold: 10
|
|
334
|
+
manual_approval_threshold: 50
|
|
335
|
+
|
|
336
|
+
input_contract:
|
|
337
|
+
enabled: true
|
|
338
|
+
schema:
|
|
339
|
+
type: object
|
|
340
|
+
required: [user_query]
|
|
341
|
+
properties:
|
|
342
|
+
user_query:
|
|
343
|
+
type: string
|
|
344
|
+
|
|
345
|
+
fallback_declared:
|
|
346
|
+
enabled: true
|
|
347
|
+
kill_switch:
|
|
348
|
+
type: feature_flag
|
|
349
|
+
name: disable_agent
|
|
350
|
+
fallback:
|
|
351
|
+
mode: escalate_to_human
|
|
352
|
+
ownership:
|
|
353
|
+
team: support-team
|
|
354
|
+
oncall: "oncall@company.com"
|
|
355
|
+
|
|
356
|
+
identity_boundary:
|
|
357
|
+
enabled: true
|
|
358
|
+
authentication: required
|
|
359
|
+
rate_limit: 10
|
|
360
|
+
data_isolation:
|
|
361
|
+
- customer_data_only
|
|
362
|
+
```
|
|
363
|
+
|
|
364
|
+
---
|
|
365
|
+
|
|
366
|
+
## Usage
|
|
367
|
+
|
|
368
|
+
### Command Line
|
|
369
|
+
```bash
|
|
370
|
+
# Simple validation
|
|
371
|
+
release-gate check --config governance.yaml
|
|
372
|
+
|
|
373
|
+
# JSON output (for CI/CD)
|
|
374
|
+
release-gate check --config governance.yaml --output json
|
|
375
|
+
|
|
376
|
+
# YAML output (save to repo)
|
|
377
|
+
release-gate check --config governance.yaml --output yaml > audit.yaml
|
|
378
|
+
```
|
|
379
|
+
|
|
380
|
+
### GitHub Actions
|
|
381
|
+
```yaml
|
|
382
|
+
name: Governance Gate
|
|
383
|
+
on: [pull_request, push]
|
|
384
|
+
|
|
385
|
+
jobs:
|
|
386
|
+
governance:
|
|
387
|
+
runs-on: ubuntu-latest
|
|
388
|
+
steps:
|
|
389
|
+
- uses: actions/checkout@v4
|
|
390
|
+
- uses: actions/setup-python@v4
|
|
391
|
+
with:
|
|
392
|
+
python-version: '3.11'
|
|
393
|
+
- run: pip install release-gate
|
|
394
|
+
- run: release-gate check --config governance.yaml
|
|
395
|
+
```
|
|
396
|
+
|
|
397
|
+
### Python API
|
|
398
|
+
```python
|
|
399
|
+
from release_gate.checks.action_budget import ActionBudgetCheck
|
|
400
|
+
import yaml
|
|
401
|
+
|
|
402
|
+
with open('governance.yaml') as f:
|
|
403
|
+
config = yaml.safe_load(f)
|
|
404
|
+
|
|
405
|
+
check = ActionBudgetCheck()
|
|
406
|
+
result = check.evaluate(config)
|
|
407
|
+
|
|
408
|
+
if result['status'] == 'PASS':
|
|
409
|
+
print("✅ Safe to deploy")
|
|
410
|
+
else:
|
|
411
|
+
print("❌ Fix cost configuration")
|
|
412
|
+
for step in result.get('remediation_steps', []):
|
|
413
|
+
print(f" - {step}")
|
|
414
|
+
```
|
|
415
|
+
|
|
416
|
+
---
|
|
417
|
+
|
|
418
|
+
## Exit Codes
|
|
419
|
+
|
|
420
|
+
- **0** = PASS (all checks passed, safe to deploy)
|
|
421
|
+
- **10** = WARN (manual review recommended)
|
|
422
|
+
- **1** = FAIL (deployment blocked, fix issues)
|
|
423
|
+
|
|
424
|
+
Perfect for CI/CD pipelines.
|
|
425
|
+
|
|
426
|
+
---
|
|
427
|
+
|
|
428
|
+
## Roadmap
|
|
429
|
+
|
|
430
|
+
### v0.3 (Current) ✅
|
|
431
|
+
- [x] ACTION_BUDGET check (cost control)
|
|
432
|
+
- [x] Dynamic pricing system
|
|
433
|
+
- [x] Auto-model detection
|
|
434
|
+
- [x] Custom model support
|
|
435
|
+
- [x] All 4 checks working together
|
|
436
|
+
|
|
437
|
+
### v0.4 (Planned)
|
|
438
|
+
- [ ] GitHub Actions marketplace integration
|
|
439
|
+
- [ ] Web dashboard
|
|
440
|
+
- [ ] Advanced approval workflows
|
|
441
|
+
- [ ] Enterprise SSO/RBAC
|
|
442
|
+
|
|
443
|
+
### v1.0 (Vision)
|
|
444
|
+
- [ ] Real-time pricing API integration
|
|
445
|
+
- [ ] Advanced policy templates
|
|
446
|
+
- [ ] Multi-agent governance
|
|
447
|
+
- [ ] Analytics & reporting
|
|
448
|
+
|
|
449
|
+
---
|
|
450
|
+
|
|
451
|
+
## Supported Models
|
|
452
|
+
|
|
453
|
+
### OpenAI
|
|
454
|
+
- GPT-4
|
|
455
|
+
- GPT-4 Turbo
|
|
456
|
+
- GPT-4o
|
|
457
|
+
|
|
458
|
+
### Anthropic
|
|
459
|
+
- Claude 3 Opus
|
|
460
|
+
- Claude 3 Sonnet
|
|
461
|
+
- Claude 3.5 Sonnet
|
|
462
|
+
|
|
463
|
+
### Open Source
|
|
464
|
+
- Llama 70B
|
|
465
|
+
- Mistral Large
|
|
466
|
+
|
|
467
|
+
### Custom
|
|
468
|
+
- Any model (add to pricing.json)
|
|
469
|
+
|
|
470
|
+
---
|
|
471
|
+
|
|
472
|
+
## Examples
|
|
473
|
+
|
|
474
|
+
See `examples/` directory:
|
|
475
|
+
- `governance-simple.yaml` - Minimal setup
|
|
476
|
+
- `governance-complete.yaml` - Full setup
|
|
477
|
+
- `pricing.json` - All supported models
|
|
478
|
+
- `test_action_budget.py` - Test suite
|
|
479
|
+
|
|
480
|
+
---
|
|
481
|
+
|
|
482
|
+
## Architecture
|
|
483
|
+
|
|
484
|
+
### 4 Independent Checks
|
|
485
|
+
Each check validates independently:
|
|
486
|
+
- **ACTION_BUDGET** validates cost
|
|
487
|
+
- **INPUT_CONTRACT** validates schema
|
|
488
|
+
- **FALLBACK_DECLARED** validates safety
|
|
489
|
+
- **IDENTITY_BOUNDARY** validates access
|
|
490
|
+
|
|
491
|
+
No cross-dependencies. Easy to extend.
|
|
492
|
+
|
|
493
|
+
### Decision Logic
|
|
494
|
+
```
|
|
495
|
+
If ANY check FAILS → FAIL (deployment blocked)
|
|
496
|
+
Else if ANY check WARNS → WARN (manual review)
|
|
497
|
+
Else → PASS (all good)
|
|
498
|
+
```
|
|
499
|
+
|
|
500
|
+
All 4 checks are equal partners in the decision.
|
|
501
|
+
|
|
502
|
+
---
|
|
503
|
+
|
|
504
|
+
## Contributing
|
|
505
|
+
|
|
506
|
+
Contributions welcome! See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
|
|
507
|
+
|
|
508
|
+
---
|
|
509
|
+
|
|
510
|
+
## Support
|
|
511
|
+
|
|
512
|
+
- 📖 [Documentation](docs/)
|
|
513
|
+
- 🐛 [Issues](https://github.com/VamsiSudhakaran1/release-gate/issues)
|
|
514
|
+
- 💬 [Discussions](https://github.com/VamsiSudhakaran1/release-gate/discussions)
|
|
515
|
+
- 🌐 [Website](https://release-gate.com)
|
|
516
|
+
|
|
517
|
+
---
|
|
518
|
+
|
|
519
|
+
## License
|
|
520
|
+
|
|
521
|
+
MIT — See [LICENSE](LICENSE) for details.
|
|
522
|
+
|
|
523
|
+
---
|
|
524
|
+
|
|
525
|
+
## The Vision
|
|
526
|
+
|
|
527
|
+
**Every AI agent should have cost limits, safety measures, and access controls before deployment.**
|
|
528
|
+
|
|
529
|
+
release-gate makes this simple, automatic, and transparent.
|
|
530
|
+
|
|
531
|
+
---
|
|
532
|
+
|
|
533
|
+
**Prevent cost explosions. Enforce governance. Deploy with confidence.** 🚀
|
|
534
|
+
|
|
535
|
+
Built by [Vamsi](https://github.com/VamsiSudhakaran1) • [release-gate.com](https://release-gate.com)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: release-gate
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Governance enforcement for AI agents
|
|
5
|
+
Home-page: https://github.com/VamsiSudhakaran1/release-gate
|
|
6
|
+
Author: Vamsi Sudhakaran
|
|
7
|
+
Author-email: vamsi.sudhakaran@gmail.com
|
|
8
|
+
Requires-Python: >=3.8
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: pyyaml>=6.0
|
|
11
|
+
Requires-Dist: jsonschema>=4.0
|
|
12
|
+
Dynamic: author
|
|
13
|
+
Dynamic: author-email
|
|
14
|
+
Dynamic: home-page
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
Dynamic: requires-dist
|
|
17
|
+
Dynamic: requires-python
|
|
18
|
+
Dynamic: summary
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
setup.py
|
|
4
|
+
release_gate.egg-info/PKG-INFO
|
|
5
|
+
release_gate.egg-info/SOURCES.txt
|
|
6
|
+
release_gate.egg-info/dependency_links.txt
|
|
7
|
+
release_gate.egg-info/entry_points.txt
|
|
8
|
+
release_gate.egg-info/requires.txt
|
|
9
|
+
release_gate.egg-info/top_level.txt
|
|
10
|
+
tests/test_release_gate.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from setuptools import setup, find_packages
|
|
2
|
+
|
|
3
|
+
setup(
|
|
4
|
+
name="release-gate",
|
|
5
|
+
version="0.3.0",
|
|
6
|
+
description="Governance enforcement for AI agents",
|
|
7
|
+
author="Vamsi Sudhakaran",
|
|
8
|
+
author_email="vamsi.sudhakaran@gmail.com",
|
|
9
|
+
url="https://github.com/VamsiSudhakaran1/release-gate",
|
|
10
|
+
packages=find_packages(),
|
|
11
|
+
install_requires=[
|
|
12
|
+
"pyyaml>=6.0",
|
|
13
|
+
"jsonschema>=4.0",
|
|
14
|
+
],
|
|
15
|
+
python_requires=">=3.8",
|
|
16
|
+
entry_points={
|
|
17
|
+
"console_scripts": [
|
|
18
|
+
"release-gate=release_gate.cli:main",
|
|
19
|
+
],
|
|
20
|
+
},
|
|
21
|
+
)
|
|
@@ -0,0 +1,444 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Minimal automated smoke test suite for release-gate CLI
|
|
4
|
+
|
|
5
|
+
Tests:
|
|
6
|
+
1. Initialization creates files
|
|
7
|
+
2. PASS case returns exit code 0
|
|
8
|
+
3. WARN case returns exit code 10
|
|
9
|
+
4. FAIL case returns exit code 1
|
|
10
|
+
5. JSON output is valid
|
|
11
|
+
6. Custom output file is created
|
|
12
|
+
|
|
13
|
+
No pytest required - just run: python test_release_gate.py
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import subprocess
|
|
17
|
+
import json
|
|
18
|
+
import sys
|
|
19
|
+
import tempfile
|
|
20
|
+
import shutil
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def run_cli(*args, cwd=None):
|
|
25
|
+
"""Run CLI command and return (exit_code, stdout, stderr)"""
|
|
26
|
+
cmd = [sys.executable, "cli.py"] + list(args)
|
|
27
|
+
result = subprocess.run(
|
|
28
|
+
cmd,
|
|
29
|
+
cwd=cwd,
|
|
30
|
+
capture_output=True,
|
|
31
|
+
text=True
|
|
32
|
+
)
|
|
33
|
+
return result.returncode, result.stdout, result.stderr
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_init():
|
|
37
|
+
"""Test 1: Initialization creates files"""
|
|
38
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
39
|
+
tmpdir = Path(tmpdir)
|
|
40
|
+
# Copy cli.py to temp dir
|
|
41
|
+
import shutil as sh
|
|
42
|
+
sh.copy("cli.py", tmpdir / "cli.py")
|
|
43
|
+
|
|
44
|
+
code, stdout, stderr = run_cli("init", "--project", "test-init", cwd=tmpdir)
|
|
45
|
+
|
|
46
|
+
assert code == 0, f"Init failed: {stderr}"
|
|
47
|
+
assert (tmpdir / "release-gate.yaml").exists(), "Config not created"
|
|
48
|
+
assert (tmpdir / "valid_requests.jsonl").exists(), "Valid samples not created"
|
|
49
|
+
assert (tmpdir / "invalid_requests.jsonl").exists(), "Invalid samples not created"
|
|
50
|
+
print("✓ Test 1: Initialization - PASSED")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_pass_case():
|
|
54
|
+
"""Test 2: Valid config returns PASS (exit code 0)"""
|
|
55
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
56
|
+
tmpdir = Path(tmpdir)
|
|
57
|
+
import shutil as sh
|
|
58
|
+
sh.copy("cli.py", tmpdir / "cli.py")
|
|
59
|
+
|
|
60
|
+
# Initialize
|
|
61
|
+
run_cli("init", "--project", "test-pass", cwd=tmpdir)
|
|
62
|
+
|
|
63
|
+
# Run (should PASS)
|
|
64
|
+
code, stdout, stderr = run_cli("run", "--config", "release-gate.yaml", cwd=tmpdir)
|
|
65
|
+
|
|
66
|
+
assert code == 0, f"Expected exit code 0 for PASS, got {code}. Stderr: {stderr}"
|
|
67
|
+
assert "PASS" in stdout or "pass" in stdout.lower(), "PASS not in output"
|
|
68
|
+
print("✓ Test 2: PASS case (exit 0) - PASSED")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def test_fail_case():
|
|
72
|
+
"""Test 3: Invalid config returns FAIL (exit code 1)"""
|
|
73
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
74
|
+
tmpdir = Path(tmpdir)
|
|
75
|
+
import shutil as sh
|
|
76
|
+
sh.copy("cli.py", tmpdir / "cli.py")
|
|
77
|
+
|
|
78
|
+
# Create broken config (missing fallback)
|
|
79
|
+
broken_config = """
|
|
80
|
+
project:
|
|
81
|
+
name: broken
|
|
82
|
+
|
|
83
|
+
checks:
|
|
84
|
+
input_contract:
|
|
85
|
+
enabled: true
|
|
86
|
+
schema:
|
|
87
|
+
type: object
|
|
88
|
+
fallback_declared:
|
|
89
|
+
enabled: true
|
|
90
|
+
"""
|
|
91
|
+
(tmpdir / "broken.yaml").write_text(broken_config)
|
|
92
|
+
|
|
93
|
+
# Run (should FAIL)
|
|
94
|
+
code, stdout, stderr = run_cli("run", "--config", "broken.yaml", cwd=tmpdir)
|
|
95
|
+
|
|
96
|
+
assert code == 1, f"Expected exit code 1 for FAIL, got {code}"
|
|
97
|
+
print("✓ Test 3: FAIL case (exit 1) - PASSED")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def test_json_output():
|
|
101
|
+
"""Test 4: JSON output is valid"""
|
|
102
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
103
|
+
tmpdir = Path(tmpdir)
|
|
104
|
+
import shutil as sh
|
|
105
|
+
sh.copy("cli.py", tmpdir / "cli.py")
|
|
106
|
+
|
|
107
|
+
# Initialize
|
|
108
|
+
run_cli("init", "--project", "test-json", cwd=tmpdir)
|
|
109
|
+
|
|
110
|
+
# Run with JSON format
|
|
111
|
+
code, stdout, stderr = run_cli("run", "--config", "release-gate.yaml", "--format", "json", cwd=tmpdir)
|
|
112
|
+
|
|
113
|
+
assert code == 0, f"Run failed: {stderr}"
|
|
114
|
+
|
|
115
|
+
# Parse JSON
|
|
116
|
+
try:
|
|
117
|
+
data = json.loads(stdout)
|
|
118
|
+
assert "overall" in data, "JSON missing 'overall' field"
|
|
119
|
+
assert "checks" in data, "JSON missing 'checks' field"
|
|
120
|
+
assert data["overall"] == "PASS", f"Expected PASS, got {data['overall']}"
|
|
121
|
+
print("✓ Test 4: JSON output - PASSED")
|
|
122
|
+
except json.JSONDecodeError as e:
|
|
123
|
+
assert False, f"Invalid JSON output: {e}\n{stdout}"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def test_custom_output_file():
|
|
127
|
+
"""Test 5: Custom output file is created"""
|
|
128
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
129
|
+
tmpdir = Path(tmpdir)
|
|
130
|
+
import shutil as sh
|
|
131
|
+
sh.copy("cli.py", tmpdir / "cli.py")
|
|
132
|
+
|
|
133
|
+
# Initialize
|
|
134
|
+
run_cli("init", "--project", "test-output", cwd=tmpdir)
|
|
135
|
+
|
|
136
|
+
# Run with custom output
|
|
137
|
+
code, stdout, stderr = run_cli(
|
|
138
|
+
"run",
|
|
139
|
+
"--config", "release-gate.yaml",
|
|
140
|
+
"--output", "custom-report.json",
|
|
141
|
+
cwd=tmpdir
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
assert code == 0, f"Run failed: {stderr}"
|
|
145
|
+
|
|
146
|
+
output_file = tmpdir / "custom-report.json"
|
|
147
|
+
assert output_file.exists(), f"Custom output file not created: {output_file}"
|
|
148
|
+
|
|
149
|
+
# Verify it's valid JSON
|
|
150
|
+
data = json.loads(output_file.read_text())
|
|
151
|
+
assert data["overall"] == "PASS"
|
|
152
|
+
print("✓ Test 5: Custom output file - PASSED")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def test_sample_validation():
|
|
156
|
+
"""Test 6: INPUT_CONTRACT tests samples"""
|
|
157
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
158
|
+
tmpdir = Path(tmpdir)
|
|
159
|
+
import shutil as sh
|
|
160
|
+
sh.copy("cli.py", tmpdir / "cli.py")
|
|
161
|
+
|
|
162
|
+
# Initialize
|
|
163
|
+
run_cli("init", "--project", "test-samples", cwd=tmpdir)
|
|
164
|
+
|
|
165
|
+
# Run
|
|
166
|
+
code, stdout, stderr = run_cli("run", "--config", "release-gate.yaml", "--format", "json", cwd=tmpdir)
|
|
167
|
+
|
|
168
|
+
assert code == 0
|
|
169
|
+
data = json.loads(stdout)
|
|
170
|
+
|
|
171
|
+
# Find INPUT_CONTRACT check
|
|
172
|
+
input_contract = next((c for c in data["checks"] if c["name"] == "input_contract"), None)
|
|
173
|
+
assert input_contract is not None, "input_contract not found"
|
|
174
|
+
|
|
175
|
+
# Check evidence includes sample counts
|
|
176
|
+
evidence = input_contract["evidence"]
|
|
177
|
+
assert "valid_samples_tested" in evidence, "valid_samples_tested not in evidence"
|
|
178
|
+
assert "invalid_samples_tested" in evidence, "invalid_samples_tested not in evidence"
|
|
179
|
+
assert evidence["valid_samples_tested"] > 0, "No valid samples tested"
|
|
180
|
+
assert evidence["invalid_samples_tested"] > 0, "No invalid samples tested"
|
|
181
|
+
print("✓ Test 6: Sample validation - PASSED")
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def test_warn_case_invalid_samples_accepted():
|
|
185
|
+
"""Test 7: WARN when invalid samples incorrectly pass schema"""
|
|
186
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
187
|
+
tmpdir = Path(tmpdir)
|
|
188
|
+
import shutil as sh
|
|
189
|
+
sh.copy("cli.py", tmpdir / "cli.py")
|
|
190
|
+
|
|
191
|
+
# Create config with schema that's too loose (accepts everything)
|
|
192
|
+
config = """
|
|
193
|
+
project:
|
|
194
|
+
name: warn-test
|
|
195
|
+
|
|
196
|
+
gate:
|
|
197
|
+
policy: default-v0.1
|
|
198
|
+
|
|
199
|
+
checks:
|
|
200
|
+
input_contract:
|
|
201
|
+
enabled: true
|
|
202
|
+
schema:
|
|
203
|
+
type: object
|
|
204
|
+
samples:
|
|
205
|
+
valid_path: valid_requests.jsonl
|
|
206
|
+
invalid_path: invalid_requests.jsonl
|
|
207
|
+
|
|
208
|
+
fallback_declared:
|
|
209
|
+
enabled: true
|
|
210
|
+
kill_switch:
|
|
211
|
+
type: feature_flag
|
|
212
|
+
name: disable_warn_test
|
|
213
|
+
fallback:
|
|
214
|
+
mode: static_placeholder
|
|
215
|
+
ownership:
|
|
216
|
+
team: platform-team
|
|
217
|
+
oncall: oncall-platform
|
|
218
|
+
runbook_url: https://wiki.internal/runbooks/test
|
|
219
|
+
"""
|
|
220
|
+
(tmpdir / "release-gate.yaml").write_text(config)
|
|
221
|
+
|
|
222
|
+
# Create invalid samples that will pass the loose schema
|
|
223
|
+
invalid_samples = """{"any": "value"}
|
|
224
|
+
{"random": "data"}
|
|
225
|
+
{"test": 123}
|
|
226
|
+
"""
|
|
227
|
+
(tmpdir / "invalid_requests.jsonl").write_text(invalid_samples)
|
|
228
|
+
|
|
229
|
+
# Create valid samples
|
|
230
|
+
valid_samples = """{"test": "valid"}
|
|
231
|
+
{"data": "here"}
|
|
232
|
+
{"foo": "bar"}
|
|
233
|
+
"""
|
|
234
|
+
(tmpdir / "valid_requests.jsonl").write_text(valid_samples)
|
|
235
|
+
|
|
236
|
+
# Run
|
|
237
|
+
code, stdout, stderr = run_cli("run", "--config", "release-gate.yaml", "--format", "json", cwd=tmpdir)
|
|
238
|
+
|
|
239
|
+
# Should return WARN (exit code 10) because invalid samples are accepted
|
|
240
|
+
assert code == 10, f"Expected exit code 10 for WARN, got {code}. Stderr: {stderr}"
|
|
241
|
+
|
|
242
|
+
data = json.loads(stdout)
|
|
243
|
+
assert data["overall"] == "WARN", f"Expected overall WARN, got {data['overall']}"
|
|
244
|
+
|
|
245
|
+
# Check INPUT_CONTRACT returned WARN
|
|
246
|
+
input_contract = next((c for c in data["checks"] if c["name"] == "input_contract"), None)
|
|
247
|
+
assert input_contract is not None
|
|
248
|
+
assert input_contract["result"] == "WARN", f"Expected INPUT_CONTRACT WARN, got {input_contract['result']}"
|
|
249
|
+
|
|
250
|
+
# Check evidence shows invalid samples were accepted
|
|
251
|
+
evidence = input_contract["evidence"]
|
|
252
|
+
assert evidence["invalid_samples_accepted"] > 0, "Expected some invalid samples to be accepted"
|
|
253
|
+
|
|
254
|
+
print("✓ Test 7: WARN case (invalid samples accepted) - PASSED")
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def test_warn_exit_code_10():
|
|
258
|
+
"""Test 8: WARN returns exit code 10"""
|
|
259
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
260
|
+
tmpdir = Path(tmpdir)
|
|
261
|
+
import shutil as sh
|
|
262
|
+
sh.copy("cli.py", tmpdir / "cli.py")
|
|
263
|
+
|
|
264
|
+
# Create config with loose schema
|
|
265
|
+
config = """
|
|
266
|
+
project:
|
|
267
|
+
name: warn-exit-test
|
|
268
|
+
|
|
269
|
+
checks:
|
|
270
|
+
input_contract:
|
|
271
|
+
enabled: true
|
|
272
|
+
schema:
|
|
273
|
+
type: object
|
|
274
|
+
samples:
|
|
275
|
+
valid_path: valid_requests.jsonl
|
|
276
|
+
invalid_path: invalid_requests.jsonl
|
|
277
|
+
|
|
278
|
+
fallback_declared:
|
|
279
|
+
enabled: true
|
|
280
|
+
kill_switch:
|
|
281
|
+
type: feature_flag
|
|
282
|
+
name: test
|
|
283
|
+
fallback:
|
|
284
|
+
mode: static_placeholder
|
|
285
|
+
ownership:
|
|
286
|
+
team: team
|
|
287
|
+
oncall: oncall
|
|
288
|
+
runbook_url: https://test.com
|
|
289
|
+
"""
|
|
290
|
+
(tmpdir / "release-gate.yaml").write_text(config)
|
|
291
|
+
|
|
292
|
+
# Create samples where invalid ones pass
|
|
293
|
+
(tmpdir / "valid_requests.jsonl").write_text('{"valid": true}\n')
|
|
294
|
+
(tmpdir / "invalid_requests.jsonl").write_text('{"invalid": true}\n')
|
|
295
|
+
|
|
296
|
+
# Run
|
|
297
|
+
code, _, stderr = run_cli("run", "--config", "release-gate.yaml", cwd=tmpdir)
|
|
298
|
+
|
|
299
|
+
# Should return exit code 10 for WARN
|
|
300
|
+
assert code == 10, f"Expected exit code 10 for WARN, got {code}"
|
|
301
|
+
print("✓ Test 8: WARN exit code (10) - PASSED")
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def test_warn_in_summary():
|
|
305
|
+
"""Test 9: WARN appears in summary"""
|
|
306
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
307
|
+
tmpdir = Path(tmpdir)
|
|
308
|
+
import shutil as sh
|
|
309
|
+
sh.copy("cli.py", tmpdir / "cli.py")
|
|
310
|
+
|
|
311
|
+
# Create loose schema config
|
|
312
|
+
config = """
|
|
313
|
+
project:
|
|
314
|
+
name: warn-summary-test
|
|
315
|
+
|
|
316
|
+
checks:
|
|
317
|
+
input_contract:
|
|
318
|
+
enabled: true
|
|
319
|
+
schema:
|
|
320
|
+
type: object
|
|
321
|
+
samples:
|
|
322
|
+
valid_path: valid_requests.jsonl
|
|
323
|
+
invalid_path: invalid_requests.jsonl
|
|
324
|
+
|
|
325
|
+
fallback_declared:
|
|
326
|
+
enabled: true
|
|
327
|
+
kill_switch:
|
|
328
|
+
type: feature_flag
|
|
329
|
+
name: test
|
|
330
|
+
fallback:
|
|
331
|
+
mode: static_placeholder
|
|
332
|
+
ownership:
|
|
333
|
+
team: team
|
|
334
|
+
oncall: oncall
|
|
335
|
+
runbook_url: https://test.com
|
|
336
|
+
"""
|
|
337
|
+
(tmpdir / "release-gate.yaml").write_text(config)
|
|
338
|
+
(tmpdir / "valid_requests.jsonl").write_text('{"valid": true}\n')
|
|
339
|
+
(tmpdir / "invalid_requests.jsonl").write_text('{"invalid": true}\n')
|
|
340
|
+
|
|
341
|
+
# Run with JSON format
|
|
342
|
+
code, stdout, stderr = run_cli("run", "--config", "release-gate.yaml", "--format", "json", cwd=tmpdir)
|
|
343
|
+
|
|
344
|
+
data = json.loads(stdout)
|
|
345
|
+
|
|
346
|
+
# Check summary counts warn
|
|
347
|
+
summary = data["summary"]["counts"]
|
|
348
|
+
assert summary["warn"] > 0, "Expected warn count > 0 in summary"
|
|
349
|
+
|
|
350
|
+
print("✓ Test 9: WARN in summary counts - PASSED")
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def test_warn_precedence_fail_wins():
|
|
354
|
+
"""Test 10: FAIL takes precedence over WARN"""
|
|
355
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
356
|
+
tmpdir = Path(tmpdir)
|
|
357
|
+
import shutil as sh
|
|
358
|
+
sh.copy("cli.py", tmpdir / "cli.py")
|
|
359
|
+
|
|
360
|
+
# Create config: INPUT_CONTRACT WARN + FALLBACK_DECLARED FAIL
|
|
361
|
+
config = """
|
|
362
|
+
project:
|
|
363
|
+
name: fail-precedence-test
|
|
364
|
+
|
|
365
|
+
checks:
|
|
366
|
+
input_contract:
|
|
367
|
+
enabled: true
|
|
368
|
+
schema:
|
|
369
|
+
type: object
|
|
370
|
+
samples:
|
|
371
|
+
valid_path: valid_requests.jsonl
|
|
372
|
+
invalid_path: invalid_requests.jsonl
|
|
373
|
+
|
|
374
|
+
fallback_declared:
|
|
375
|
+
enabled: true
|
|
376
|
+
"""
|
|
377
|
+
(tmpdir / "release-gate.yaml").write_text(config)
|
|
378
|
+
(tmpdir / "valid_requests.jsonl").write_text('{"valid": true}\n')
|
|
379
|
+
(tmpdir / "invalid_requests.jsonl").write_text('{"invalid": true}\n')
|
|
380
|
+
|
|
381
|
+
# Run
|
|
382
|
+
code, stdout, stderr = run_cli("run", "--config", "release-gate.yaml", "--format", "json", cwd=tmpdir)
|
|
383
|
+
|
|
384
|
+
data = json.loads(stdout)
|
|
385
|
+
|
|
386
|
+
# Even though INPUT_CONTRACT might WARN, overall should be FAIL (fallback missing)
|
|
387
|
+
assert data["overall"] == "FAIL", f"Expected FAIL to take precedence, got {data['overall']}"
|
|
388
|
+
assert code == 1, f"Expected exit code 1 (FAIL), got {code}"
|
|
389
|
+
|
|
390
|
+
print("✓ Test 10: FAIL precedence over WARN - PASSED")
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def main():
|
|
394
|
+
"""Run all tests"""
|
|
395
|
+
print("\n" + "="*70)
|
|
396
|
+
print(" release-gate Automated Smoke Test Suite")
|
|
397
|
+
print("="*70 + "\n")
|
|
398
|
+
|
|
399
|
+
tests = [
|
|
400
|
+
("Initialization", test_init),
|
|
401
|
+
("PASS case", test_pass_case),
|
|
402
|
+
("FAIL case", test_fail_case),
|
|
403
|
+
("JSON output", test_json_output),
|
|
404
|
+
("Custom output file", test_custom_output_file),
|
|
405
|
+
("Sample validation", test_sample_validation),
|
|
406
|
+
("WARN case (invalid samples accepted)", test_warn_case_invalid_samples_accepted),
|
|
407
|
+
("WARN exit code (10)", test_warn_exit_code_10),
|
|
408
|
+
("WARN in summary", test_warn_in_summary),
|
|
409
|
+
("FAIL precedence over WARN", test_warn_precedence_fail_wins),
|
|
410
|
+
# Phase 2 tests
|
|
411
|
+
("Phase 2: IDENTITY_BOUNDARY PASS", test_phase_2_identity_boundary),
|
|
412
|
+
("Phase 2: IDENTITY_BOUNDARY FAIL", test_phase_2_identity_boundary_fail),
|
|
413
|
+
("Phase 2: ACTION_BUDGET PASS", test_phase_2_action_budget),
|
|
414
|
+
("Phase 2: ACTION_BUDGET FAIL", test_phase_2_action_budget_fail),
|
|
415
|
+
]
|
|
416
|
+
|
|
417
|
+
passed = 0
|
|
418
|
+
failed = 0
|
|
419
|
+
|
|
420
|
+
for test_name, test_func in tests:
|
|
421
|
+
try:
|
|
422
|
+
test_func()
|
|
423
|
+
passed += 1
|
|
424
|
+
except AssertionError as e:
|
|
425
|
+
print(f"✗ {test_name} - FAILED: {e}")
|
|
426
|
+
failed += 1
|
|
427
|
+
except Exception as e:
|
|
428
|
+
print(f"✗ {test_name} - ERROR: {e}")
|
|
429
|
+
failed += 1
|
|
430
|
+
|
|
431
|
+
print("\n" + "="*70)
|
|
432
|
+
print(f"Results: {passed} passed, {failed} failed")
|
|
433
|
+
print("="*70 + "\n")
|
|
434
|
+
|
|
435
|
+
if failed == 0:
|
|
436
|
+
print("✅ All tests passed! release-gate is working correctly.\n")
|
|
437
|
+
return 0
|
|
438
|
+
else:
|
|
439
|
+
print("❌ Some tests failed. See details above.\n")
|
|
440
|
+
return 1
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
if __name__ == "__main__":
|
|
444
|
+
sys.exit(main())
|