kaizeval 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kaizeval/__init__.py +3 -0
- kaizeval/agentkit.py +234 -0
- kaizeval/api/__init__.py +1 -0
- kaizeval/api/auth.py +179 -0
- kaizeval/api/deps.py +115 -0
- kaizeval/api/knowledge.py +313 -0
- kaizeval/api/runs.py +472 -0
- kaizeval/api/setup.py +356 -0
- kaizeval/app.py +131 -0
- kaizeval/audit.py +16 -0
- kaizeval/cli.py +420 -0
- kaizeval/client.py +51 -0
- kaizeval/config.py +91 -0
- kaizeval/connectors/__init__.py +78 -0
- kaizeval/connectors/base.py +115 -0
- kaizeval/connectors/http_chat.py +143 -0
- kaizeval/connectors/local_config.py +122 -0
- kaizeval/db.py +77 -0
- kaizeval/documents.py +221 -0
- kaizeval/gateway/__init__.py +7 -0
- kaizeval/gateway/core.py +406 -0
- kaizeval/gateway/fake.py +235 -0
- kaizeval/gateway/prompts.py +176 -0
- kaizeval/gateway/schemas.py +125 -0
- kaizeval/ground_truth.py +122 -0
- kaizeval/handlers.py +145 -0
- kaizeval/jobs.py +253 -0
- kaizeval/mcp_server.py +113 -0
- kaizeval/migrations/env.py +29 -0
- kaizeval/migrations/script.py.mako +22 -0
- kaizeval/migrations/versions/0001_initial_schema.py +632 -0
- kaizeval/models.py +395 -0
- kaizeval/netguard.py +108 -0
- kaizeval/project.py +701 -0
- kaizeval/py.typed +0 -0
- kaizeval/sample_agent/__init__.py +5 -0
- kaizeval/sample_agent/app.py +193 -0
- kaizeval/sample_agent/kb/returns-and-refunds.md +23 -0
- kaizeval/sample_agent/kb/shipping-and-delivery.md +23 -0
- kaizeval/sample_agent/kb/warranty-and-repairs.md +23 -0
- kaizeval/security.py +109 -0
- kaizeval/services/__init__.py +1 -0
- kaizeval/services/datasets.py +521 -0
- kaizeval/services/diagnostics.py +238 -0
- kaizeval/services/evaluation.py +557 -0
- kaizeval/services/holdout.py +58 -0
- kaizeval/services/optimization.py +676 -0
- kaizeval/services/reporting.py +223 -0
- kaizeval/services/specs.py +150 -0
- kaizeval/services/stats.py +122 -0
- kaizeval/storage.py +115 -0
- kaizeval/textutil.py +186 -0
- kaizeval/webui/README.txt +2 -0
- kaizeval/webui/assets/index-BIG40jck.css +1 -0
- kaizeval/webui/assets/index-D8bsydB0.js +72 -0
- kaizeval/webui/favicon.svg +1 -0
- kaizeval/webui/index.html +15 -0
- kaizeval-0.2.0.dist-info/METADATA +427 -0
- kaizeval-0.2.0.dist-info/RECORD +63 -0
- kaizeval-0.2.0.dist-info/WHEEL +4 -0
- kaizeval-0.2.0.dist-info/entry_points.txt +2 -0
- kaizeval-0.2.0.dist-info/licenses/LICENSE +202 -0
- kaizeval-0.2.0.dist-info/licenses/NOTICE +16 -0
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
"""Knowledge documents, specifications and test datasets."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Literal
|
|
6
|
+
|
|
7
|
+
from fastapi import APIRouter, Depends, File, Form, Request, UploadFile
|
|
8
|
+
from pydantic import BaseModel, Field
|
|
9
|
+
from sqlalchemy import select
|
|
10
|
+
|
|
11
|
+
from .. import audit, jobs
|
|
12
|
+
from ..config import get_settings
|
|
13
|
+
from ..documents import ingest
|
|
14
|
+
from ..models import Agent, Dataset, Document, EvidenceChunk, Specification, TestCase, iso
|
|
15
|
+
from ..services import datasets as dataset_service
|
|
16
|
+
from ..services import specs as spec_service
|
|
17
|
+
from ..services.datasets import HoldoutAccess
|
|
18
|
+
from .deps import Ctx, fail, idempotency_key, job_dict, page, scoped, ser, workspace_ctx
|
|
19
|
+
|
|
20
|
+
router = APIRouter(prefix="/workspaces/{workspace_id}")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
async def _read(file: UploadFile) -> bytes:
|
|
24
|
+
limit = get_settings().max_upload_mb * 1024 * 1024
|
|
25
|
+
data = await file.read(limit + 1)
|
|
26
|
+
if len(data) > limit:
|
|
27
|
+
raise fail(413, "FILE_TOO_LARGE", f"files are limited to {get_settings().max_upload_mb} MB")
|
|
28
|
+
return data
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
# --- documents ----------------------------------------------------------------------------
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@router.post("/documents", status_code=201)
|
|
35
|
+
async def upload_document(file: UploadFile = File(...), kind: Literal["kb", "requirements"] = Form("kb"),
|
|
36
|
+
ctx: Ctx = Depends(workspace_ctx("editor"))) -> dict[str, Any]:
|
|
37
|
+
data = await _read(file)
|
|
38
|
+
document = ingest(ctx.db, workspace_id=ctx.ws, filename=file.filename or "document", data=data, kind=kind,
|
|
39
|
+
content_type=file.content_type or "")
|
|
40
|
+
audit.record(ctx.db, workspace_id=ctx.ws, user_id=ctx.user.id, action="document.uploaded", subject_type="document",
|
|
41
|
+
subject_id=document.id, detail={"filename": document.filename, "kind": kind, "version": document.version,
|
|
42
|
+
"sha256": document.sha256, "status": document.status})
|
|
43
|
+
ctx.db.commit()
|
|
44
|
+
flagged = ctx.db.scalars(select(EvidenceChunk).where(EvidenceChunk.document_id == document.id)).all()
|
|
45
|
+
return ser(document, exclude={"storage_key"},
|
|
46
|
+
extra={"flagged_chunks": sum(1 for c in flagged if c.flags)})
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@router.get("/documents")
|
|
50
|
+
def list_documents(kind: str | None = None, include_superseded: bool = False, limit: int = 100, offset: int = 0,
|
|
51
|
+
ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
|
|
52
|
+
query = select(Document).where(Document.workspace_id == ctx.ws).order_by(Document.filename, Document.version.desc())
|
|
53
|
+
if kind:
|
|
54
|
+
query = query.where(Document.kind == kind)
|
|
55
|
+
if not include_superseded:
|
|
56
|
+
query = query.where(Document.status != "superseded")
|
|
57
|
+
return page(ctx.db, query, limit=limit, offset=offset, transform=lambda d: ser(d, exclude={"storage_key"}))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@router.get("/documents/{document_id}/chunks")
|
|
61
|
+
def document_chunks(document_id: str, limit: int = 100, offset: int = 0, ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
|
|
62
|
+
document = scoped(ctx, Document, document_id)
|
|
63
|
+
query = select(EvidenceChunk).where(EvidenceChunk.document_id == document.id).order_by(EvidenceChunk.ordinal)
|
|
64
|
+
result = page(ctx.db, query, limit=limit, offset=offset)
|
|
65
|
+
result["document"] = ser(document, exclude={"storage_key"})
|
|
66
|
+
return result
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@router.get("/evidence/{chunk_id}")
|
|
70
|
+
def evidence(chunk_id: str, ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
|
|
71
|
+
chunk = scoped(ctx, EvidenceChunk, chunk_id)
|
|
72
|
+
document = ctx.db.get(Document, chunk.document_id)
|
|
73
|
+
return ser(chunk, extra={"filename": document.filename if document else None,
|
|
74
|
+
"document_version": document.version if document else None,
|
|
75
|
+
"sha256": document.sha256 if document else None})
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# --- specifications -----------------------------------------------------------------------
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class SpecBuild(BaseModel):
|
|
82
|
+
agent_id: str | None = None
|
|
83
|
+
requirements_text: str | None = Field(default=None, max_length=50000)
|
|
84
|
+
requirement_document_ids: list[str] = Field(default_factory=list)
|
|
85
|
+
infer: bool = True
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class SpecEdit(BaseModel):
|
|
89
|
+
content: dict[str, Any]
|
|
90
|
+
approve: bool = False
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class SpecReview(BaseModel):
|
|
94
|
+
decision: Literal["approve", "skip"]
|
|
95
|
+
note: str | None = Field(default=None, max_length=2000)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@router.post("/specifications:build", status_code=202)
|
|
99
|
+
def build_spec(body: SpecBuild, request: Request, ctx: Ctx = Depends(workspace_ctx("editor"))) -> dict[str, Any]:
|
|
100
|
+
if body.agent_id:
|
|
101
|
+
scoped(ctx, Agent, body.agent_id)
|
|
102
|
+
for document_id in body.requirement_document_ids:
|
|
103
|
+
scoped(ctx, Document, document_id)
|
|
104
|
+
job = jobs.enqueue(ctx.db, workspace_id=ctx.ws, kind="specification.build", payload=body.model_dump(),
|
|
105
|
+
created_by=ctx.user.id, idempotency_key=idempotency_key(request))
|
|
106
|
+
return job_dict(job)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@router.get("/specifications")
|
|
110
|
+
def list_specs(limit: int = 50, offset: int = 0, ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
|
|
111
|
+
query = select(Specification).where(Specification.workspace_id == ctx.ws).order_by(Specification.created_at.desc())
|
|
112
|
+
return page(ctx.db, query, limit=limit, offset=offset)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@router.get("/specifications/{spec_id}")
|
|
116
|
+
def get_spec(spec_id: str, ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
|
|
117
|
+
spec = scoped(ctx, Specification, spec_id)
|
|
118
|
+
versions = ctx.db.scalars(select(Specification).where(Specification.lineage_id == spec.lineage_id)
|
|
119
|
+
.order_by(Specification.version))
|
|
120
|
+
return ser(spec, extra={"versions": [{"id": v.id, "version": v.version, "status": v.status, "source": v.source,
|
|
121
|
+
"created_at": iso(v.created_at), "created_by": v.created_by}
|
|
122
|
+
for v in versions]})
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@router.patch("/specifications/{spec_id}")
|
|
126
|
+
def edit_spec(spec_id: str, body: SpecEdit, ctx: Ctx = Depends(workspace_ctx("reviewer"))) -> dict[str, Any]:
|
|
127
|
+
spec = scoped(ctx, Specification, spec_id)
|
|
128
|
+
new = spec_service.edit_specification(ctx.db, spec, content=body.content, user_id=ctx.user.id, approve=body.approve)
|
|
129
|
+
ctx.db.commit()
|
|
130
|
+
return ser(new)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@router.post("/specifications/{spec_id}/review")
|
|
134
|
+
def review_spec(spec_id: str, body: SpecReview, ctx: Ctx = Depends(workspace_ctx("reviewer"))) -> dict[str, Any]:
|
|
135
|
+
spec = scoped(ctx, Specification, spec_id)
|
|
136
|
+
spec_service.review_specification(ctx.db, spec, decision=body.decision, user_id=ctx.user.id, note=body.note)
|
|
137
|
+
ctx.db.commit()
|
|
138
|
+
return ser(spec)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
# --- datasets -----------------------------------------------------------------------------
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class DatasetGenerate(BaseModel):
|
|
145
|
+
name: str = Field(min_length=1, max_length=200)
|
|
146
|
+
agent_id: str | None = None
|
|
147
|
+
specification_id: str | None = None
|
|
148
|
+
n_cases: int = Field(default=20, ge=4, le=500)
|
|
149
|
+
testers: int = Field(default=1, ge=1, le=8) # independent test-writing agents
|
|
150
|
+
keep_unverified: bool = False # keep generated cases whose reference could not be linked to evidence
|
|
151
|
+
holdout_fraction: float = Field(default=0.3, ge=0.0, le=0.6)
|
|
152
|
+
seed: int = 0
|
|
153
|
+
categories: list[Literal["knowledge", "citation", "edge_case"]] = ["knowledge", "citation", "edge_case"]
|
|
154
|
+
include_multi_turn: bool = True
|
|
155
|
+
unanswerable_share: float = Field(default=0.2, ge=0.0, le=0.5)
|
|
156
|
+
languages: list[str] = Field(default_factory=list)
|
|
157
|
+
document_ids: list[str] | None = None
|
|
158
|
+
limits: dict[str, float] = Field(default_factory=dict)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class CaseUpdate(BaseModel):
|
|
162
|
+
expected: dict[str, Any] | None = None
|
|
163
|
+
answer_status: Literal["approved", "needs_human_review", "unverified"] | None = None
|
|
164
|
+
severity: Literal["low", "normal", "high", "critical"] | None = None
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
class RotateBody(BaseModel):
|
|
168
|
+
n_cases: int | None = Field(default=None, ge=4, le=500)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _dataset_dict(ctx: Ctx, dataset: Dataset) -> dict[str, Any]:
|
|
172
|
+
return ser(dataset, extra={"counts": dataset_service.split_counts(ctx.db, dataset.id)})
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _case_dict(case: TestCase) -> dict[str, Any]:
|
|
176
|
+
return ser(case)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
@router.post("/datasets:generate", status_code=202)
|
|
180
|
+
def generate_dataset(body: DatasetGenerate, request: Request, ctx: Ctx = Depends(workspace_ctx("editor"))) -> dict[str, Any]:
|
|
181
|
+
"""Generate evidence-linked cases and split them into regression and sealed holdout, before any optimization."""
|
|
182
|
+
key = idempotency_key(request)
|
|
183
|
+
if key:
|
|
184
|
+
existing = ctx.db.scalar(select(jobs.Job).where(jobs.Job.workspace_id == ctx.ws, jobs.Job.idempotency_key == key))
|
|
185
|
+
if existing is not None:
|
|
186
|
+
return {"job": job_dict(existing), "dataset": _dataset_dict(ctx, scoped(ctx, Dataset, existing.payload["dataset_id"]))}
|
|
187
|
+
if body.agent_id:
|
|
188
|
+
scoped(ctx, Agent, body.agent_id)
|
|
189
|
+
if body.specification_id:
|
|
190
|
+
scoped(ctx, Specification, body.specification_id)
|
|
191
|
+
dataset = Dataset(workspace_id=ctx.ws, name=body.name.strip(), agent_id=body.agent_id,
|
|
192
|
+
specification_id=body.specification_id, seed=body.seed, holdout_fraction=body.holdout_fraction)
|
|
193
|
+
ctx.db.add(dataset)
|
|
194
|
+
ctx.db.flush()
|
|
195
|
+
params = body.model_dump(exclude={"name", "agent_id", "specification_id", "seed", "holdout_fraction"})
|
|
196
|
+
job = jobs.enqueue(ctx.db, workspace_id=ctx.ws, kind="dataset.generate",
|
|
197
|
+
payload={"dataset_id": dataset.id, "params": params}, created_by=ctx.user.id, idempotency_key=key)
|
|
198
|
+
ctx.db.expire_all()
|
|
199
|
+
return {"job": job_dict(job), "dataset": _dataset_dict(ctx, scoped(ctx, Dataset, dataset.id))}
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
@router.post("/datasets:import", status_code=201)
|
|
203
|
+
async def import_dataset(file: UploadFile = File(...), name: str | None = Form(None), dataset_id: str | None = Form(None),
|
|
204
|
+
approved: bool = Form(True), holdout_fraction: float = Form(0.0),
|
|
205
|
+
ctx: Ctx = Depends(workspace_ctx("editor"))) -> dict[str, Any]:
|
|
206
|
+
"""Import business-owned Q&A or real conversations (.csv, .jsonl, .json)."""
|
|
207
|
+
data = await _read(file)
|
|
208
|
+
try:
|
|
209
|
+
rows = dataset_service.parse_import(file.filename or "", data)
|
|
210
|
+
except (dataset_service.DatasetError, ValueError) as exc:
|
|
211
|
+
raise fail(422, "INVALID_IMPORT", str(exc)) from exc
|
|
212
|
+
if dataset_id:
|
|
213
|
+
dataset = scoped(ctx, Dataset, dataset_id)
|
|
214
|
+
else:
|
|
215
|
+
dataset = Dataset(workspace_id=ctx.ws, name=(name or file.filename or "Imported cases").strip(), status="ready")
|
|
216
|
+
ctx.db.add(dataset)
|
|
217
|
+
ctx.db.flush()
|
|
218
|
+
try:
|
|
219
|
+
result = dataset_service.import_cases(ctx.db, dataset, rows, approved=approved, user_id=ctx.user.id,
|
|
220
|
+
holdout_fraction=max(0.0, min(0.6, holdout_fraction)),
|
|
221
|
+
source_name=file.filename or "upload")
|
|
222
|
+
except dataset_service.DatasetError as exc:
|
|
223
|
+
ctx.db.rollback()
|
|
224
|
+
raise fail(422, "INVALID_IMPORT", str(exc)) from exc
|
|
225
|
+
ctx.db.commit()
|
|
226
|
+
return {"dataset": _dataset_dict(ctx, dataset), **result}
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
@router.get("/datasets")
|
|
230
|
+
def list_datasets(limit: int = 50, offset: int = 0, ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
|
|
231
|
+
query = select(Dataset).where(Dataset.workspace_id == ctx.ws).order_by(Dataset.created_at.desc())
|
|
232
|
+
return page(ctx.db, query, limit=limit, offset=offset, transform=lambda d: _dataset_dict(ctx, d))
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
@router.get("/datasets/{dataset_id}")
|
|
236
|
+
def get_dataset(dataset_id: str, ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
|
|
237
|
+
return _dataset_dict(ctx, scoped(ctx, Dataset, dataset_id))
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
@router.get("/datasets/{dataset_id}/coverage")
|
|
241
|
+
def dataset_coverage(dataset_id: str, ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
|
|
242
|
+
return dataset_service.coverage(ctx.db, scoped(ctx, Dataset, dataset_id))
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
@router.get("/datasets/{dataset_id}/cases")
|
|
246
|
+
def list_cases(dataset_id: str, split: Literal["regression", "holdout"] = "regression", acknowledge: bool = False,
|
|
247
|
+
limit: int = 50, offset: int = 0, ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
|
|
248
|
+
dataset = scoped(ctx, Dataset, dataset_id)
|
|
249
|
+
if split == "holdout":
|
|
250
|
+
# Holdout items are sealed. A reviewer may open them deliberately; doing so is audited and
|
|
251
|
+
# marks the holdout as exposed, so it must be rotated before it can gate further tuning.
|
|
252
|
+
ctx.require("reviewer")
|
|
253
|
+
if not acknowledge:
|
|
254
|
+
raise fail(403, "HOLDOUT_SEALED", "holdout items are sealed. Pass acknowledge=true to open them; "
|
|
255
|
+
"this is audited and the holdout will need rotating before it is used again.")
|
|
256
|
+
dataset_service.holdout_cases(ctx.db, dataset.id, HoldoutAccess("human_review", ctx.user.id))
|
|
257
|
+
dataset.holdout_exposed = True
|
|
258
|
+
audit.record(ctx.db, workspace_id=ctx.ws, user_id=ctx.user.id, action="holdout.viewed",
|
|
259
|
+
subject_type="dataset", subject_id=dataset.id, detail={"generation": dataset.holdout_generation})
|
|
260
|
+
ctx.db.commit()
|
|
261
|
+
query = (select(TestCase).where(TestCase.dataset_id == dataset.id, TestCase.split == split)
|
|
262
|
+
.order_by(TestCase.created_at, TestCase.id))
|
|
263
|
+
return page(ctx.db, query, limit=limit, offset=offset, transform=_case_dict)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
@router.patch("/datasets/{dataset_id}/cases/{case_id}")
|
|
267
|
+
def update_case(dataset_id: str, case_id: str, body: CaseUpdate, ctx: Ctx = Depends(workspace_ctx("reviewer"))) -> dict[str, Any]:
|
|
268
|
+
"""Human review of a reference: approve it, correct it, or flag it. Approved references outrank drafts."""
|
|
269
|
+
dataset = scoped(ctx, Dataset, dataset_id)
|
|
270
|
+
case = scoped(ctx, TestCase, case_id)
|
|
271
|
+
if case.dataset_id != dataset.id:
|
|
272
|
+
raise fail(404, "NOT_FOUND", "case not found in this dataset")
|
|
273
|
+
if case.split != "regression":
|
|
274
|
+
raise fail(403, "HOLDOUT_SEALED", "only regression cases can be edited")
|
|
275
|
+
if body.expected is not None:
|
|
276
|
+
case.expected = {**(case.expected or {}), **body.expected}
|
|
277
|
+
if body.answer_status:
|
|
278
|
+
case.answer_status = body.answer_status
|
|
279
|
+
case.provenance = {**(case.provenance or {}), "reviewed_by": ctx.user.id}
|
|
280
|
+
if body.severity:
|
|
281
|
+
case.severity = body.severity
|
|
282
|
+
case.weight = 2.0 if body.severity == "critical" else 1.0
|
|
283
|
+
dataset.version += 1
|
|
284
|
+
audit.record(ctx.db, workspace_id=ctx.ws, user_id=ctx.user.id, action="case.reviewed", subject_type="test_case",
|
|
285
|
+
subject_id=case.id, detail={"answer_status": case.answer_status, "severity": case.severity})
|
|
286
|
+
ctx.db.commit()
|
|
287
|
+
return _case_dict(case)
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
@router.post("/datasets/{dataset_id}/holdout:rotate", status_code=202)
|
|
291
|
+
def rotate(dataset_id: str, body: RotateBody, request: Request, ctx: Ctx = Depends(workspace_ctx("editor"))) -> dict[str, Any]:
|
|
292
|
+
dataset = scoped(ctx, Dataset, dataset_id)
|
|
293
|
+
job = jobs.enqueue(ctx.db, workspace_id=ctx.ws, kind="dataset.rotate_holdout",
|
|
294
|
+
payload={"dataset_id": dataset.id, "params": body.model_dump(exclude_none=True)},
|
|
295
|
+
created_by=ctx.user.id, idempotency_key=idempotency_key(request))
|
|
296
|
+
return job_dict(job)
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
@router.post("/datasets/{dataset_id}/stale:retire")
|
|
300
|
+
def retire_stale(dataset_id: str, ctx: Ctx = Depends(workspace_ctx("editor"))) -> dict[str, Any]:
|
|
301
|
+
"""Retire regression cases whose evidence came from a document version that has been replaced."""
|
|
302
|
+
dataset = scoped(ctx, Dataset, dataset_id)
|
|
303
|
+
retired = 0
|
|
304
|
+
for case in ctx.db.scalars(select(TestCase).where(TestCase.dataset_id == dataset.id, TestCase.stale.is_(True),
|
|
305
|
+
TestCase.split == "regression")):
|
|
306
|
+
case.split = "retired"
|
|
307
|
+
retired += 1
|
|
308
|
+
if retired:
|
|
309
|
+
dataset.version += 1
|
|
310
|
+
audit.record(ctx.db, workspace_id=ctx.ws, user_id=ctx.user.id, action="dataset.stale_retired",
|
|
311
|
+
subject_type="dataset", subject_id=dataset.id, detail={"retired": retired})
|
|
312
|
+
ctx.db.commit()
|
|
313
|
+
return {"retired": retired, "dataset": _dataset_dict(ctx, dataset)}
|