kaizeval 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. kaizeval/__init__.py +3 -0
  2. kaizeval/agentkit.py +234 -0
  3. kaizeval/api/__init__.py +1 -0
  4. kaizeval/api/auth.py +179 -0
  5. kaizeval/api/deps.py +115 -0
  6. kaizeval/api/knowledge.py +313 -0
  7. kaizeval/api/runs.py +472 -0
  8. kaizeval/api/setup.py +356 -0
  9. kaizeval/app.py +131 -0
  10. kaizeval/audit.py +16 -0
  11. kaizeval/cli.py +420 -0
  12. kaizeval/client.py +51 -0
  13. kaizeval/config.py +91 -0
  14. kaizeval/connectors/__init__.py +78 -0
  15. kaizeval/connectors/base.py +115 -0
  16. kaizeval/connectors/http_chat.py +143 -0
  17. kaizeval/connectors/local_config.py +122 -0
  18. kaizeval/db.py +77 -0
  19. kaizeval/documents.py +221 -0
  20. kaizeval/gateway/__init__.py +7 -0
  21. kaizeval/gateway/core.py +406 -0
  22. kaizeval/gateway/fake.py +235 -0
  23. kaizeval/gateway/prompts.py +176 -0
  24. kaizeval/gateway/schemas.py +125 -0
  25. kaizeval/ground_truth.py +122 -0
  26. kaizeval/handlers.py +145 -0
  27. kaizeval/jobs.py +253 -0
  28. kaizeval/mcp_server.py +113 -0
  29. kaizeval/migrations/env.py +29 -0
  30. kaizeval/migrations/script.py.mako +22 -0
  31. kaizeval/migrations/versions/0001_initial_schema.py +632 -0
  32. kaizeval/models.py +395 -0
  33. kaizeval/netguard.py +108 -0
  34. kaizeval/project.py +701 -0
  35. kaizeval/py.typed +0 -0
  36. kaizeval/sample_agent/__init__.py +5 -0
  37. kaizeval/sample_agent/app.py +193 -0
  38. kaizeval/sample_agent/kb/returns-and-refunds.md +23 -0
  39. kaizeval/sample_agent/kb/shipping-and-delivery.md +23 -0
  40. kaizeval/sample_agent/kb/warranty-and-repairs.md +23 -0
  41. kaizeval/security.py +109 -0
  42. kaizeval/services/__init__.py +1 -0
  43. kaizeval/services/datasets.py +521 -0
  44. kaizeval/services/diagnostics.py +238 -0
  45. kaizeval/services/evaluation.py +557 -0
  46. kaizeval/services/holdout.py +58 -0
  47. kaizeval/services/optimization.py +676 -0
  48. kaizeval/services/reporting.py +223 -0
  49. kaizeval/services/specs.py +150 -0
  50. kaizeval/services/stats.py +122 -0
  51. kaizeval/storage.py +115 -0
  52. kaizeval/textutil.py +186 -0
  53. kaizeval/webui/README.txt +2 -0
  54. kaizeval/webui/assets/index-BIG40jck.css +1 -0
  55. kaizeval/webui/assets/index-D8bsydB0.js +72 -0
  56. kaizeval/webui/favicon.svg +1 -0
  57. kaizeval/webui/index.html +15 -0
  58. kaizeval-0.2.0.dist-info/METADATA +427 -0
  59. kaizeval-0.2.0.dist-info/RECORD +63 -0
  60. kaizeval-0.2.0.dist-info/WHEEL +4 -0
  61. kaizeval-0.2.0.dist-info/entry_points.txt +2 -0
  62. kaizeval-0.2.0.dist-info/licenses/LICENSE +202 -0
  63. kaizeval-0.2.0.dist-info/licenses/NOTICE +16 -0
@@ -0,0 +1,313 @@
1
+ """Knowledge documents, specifications and test datasets."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any, Literal
6
+
7
+ from fastapi import APIRouter, Depends, File, Form, Request, UploadFile
8
+ from pydantic import BaseModel, Field
9
+ from sqlalchemy import select
10
+
11
+ from .. import audit, jobs
12
+ from ..config import get_settings
13
+ from ..documents import ingest
14
+ from ..models import Agent, Dataset, Document, EvidenceChunk, Specification, TestCase, iso
15
+ from ..services import datasets as dataset_service
16
+ from ..services import specs as spec_service
17
+ from ..services.datasets import HoldoutAccess
18
+ from .deps import Ctx, fail, idempotency_key, job_dict, page, scoped, ser, workspace_ctx
19
+
20
+ router = APIRouter(prefix="/workspaces/{workspace_id}")
21
+
22
+
23
+ async def _read(file: UploadFile) -> bytes:
24
+ limit = get_settings().max_upload_mb * 1024 * 1024
25
+ data = await file.read(limit + 1)
26
+ if len(data) > limit:
27
+ raise fail(413, "FILE_TOO_LARGE", f"files are limited to {get_settings().max_upload_mb} MB")
28
+ return data
29
+
30
+
31
+ # --- documents ----------------------------------------------------------------------------
32
+
33
+
34
+ @router.post("/documents", status_code=201)
35
+ async def upload_document(file: UploadFile = File(...), kind: Literal["kb", "requirements"] = Form("kb"),
36
+ ctx: Ctx = Depends(workspace_ctx("editor"))) -> dict[str, Any]:
37
+ data = await _read(file)
38
+ document = ingest(ctx.db, workspace_id=ctx.ws, filename=file.filename or "document", data=data, kind=kind,
39
+ content_type=file.content_type or "")
40
+ audit.record(ctx.db, workspace_id=ctx.ws, user_id=ctx.user.id, action="document.uploaded", subject_type="document",
41
+ subject_id=document.id, detail={"filename": document.filename, "kind": kind, "version": document.version,
42
+ "sha256": document.sha256, "status": document.status})
43
+ ctx.db.commit()
44
+ flagged = ctx.db.scalars(select(EvidenceChunk).where(EvidenceChunk.document_id == document.id)).all()
45
+ return ser(document, exclude={"storage_key"},
46
+ extra={"flagged_chunks": sum(1 for c in flagged if c.flags)})
47
+
48
+
49
+ @router.get("/documents")
50
+ def list_documents(kind: str | None = None, include_superseded: bool = False, limit: int = 100, offset: int = 0,
51
+ ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
52
+ query = select(Document).where(Document.workspace_id == ctx.ws).order_by(Document.filename, Document.version.desc())
53
+ if kind:
54
+ query = query.where(Document.kind == kind)
55
+ if not include_superseded:
56
+ query = query.where(Document.status != "superseded")
57
+ return page(ctx.db, query, limit=limit, offset=offset, transform=lambda d: ser(d, exclude={"storage_key"}))
58
+
59
+
60
+ @router.get("/documents/{document_id}/chunks")
61
+ def document_chunks(document_id: str, limit: int = 100, offset: int = 0, ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
62
+ document = scoped(ctx, Document, document_id)
63
+ query = select(EvidenceChunk).where(EvidenceChunk.document_id == document.id).order_by(EvidenceChunk.ordinal)
64
+ result = page(ctx.db, query, limit=limit, offset=offset)
65
+ result["document"] = ser(document, exclude={"storage_key"})
66
+ return result
67
+
68
+
69
+ @router.get("/evidence/{chunk_id}")
70
+ def evidence(chunk_id: str, ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
71
+ chunk = scoped(ctx, EvidenceChunk, chunk_id)
72
+ document = ctx.db.get(Document, chunk.document_id)
73
+ return ser(chunk, extra={"filename": document.filename if document else None,
74
+ "document_version": document.version if document else None,
75
+ "sha256": document.sha256 if document else None})
76
+
77
+
78
+ # --- specifications -----------------------------------------------------------------------
79
+
80
+
81
+ class SpecBuild(BaseModel):
82
+ agent_id: str | None = None
83
+ requirements_text: str | None = Field(default=None, max_length=50000)
84
+ requirement_document_ids: list[str] = Field(default_factory=list)
85
+ infer: bool = True
86
+
87
+
88
+ class SpecEdit(BaseModel):
89
+ content: dict[str, Any]
90
+ approve: bool = False
91
+
92
+
93
+ class SpecReview(BaseModel):
94
+ decision: Literal["approve", "skip"]
95
+ note: str | None = Field(default=None, max_length=2000)
96
+
97
+
98
+ @router.post("/specifications:build", status_code=202)
99
+ def build_spec(body: SpecBuild, request: Request, ctx: Ctx = Depends(workspace_ctx("editor"))) -> dict[str, Any]:
100
+ if body.agent_id:
101
+ scoped(ctx, Agent, body.agent_id)
102
+ for document_id in body.requirement_document_ids:
103
+ scoped(ctx, Document, document_id)
104
+ job = jobs.enqueue(ctx.db, workspace_id=ctx.ws, kind="specification.build", payload=body.model_dump(),
105
+ created_by=ctx.user.id, idempotency_key=idempotency_key(request))
106
+ return job_dict(job)
107
+
108
+
109
+ @router.get("/specifications")
110
+ def list_specs(limit: int = 50, offset: int = 0, ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
111
+ query = select(Specification).where(Specification.workspace_id == ctx.ws).order_by(Specification.created_at.desc())
112
+ return page(ctx.db, query, limit=limit, offset=offset)
113
+
114
+
115
+ @router.get("/specifications/{spec_id}")
116
+ def get_spec(spec_id: str, ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
117
+ spec = scoped(ctx, Specification, spec_id)
118
+ versions = ctx.db.scalars(select(Specification).where(Specification.lineage_id == spec.lineage_id)
119
+ .order_by(Specification.version))
120
+ return ser(spec, extra={"versions": [{"id": v.id, "version": v.version, "status": v.status, "source": v.source,
121
+ "created_at": iso(v.created_at), "created_by": v.created_by}
122
+ for v in versions]})
123
+
124
+
125
+ @router.patch("/specifications/{spec_id}")
126
+ def edit_spec(spec_id: str, body: SpecEdit, ctx: Ctx = Depends(workspace_ctx("reviewer"))) -> dict[str, Any]:
127
+ spec = scoped(ctx, Specification, spec_id)
128
+ new = spec_service.edit_specification(ctx.db, spec, content=body.content, user_id=ctx.user.id, approve=body.approve)
129
+ ctx.db.commit()
130
+ return ser(new)
131
+
132
+
133
+ @router.post("/specifications/{spec_id}/review")
134
+ def review_spec(spec_id: str, body: SpecReview, ctx: Ctx = Depends(workspace_ctx("reviewer"))) -> dict[str, Any]:
135
+ spec = scoped(ctx, Specification, spec_id)
136
+ spec_service.review_specification(ctx.db, spec, decision=body.decision, user_id=ctx.user.id, note=body.note)
137
+ ctx.db.commit()
138
+ return ser(spec)
139
+
140
+
141
+ # --- datasets -----------------------------------------------------------------------------
142
+
143
+
144
+ class DatasetGenerate(BaseModel):
145
+ name: str = Field(min_length=1, max_length=200)
146
+ agent_id: str | None = None
147
+ specification_id: str | None = None
148
+ n_cases: int = Field(default=20, ge=4, le=500)
149
+ testers: int = Field(default=1, ge=1, le=8) # independent test-writing agents
150
+ keep_unverified: bool = False # keep generated cases whose reference could not be linked to evidence
151
+ holdout_fraction: float = Field(default=0.3, ge=0.0, le=0.6)
152
+ seed: int = 0
153
+ categories: list[Literal["knowledge", "citation", "edge_case"]] = ["knowledge", "citation", "edge_case"]
154
+ include_multi_turn: bool = True
155
+ unanswerable_share: float = Field(default=0.2, ge=0.0, le=0.5)
156
+ languages: list[str] = Field(default_factory=list)
157
+ document_ids: list[str] | None = None
158
+ limits: dict[str, float] = Field(default_factory=dict)
159
+
160
+
161
+ class CaseUpdate(BaseModel):
162
+ expected: dict[str, Any] | None = None
163
+ answer_status: Literal["approved", "needs_human_review", "unverified"] | None = None
164
+ severity: Literal["low", "normal", "high", "critical"] | None = None
165
+
166
+
167
+ class RotateBody(BaseModel):
168
+ n_cases: int | None = Field(default=None, ge=4, le=500)
169
+
170
+
171
+ def _dataset_dict(ctx: Ctx, dataset: Dataset) -> dict[str, Any]:
172
+ return ser(dataset, extra={"counts": dataset_service.split_counts(ctx.db, dataset.id)})
173
+
174
+
175
+ def _case_dict(case: TestCase) -> dict[str, Any]:
176
+ return ser(case)
177
+
178
+
179
+ @router.post("/datasets:generate", status_code=202)
180
+ def generate_dataset(body: DatasetGenerate, request: Request, ctx: Ctx = Depends(workspace_ctx("editor"))) -> dict[str, Any]:
181
+ """Generate evidence-linked cases and split them into regression and sealed holdout, before any optimization."""
182
+ key = idempotency_key(request)
183
+ if key:
184
+ existing = ctx.db.scalar(select(jobs.Job).where(jobs.Job.workspace_id == ctx.ws, jobs.Job.idempotency_key == key))
185
+ if existing is not None:
186
+ return {"job": job_dict(existing), "dataset": _dataset_dict(ctx, scoped(ctx, Dataset, existing.payload["dataset_id"]))}
187
+ if body.agent_id:
188
+ scoped(ctx, Agent, body.agent_id)
189
+ if body.specification_id:
190
+ scoped(ctx, Specification, body.specification_id)
191
+ dataset = Dataset(workspace_id=ctx.ws, name=body.name.strip(), agent_id=body.agent_id,
192
+ specification_id=body.specification_id, seed=body.seed, holdout_fraction=body.holdout_fraction)
193
+ ctx.db.add(dataset)
194
+ ctx.db.flush()
195
+ params = body.model_dump(exclude={"name", "agent_id", "specification_id", "seed", "holdout_fraction"})
196
+ job = jobs.enqueue(ctx.db, workspace_id=ctx.ws, kind="dataset.generate",
197
+ payload={"dataset_id": dataset.id, "params": params}, created_by=ctx.user.id, idempotency_key=key)
198
+ ctx.db.expire_all()
199
+ return {"job": job_dict(job), "dataset": _dataset_dict(ctx, scoped(ctx, Dataset, dataset.id))}
200
+
201
+
202
+ @router.post("/datasets:import", status_code=201)
203
+ async def import_dataset(file: UploadFile = File(...), name: str | None = Form(None), dataset_id: str | None = Form(None),
204
+ approved: bool = Form(True), holdout_fraction: float = Form(0.0),
205
+ ctx: Ctx = Depends(workspace_ctx("editor"))) -> dict[str, Any]:
206
+ """Import business-owned Q&A or real conversations (.csv, .jsonl, .json)."""
207
+ data = await _read(file)
208
+ try:
209
+ rows = dataset_service.parse_import(file.filename or "", data)
210
+ except (dataset_service.DatasetError, ValueError) as exc:
211
+ raise fail(422, "INVALID_IMPORT", str(exc)) from exc
212
+ if dataset_id:
213
+ dataset = scoped(ctx, Dataset, dataset_id)
214
+ else:
215
+ dataset = Dataset(workspace_id=ctx.ws, name=(name or file.filename or "Imported cases").strip(), status="ready")
216
+ ctx.db.add(dataset)
217
+ ctx.db.flush()
218
+ try:
219
+ result = dataset_service.import_cases(ctx.db, dataset, rows, approved=approved, user_id=ctx.user.id,
220
+ holdout_fraction=max(0.0, min(0.6, holdout_fraction)),
221
+ source_name=file.filename or "upload")
222
+ except dataset_service.DatasetError as exc:
223
+ ctx.db.rollback()
224
+ raise fail(422, "INVALID_IMPORT", str(exc)) from exc
225
+ ctx.db.commit()
226
+ return {"dataset": _dataset_dict(ctx, dataset), **result}
227
+
228
+
229
+ @router.get("/datasets")
230
+ def list_datasets(limit: int = 50, offset: int = 0, ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
231
+ query = select(Dataset).where(Dataset.workspace_id == ctx.ws).order_by(Dataset.created_at.desc())
232
+ return page(ctx.db, query, limit=limit, offset=offset, transform=lambda d: _dataset_dict(ctx, d))
233
+
234
+
235
+ @router.get("/datasets/{dataset_id}")
236
+ def get_dataset(dataset_id: str, ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
237
+ return _dataset_dict(ctx, scoped(ctx, Dataset, dataset_id))
238
+
239
+
240
+ @router.get("/datasets/{dataset_id}/coverage")
241
+ def dataset_coverage(dataset_id: str, ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
242
+ return dataset_service.coverage(ctx.db, scoped(ctx, Dataset, dataset_id))
243
+
244
+
245
+ @router.get("/datasets/{dataset_id}/cases")
246
+ def list_cases(dataset_id: str, split: Literal["regression", "holdout"] = "regression", acknowledge: bool = False,
247
+ limit: int = 50, offset: int = 0, ctx: Ctx = Depends(workspace_ctx())) -> dict[str, Any]:
248
+ dataset = scoped(ctx, Dataset, dataset_id)
249
+ if split == "holdout":
250
+ # Holdout items are sealed. A reviewer may open them deliberately; doing so is audited and
251
+ # marks the holdout as exposed, so it must be rotated before it can gate further tuning.
252
+ ctx.require("reviewer")
253
+ if not acknowledge:
254
+ raise fail(403, "HOLDOUT_SEALED", "holdout items are sealed. Pass acknowledge=true to open them; "
255
+ "this is audited and the holdout will need rotating before it is used again.")
256
+ dataset_service.holdout_cases(ctx.db, dataset.id, HoldoutAccess("human_review", ctx.user.id))
257
+ dataset.holdout_exposed = True
258
+ audit.record(ctx.db, workspace_id=ctx.ws, user_id=ctx.user.id, action="holdout.viewed",
259
+ subject_type="dataset", subject_id=dataset.id, detail={"generation": dataset.holdout_generation})
260
+ ctx.db.commit()
261
+ query = (select(TestCase).where(TestCase.dataset_id == dataset.id, TestCase.split == split)
262
+ .order_by(TestCase.created_at, TestCase.id))
263
+ return page(ctx.db, query, limit=limit, offset=offset, transform=_case_dict)
264
+
265
+
266
+ @router.patch("/datasets/{dataset_id}/cases/{case_id}")
267
+ def update_case(dataset_id: str, case_id: str, body: CaseUpdate, ctx: Ctx = Depends(workspace_ctx("reviewer"))) -> dict[str, Any]:
268
+ """Human review of a reference: approve it, correct it, or flag it. Approved references outrank drafts."""
269
+ dataset = scoped(ctx, Dataset, dataset_id)
270
+ case = scoped(ctx, TestCase, case_id)
271
+ if case.dataset_id != dataset.id:
272
+ raise fail(404, "NOT_FOUND", "case not found in this dataset")
273
+ if case.split != "regression":
274
+ raise fail(403, "HOLDOUT_SEALED", "only regression cases can be edited")
275
+ if body.expected is not None:
276
+ case.expected = {**(case.expected or {}), **body.expected}
277
+ if body.answer_status:
278
+ case.answer_status = body.answer_status
279
+ case.provenance = {**(case.provenance or {}), "reviewed_by": ctx.user.id}
280
+ if body.severity:
281
+ case.severity = body.severity
282
+ case.weight = 2.0 if body.severity == "critical" else 1.0
283
+ dataset.version += 1
284
+ audit.record(ctx.db, workspace_id=ctx.ws, user_id=ctx.user.id, action="case.reviewed", subject_type="test_case",
285
+ subject_id=case.id, detail={"answer_status": case.answer_status, "severity": case.severity})
286
+ ctx.db.commit()
287
+ return _case_dict(case)
288
+
289
+
290
+ @router.post("/datasets/{dataset_id}/holdout:rotate", status_code=202)
291
+ def rotate(dataset_id: str, body: RotateBody, request: Request, ctx: Ctx = Depends(workspace_ctx("editor"))) -> dict[str, Any]:
292
+ dataset = scoped(ctx, Dataset, dataset_id)
293
+ job = jobs.enqueue(ctx.db, workspace_id=ctx.ws, kind="dataset.rotate_holdout",
294
+ payload={"dataset_id": dataset.id, "params": body.model_dump(exclude_none=True)},
295
+ created_by=ctx.user.id, idempotency_key=idempotency_key(request))
296
+ return job_dict(job)
297
+
298
+
299
+ @router.post("/datasets/{dataset_id}/stale:retire")
300
+ def retire_stale(dataset_id: str, ctx: Ctx = Depends(workspace_ctx("editor"))) -> dict[str, Any]:
301
+ """Retire regression cases whose evidence came from a document version that has been replaced."""
302
+ dataset = scoped(ctx, Dataset, dataset_id)
303
+ retired = 0
304
+ for case in ctx.db.scalars(select(TestCase).where(TestCase.dataset_id == dataset.id, TestCase.stale.is_(True),
305
+ TestCase.split == "regression")):
306
+ case.split = "retired"
307
+ retired += 1
308
+ if retired:
309
+ dataset.version += 1
310
+ audit.record(ctx.db, workspace_id=ctx.ws, user_id=ctx.user.id, action="dataset.stale_retired",
311
+ subject_type="dataset", subject_id=dataset.id, detail={"retired": retired})
312
+ ctx.db.commit()
313
+ return {"retired": retired, "dataset": _dataset_dict(ctx, dataset)}