nufi-python 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nufi/__init__.py +52 -0
- nufi/auth.py +364 -0
- nufi/cli/__init__.py +0 -0
- nufi/cli/commands/__init__.py +1 -0
- nufi/cli/commands/auth.py +113 -0
- nufi/cli/commands/common.py +198 -0
- nufi/cli/commands/compiles.py +507 -0
- nufi/cli/commands/custom_evaluation_templates.py +206 -0
- nufi/cli/commands/datasets.py +697 -0
- nufi/cli/commands/evaluations.py +854 -0
- nufi/cli/commands/leaderboards.py +763 -0
- nufi/cli/commands/models.py +1347 -0
- nufi/cli/commands/monitoring.py +350 -0
- nufi/cli/commands/profile.py +123 -0
- nufi/cli/commands/projects.py +308 -0
- nufi/cli/commands/servings.py +306 -0
- nufi/cli/main.py +66 -0
- nufi/client.py +103 -0
- nufi/config.py +281 -0
- nufi/evaluator.py +559 -0
- nufi/exceptions.py +58 -0
- nufi/http_utils.py +51 -0
- nufi/models/__init__.py +185 -0
- nufi/models/compiles.py +79 -0
- nufi/models/custom_evaluation_templates.py +111 -0
- nufi/models/datasets.py +94 -0
- nufi/models/evaluations.py +253 -0
- nufi/models/leaderboards.py +178 -0
- nufi/models/monitoring.py +179 -0
- nufi/models/pagination.py +33 -0
- nufi/models/projects.py +32 -0
- nufi/models/registry.py +138 -0
- nufi/models/servings.py +27 -0
- nufi/py.typed +1 -0
- nufi/resources/__init__.py +34 -0
- nufi/resources/base.py +190 -0
- nufi/resources/compiles.py +193 -0
- nufi/resources/custom_evaluation_templates.py +126 -0
- nufi/resources/datasets.py +786 -0
- nufi/resources/evaluations.py +428 -0
- nufi/resources/leaderboards.py +413 -0
- nufi/resources/models.py +726 -0
- nufi/resources/monitoring.py +82 -0
- nufi/resources/projects.py +114 -0
- nufi/resources/servings.py +80 -0
- nufi/transport.py +248 -0
- nufi_python-1.0.0.dist-info/METADATA +228 -0
- nufi_python-1.0.0.dist-info/RECORD +52 -0
- nufi_python-1.0.0.dist-info/WHEEL +5 -0
- nufi_python-1.0.0.dist-info/entry_points.txt +2 -0
- nufi_python-1.0.0.dist-info/licenses/LICENSE +21 -0
- nufi_python-1.0.0.dist-info/top_level.txt +1 -0
nufi/evaluator.py
ADDED
|
@@ -0,0 +1,559 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import math
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
from collections.abc import Callable, Iterator, Mapping
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
from urllib.parse import unquote, urlparse
|
|
12
|
+
|
|
13
|
+
import httpx
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
SCHEMA_VERSION = "nufi.evaluation.runner.v1"
|
|
17
|
+
SUBMISSION_COMPLETE_MARKER = "NUFI_EVAL_RESULT_SUBMITTED"
|
|
18
|
+
MAX_SPEC_BYTES = 256 * 1024
|
|
19
|
+
MAX_RESULT_BYTES = 64 * 1024
|
|
20
|
+
MAX_DIAGNOSTIC_BYTES = 48 * 1024
|
|
21
|
+
MAX_DIAGNOSTIC_ITEMS = 20
|
|
22
|
+
MAX_DIAGNOSTIC_VALUE_BYTES = 2 * 1024
|
|
23
|
+
MAX_TERMINATION_BYTES = 2 * 1024
|
|
24
|
+
ROLE_NAME = re.compile(r"^[a-z][a-z0-9_-]{0,62}$")
|
|
25
|
+
ROLE_TYPES = {"string", "integer", "number", "boolean", "array", "object"}
|
|
26
|
+
UNAVAILABLE_REASONS = {"runner_omitted", "upload_not_completed", "result_too_large"}
|
|
27
|
+
DEFAULT_DIAGNOSTICS = {
|
|
28
|
+
"status": "unavailable",
|
|
29
|
+
"unavailableReason": "runner_omitted",
|
|
30
|
+
"selectionPolicy": "failed_then_first_by_source_index",
|
|
31
|
+
"complete": False,
|
|
32
|
+
"omittedRowCount": 0,
|
|
33
|
+
"valuesTruncated": False,
|
|
34
|
+
"items": [],
|
|
35
|
+
}
|
|
36
|
+
FAILURE_MESSAGES = {
|
|
37
|
+
"invalid_runner_spec": "Runner spec is invalid.",
|
|
38
|
+
"invalid_evaluator_config": "Evaluator config is invalid.",
|
|
39
|
+
"dataset_read_failed": "Dataset read failed.",
|
|
40
|
+
"target_request_failed": "Target request failed.",
|
|
41
|
+
"invalid_result": "Evaluator result is invalid.",
|
|
42
|
+
"result_submission_failed": "Result submission failed.",
|
|
43
|
+
"evaluator_failed": "Evaluator failed.",
|
|
44
|
+
}
|
|
45
|
+
_TERMINATION_LOG_PATH = Path("/dev/termination-log")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class EvaluationConfigError(ValueError):
|
|
49
|
+
"""Signal that evaluator-specific configuration is invalid."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class _DatasetError(Exception):
|
|
53
|
+
pass
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class _TargetError(Exception):
|
|
57
|
+
pass
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(frozen=True)
|
|
61
|
+
class EvaluationResult:
|
|
62
|
+
metrics: Mapping[str, int | float]
|
|
63
|
+
diagnostic_samples: Mapping[str, Any] | None = None
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class _Target:
|
|
67
|
+
def __init__(self, spec: Mapping[str, Any]):
|
|
68
|
+
self.base_url = spec["baseUrl"]
|
|
69
|
+
self.model = spec["model"]
|
|
70
|
+
|
|
71
|
+
def request(self, method: str, path: str, **kwargs: Any) -> httpx.Response:
|
|
72
|
+
parsed = urlparse(path)
|
|
73
|
+
segments = unquote(parsed.path).split("/")
|
|
74
|
+
if (
|
|
75
|
+
not method.strip()
|
|
76
|
+
or parsed.scheme
|
|
77
|
+
or parsed.netloc
|
|
78
|
+
or parsed.query
|
|
79
|
+
or parsed.fragment
|
|
80
|
+
or path.startswith("/")
|
|
81
|
+
or "\\" in path
|
|
82
|
+
or any(segment in {".", ".."} for segment in segments)
|
|
83
|
+
or bool(kwargs.get("follow_redirects"))
|
|
84
|
+
):
|
|
85
|
+
raise ValueError("target request path must stay under the injected /v1 URL")
|
|
86
|
+
url = f"{self.base_url.rstrip('/')}/{path}"
|
|
87
|
+
try:
|
|
88
|
+
response = _target_request(method, url, **kwargs)
|
|
89
|
+
response.raise_for_status()
|
|
90
|
+
return response
|
|
91
|
+
except (httpx.HTTPError, OSError) as exc:
|
|
92
|
+
raise _TargetError from exc
|
|
93
|
+
|
|
94
|
+
def chat(self, prompt: str) -> str:
|
|
95
|
+
try:
|
|
96
|
+
payload = self.request(
|
|
97
|
+
"POST",
|
|
98
|
+
"chat/completions",
|
|
99
|
+
json={
|
|
100
|
+
"model": self.model,
|
|
101
|
+
"messages": [{"role": "user", "content": prompt}],
|
|
102
|
+
},
|
|
103
|
+
).json()
|
|
104
|
+
content = payload["choices"][0]["message"]["content"]
|
|
105
|
+
if not isinstance(content, str):
|
|
106
|
+
raise ValueError
|
|
107
|
+
return content
|
|
108
|
+
except (KeyError, IndexError, TypeError, ValueError) as exc:
|
|
109
|
+
raise _TargetError from exc
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class EvaluationContext:
|
|
113
|
+
def __init__(self, spec: Mapping[str, Any]):
|
|
114
|
+
self._dataset = spec["dataset"]
|
|
115
|
+
self.config = spec["evaluator"]["config"]
|
|
116
|
+
self.target = _Target(spec["target"])
|
|
117
|
+
|
|
118
|
+
def rows(self) -> Iterator[dict[str, Any]]:
|
|
119
|
+
try:
|
|
120
|
+
root = Path(self._dataset["path"]).resolve(strict=True)
|
|
121
|
+
if not root.is_dir():
|
|
122
|
+
raise _DatasetError
|
|
123
|
+
files = []
|
|
124
|
+
for candidate in sorted(root.iterdir(), key=lambda path: path.name):
|
|
125
|
+
if candidate.suffix.lower() != ".parquet":
|
|
126
|
+
continue
|
|
127
|
+
resolved = candidate.resolve(strict=True)
|
|
128
|
+
try:
|
|
129
|
+
resolved.relative_to(root)
|
|
130
|
+
except ValueError as exc:
|
|
131
|
+
raise _DatasetError from exc
|
|
132
|
+
files.append(resolved)
|
|
133
|
+
if not files:
|
|
134
|
+
raise _DatasetError
|
|
135
|
+
|
|
136
|
+
import pyarrow.parquet as pq
|
|
137
|
+
|
|
138
|
+
found = False
|
|
139
|
+
for path in files:
|
|
140
|
+
for batch in pq.ParquetFile(path).iter_batches():
|
|
141
|
+
for row in batch.to_pylist():
|
|
142
|
+
if not isinstance(row, dict):
|
|
143
|
+
raise _DatasetError
|
|
144
|
+
found = True
|
|
145
|
+
yield row
|
|
146
|
+
if not found:
|
|
147
|
+
raise _DatasetError
|
|
148
|
+
except _DatasetError:
|
|
149
|
+
raise
|
|
150
|
+
except (ImportError, OSError, ValueError) as exc:
|
|
151
|
+
raise _DatasetError from exc
|
|
152
|
+
|
|
153
|
+
def value(self, row: Mapping[str, Any], role: str) -> Any:
|
|
154
|
+
binding = self._dataset["roles"][role]
|
|
155
|
+
field = binding["field"]
|
|
156
|
+
if field is None:
|
|
157
|
+
return None
|
|
158
|
+
value = row.get(field)
|
|
159
|
+
if binding["required"] and value is None:
|
|
160
|
+
raise _DatasetError
|
|
161
|
+
return value
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def run(handler: Callable[[EvaluationContext], EvaluationResult]) -> None:
|
|
165
|
+
try:
|
|
166
|
+
spec = _load_spec(os.environ["NUFI_EVAL_SPEC_PATH"])
|
|
167
|
+
except (KeyError, OSError, ValueError):
|
|
168
|
+
_fail("invalid_runner_spec")
|
|
169
|
+
|
|
170
|
+
try:
|
|
171
|
+
result = handler(EvaluationContext(spec))
|
|
172
|
+
except EvaluationConfigError:
|
|
173
|
+
_fail("invalid_evaluator_config")
|
|
174
|
+
except _DatasetError:
|
|
175
|
+
_fail("dataset_read_failed")
|
|
176
|
+
except _TargetError:
|
|
177
|
+
_fail("target_request_failed")
|
|
178
|
+
except Exception:
|
|
179
|
+
_fail("evaluator_failed")
|
|
180
|
+
|
|
181
|
+
try:
|
|
182
|
+
envelope = _result_envelope(spec, result)
|
|
183
|
+
except (TypeError, ValueError):
|
|
184
|
+
_fail("invalid_result")
|
|
185
|
+
|
|
186
|
+
try:
|
|
187
|
+
token = Path(spec["result"]["tokenFile"]).read_text(encoding="utf-8").strip()
|
|
188
|
+
if not token or any(character.isspace() for character in token):
|
|
189
|
+
raise ValueError
|
|
190
|
+
body = json.dumps(
|
|
191
|
+
envelope, ensure_ascii=False, separators=(",", ":"), allow_nan=False
|
|
192
|
+
).encode()
|
|
193
|
+
_submit_result(spec["result"]["url"], token, body)
|
|
194
|
+
except (OSError, ValueError, httpx.HTTPError):
|
|
195
|
+
_fail("result_submission_failed")
|
|
196
|
+
|
|
197
|
+
print(SUBMISSION_COMPLETE_MARKER)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _load_spec(path: str | Path) -> dict[str, Any]:
|
|
201
|
+
raw = Path(path).read_bytes()
|
|
202
|
+
if len(raw) > MAX_SPEC_BYTES:
|
|
203
|
+
raise ValueError("runner spec exceeds 256 KiB")
|
|
204
|
+
try:
|
|
205
|
+
spec = json.loads(raw)
|
|
206
|
+
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
207
|
+
raise ValueError("runner spec must be valid UTF-8 JSON") from exc
|
|
208
|
+
_validate_spec(spec)
|
|
209
|
+
return spec
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _validate_spec(spec: Any) -> None:
|
|
213
|
+
if not isinstance(spec, dict):
|
|
214
|
+
raise ValueError("runner spec must be an object")
|
|
215
|
+
if len(_json_bytes(spec)) > MAX_SPEC_BYTES:
|
|
216
|
+
raise ValueError("runner spec exceeds 256 KiB")
|
|
217
|
+
_exact(
|
|
218
|
+
spec,
|
|
219
|
+
{"schemaVersion", "run", "dataset", "target", "evaluator", "result"},
|
|
220
|
+
"runner spec",
|
|
221
|
+
)
|
|
222
|
+
if spec["schemaVersion"] != SCHEMA_VERSION:
|
|
223
|
+
raise ValueError(f"schemaVersion must be {SCHEMA_VERSION}")
|
|
224
|
+
|
|
225
|
+
run_spec = _object(spec, "run")
|
|
226
|
+
_exact(run_spec, {"id"}, "run")
|
|
227
|
+
_string(run_spec, "id", "run.id")
|
|
228
|
+
|
|
229
|
+
dataset = _object(spec, "dataset")
|
|
230
|
+
_exact(dataset, {"path", "format", "schemaHash", "schema", "roles"}, "dataset")
|
|
231
|
+
if not Path(_string(dataset, "path", "dataset.path")).is_absolute():
|
|
232
|
+
raise ValueError("dataset.path must be absolute")
|
|
233
|
+
if dataset["format"] != "parquet":
|
|
234
|
+
raise ValueError("dataset.format must be parquet")
|
|
235
|
+
_string(dataset, "schemaHash", "dataset.schemaHash")
|
|
236
|
+
_json_object(dataset, "schema", "dataset.schema")
|
|
237
|
+
roles = _json_object(dataset, "roles", "dataset.roles")
|
|
238
|
+
if not 1 <= len(roles) <= 32:
|
|
239
|
+
raise ValueError("dataset.roles must contain 1 to 32 roles")
|
|
240
|
+
for role, binding in roles.items():
|
|
241
|
+
if ROLE_NAME.fullmatch(role) is None:
|
|
242
|
+
raise ValueError(f"invalid role name {role!r}")
|
|
243
|
+
if not isinstance(binding, dict):
|
|
244
|
+
raise ValueError(f"dataset.roles.{role} must be an object")
|
|
245
|
+
path = f"dataset.roles.{role}"
|
|
246
|
+
_exact(binding, {"field", "type", "required"}, path)
|
|
247
|
+
field = binding["field"]
|
|
248
|
+
if field is not None and (
|
|
249
|
+
not isinstance(field, str) or not field or field != field.strip()
|
|
250
|
+
):
|
|
251
|
+
raise ValueError(f"{path}.field must be a non-empty string or null")
|
|
252
|
+
if not isinstance(binding["type"], str) or binding["type"] not in ROLE_TYPES:
|
|
253
|
+
raise ValueError(f"{path}.type must be a canonical Dataset type")
|
|
254
|
+
if not isinstance(binding["required"], bool):
|
|
255
|
+
raise ValueError(f"{path}.required must be a boolean")
|
|
256
|
+
if field is None and binding["required"]:
|
|
257
|
+
raise ValueError(f"{path}.field may be null only for an optional role")
|
|
258
|
+
|
|
259
|
+
target = _object(spec, "target")
|
|
260
|
+
_exact(target, {"protocol", "baseUrl", "model"}, "target")
|
|
261
|
+
if target["protocol"] != "openai-compatible":
|
|
262
|
+
raise ValueError("target.protocol must be openai-compatible")
|
|
263
|
+
_http_url(_string(target, "baseUrl", "target.baseUrl"), "target.baseUrl", v1=True)
|
|
264
|
+
_string(target, "model", "target.model")
|
|
265
|
+
|
|
266
|
+
evaluator = _object(spec, "evaluator")
|
|
267
|
+
_exact(evaluator, {"implementation", "config", "declaredMetrics"}, "evaluator")
|
|
268
|
+
if evaluator["implementation"] not in {"lm_eval", "custom"}:
|
|
269
|
+
raise ValueError("evaluator.implementation must be lm_eval or custom")
|
|
270
|
+
_json_object(evaluator, "config", "evaluator.config")
|
|
271
|
+
metrics = evaluator["declaredMetrics"]
|
|
272
|
+
if not isinstance(metrics, list) or not 1 <= len(metrics) <= 32:
|
|
273
|
+
raise ValueError("evaluator.declaredMetrics must contain 1 to 32 metrics")
|
|
274
|
+
seen: set[str] = set()
|
|
275
|
+
for index, metric in enumerate(metrics):
|
|
276
|
+
path = f"evaluator.declaredMetrics[{index}]"
|
|
277
|
+
if not isinstance(metric, dict):
|
|
278
|
+
raise ValueError(f"{path} must be an object")
|
|
279
|
+
_exact(
|
|
280
|
+
metric,
|
|
281
|
+
{"key", "higherIsBetter", "min", "max"},
|
|
282
|
+
path,
|
|
283
|
+
required={"key", "higherIsBetter"},
|
|
284
|
+
)
|
|
285
|
+
key = _string(metric, "key", f"{path}.key")
|
|
286
|
+
if ROLE_NAME.fullmatch(key) is None or key in seen:
|
|
287
|
+
raise ValueError(f"invalid or duplicate declared metric {key!r}")
|
|
288
|
+
seen.add(key)
|
|
289
|
+
if not isinstance(metric["higherIsBetter"], bool):
|
|
290
|
+
raise ValueError(f"{path}.higherIsBetter must be a boolean")
|
|
291
|
+
minimum = _number(metric["min"]) if "min" in metric else None
|
|
292
|
+
maximum = _number(metric["max"]) if "max" in metric else None
|
|
293
|
+
if minimum is not None and maximum is not None and minimum > maximum:
|
|
294
|
+
raise ValueError(f"{path}.min must not exceed max")
|
|
295
|
+
|
|
296
|
+
result = _object(spec, "result")
|
|
297
|
+
_exact(result, {"url", "tokenFile"}, "result")
|
|
298
|
+
_http_url(_string(result, "url", "result.url"), "result.url")
|
|
299
|
+
if not Path(_string(result, "tokenFile", "result.tokenFile")).is_absolute():
|
|
300
|
+
raise ValueError("result.tokenFile must be absolute")
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _result_envelope(
|
|
304
|
+
spec: Mapping[str, Any], result: EvaluationResult
|
|
305
|
+
) -> dict[str, Any]:
|
|
306
|
+
if not isinstance(result, EvaluationResult) or not isinstance(
|
|
307
|
+
result.metrics, Mapping
|
|
308
|
+
):
|
|
309
|
+
raise TypeError("handler must return EvaluationResult")
|
|
310
|
+
declared = {
|
|
311
|
+
metric["key"]: metric for metric in spec["evaluator"]["declaredMetrics"]
|
|
312
|
+
}
|
|
313
|
+
if set(result.metrics) != set(declared):
|
|
314
|
+
raise ValueError("result metrics must match declared metrics")
|
|
315
|
+
aggregates = []
|
|
316
|
+
for key, metric in declared.items():
|
|
317
|
+
value = _bounded_metric(result.metrics[key], metric)
|
|
318
|
+
aggregates.append(
|
|
319
|
+
{"key": key, "value": value, "higherIsBetter": metric["higherIsBetter"]}
|
|
320
|
+
)
|
|
321
|
+
diagnostics = _diagnostics(result.diagnostic_samples, declared)
|
|
322
|
+
envelope = {"aggregateMetrics": aggregates, "diagnosticSamples": diagnostics}
|
|
323
|
+
if len(_json_bytes(envelope)) > MAX_RESULT_BYTES:
|
|
324
|
+
raise ValueError("result exceeds 64 KiB")
|
|
325
|
+
return envelope
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _diagnostics(
|
|
329
|
+
raw: Mapping[str, Any] | None, declared: Mapping[str, Mapping[str, Any]]
|
|
330
|
+
) -> dict[str, Any]:
|
|
331
|
+
if raw is None:
|
|
332
|
+
return deepcopy_json(DEFAULT_DIAGNOSTICS)
|
|
333
|
+
if not isinstance(raw, Mapping):
|
|
334
|
+
raise ValueError("diagnostic_samples must be a mapping")
|
|
335
|
+
diagnostic = dict(raw)
|
|
336
|
+
diagnostic.setdefault("unavailableReason", None)
|
|
337
|
+
diagnostic.setdefault("selectionPolicy", "")
|
|
338
|
+
_exact(
|
|
339
|
+
diagnostic,
|
|
340
|
+
{
|
|
341
|
+
"status",
|
|
342
|
+
"unavailableReason",
|
|
343
|
+
"selectionPolicy",
|
|
344
|
+
"complete",
|
|
345
|
+
"omittedRowCount",
|
|
346
|
+
"valuesTruncated",
|
|
347
|
+
"items",
|
|
348
|
+
},
|
|
349
|
+
"diagnosticSamples",
|
|
350
|
+
)
|
|
351
|
+
if (
|
|
352
|
+
not isinstance(diagnostic["complete"], bool)
|
|
353
|
+
or not isinstance(diagnostic["valuesTruncated"], bool)
|
|
354
|
+
or not isinstance(diagnostic["omittedRowCount"], int)
|
|
355
|
+
or isinstance(diagnostic["omittedRowCount"], bool)
|
|
356
|
+
or diagnostic["omittedRowCount"] < 0
|
|
357
|
+
or not isinstance(diagnostic["items"], list)
|
|
358
|
+
or len(diagnostic["items"]) > MAX_DIAGNOSTIC_ITEMS
|
|
359
|
+
):
|
|
360
|
+
raise ValueError("invalid diagnosticSamples bounds")
|
|
361
|
+
if diagnostic["status"] == "available":
|
|
362
|
+
if (
|
|
363
|
+
diagnostic["unavailableReason"] is not None
|
|
364
|
+
or diagnostic["selectionPolicy"] != "failed_then_first_by_source_index"
|
|
365
|
+
):
|
|
366
|
+
raise ValueError("invalid available diagnosticSamples")
|
|
367
|
+
elif diagnostic["status"] == "unavailable":
|
|
368
|
+
if (
|
|
369
|
+
diagnostic["unavailableReason"] not in UNAVAILABLE_REASONS
|
|
370
|
+
or diagnostic["items"]
|
|
371
|
+
):
|
|
372
|
+
raise ValueError("invalid unavailable diagnosticSamples")
|
|
373
|
+
else:
|
|
374
|
+
raise ValueError("invalid diagnosticSamples status")
|
|
375
|
+
if diagnostic["complete"] and (
|
|
376
|
+
diagnostic["omittedRowCount"] or diagnostic["valuesTruncated"]
|
|
377
|
+
):
|
|
378
|
+
raise ValueError("complete diagnosticSamples cannot be truncated")
|
|
379
|
+
|
|
380
|
+
items = []
|
|
381
|
+
for raw_item in diagnostic["items"]:
|
|
382
|
+
if not isinstance(raw_item, Mapping):
|
|
383
|
+
raise ValueError("diagnostic item must be a mapping")
|
|
384
|
+
item = dict(raw_item)
|
|
385
|
+
_exact(
|
|
386
|
+
item,
|
|
387
|
+
{"sourceIndex", "rowId", "input", "output", "reference", "scores", "error"},
|
|
388
|
+
"diagnosticSamples.items[]",
|
|
389
|
+
required={"sourceIndex", "rowId", "scores"},
|
|
390
|
+
)
|
|
391
|
+
if (
|
|
392
|
+
not isinstance(item["sourceIndex"], int)
|
|
393
|
+
or isinstance(item["sourceIndex"], bool)
|
|
394
|
+
or item["sourceIndex"] < 0
|
|
395
|
+
or not isinstance(item["rowId"], str)
|
|
396
|
+
or not item["rowId"].strip()
|
|
397
|
+
or not isinstance(item["scores"], Mapping)
|
|
398
|
+
):
|
|
399
|
+
raise ValueError("invalid diagnostic item identity")
|
|
400
|
+
for name in ("input", "output", "reference", "error"):
|
|
401
|
+
value = item.get(name)
|
|
402
|
+
if value is not None and (
|
|
403
|
+
not isinstance(value, str)
|
|
404
|
+
or len(value.encode()) > MAX_DIAGNOSTIC_VALUE_BYTES
|
|
405
|
+
):
|
|
406
|
+
raise ValueError("diagnostic value exceeds bounds")
|
|
407
|
+
if not set(item["scores"]).issubset(declared):
|
|
408
|
+
raise ValueError("diagnostic score is not declared")
|
|
409
|
+
scores = {
|
|
410
|
+
key: _bounded_metric(value, declared[key])
|
|
411
|
+
for key, value in item["scores"].items()
|
|
412
|
+
}
|
|
413
|
+
item["scores"] = scores
|
|
414
|
+
items.append(item)
|
|
415
|
+
diagnostic["items"] = items
|
|
416
|
+
if len(_json_bytes(diagnostic)) > MAX_DIAGNOSTIC_BYTES:
|
|
417
|
+
raise ValueError("diagnosticSamples exceeds 48 KiB")
|
|
418
|
+
return diagnostic
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _bounded_metric(value: Any, declaration: Mapping[str, Any]) -> float:
|
|
422
|
+
number = _number(value)
|
|
423
|
+
if "min" in declaration and number < declaration["min"]:
|
|
424
|
+
raise ValueError("metric is below its declared minimum")
|
|
425
|
+
if "max" in declaration and number > declaration["max"]:
|
|
426
|
+
raise ValueError("metric exceeds its declared maximum")
|
|
427
|
+
return number
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def _submit_result(url: str, token: str, body: bytes) -> None:
|
|
431
|
+
headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
|
|
432
|
+
with _http_client() as client:
|
|
433
|
+
for attempt in range(2):
|
|
434
|
+
try:
|
|
435
|
+
response = client.post(url, content=body, headers=headers)
|
|
436
|
+
if response.status_code < 500 or attempt == 1:
|
|
437
|
+
response.raise_for_status()
|
|
438
|
+
return
|
|
439
|
+
except httpx.RequestError:
|
|
440
|
+
if attempt == 1:
|
|
441
|
+
raise
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def _target_request(method: str, url: str, **kwargs: Any) -> httpx.Response:
|
|
445
|
+
return httpx.request(method, url, timeout=30, **kwargs)
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _http_client() -> httpx.Client:
|
|
449
|
+
return httpx.Client(timeout=30)
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _fail(code: str) -> None:
|
|
453
|
+
payload = json.dumps(
|
|
454
|
+
{"code": code, "message": FAILURE_MESSAGES[code]}, separators=(",", ":")
|
|
455
|
+
).encode()
|
|
456
|
+
if len(payload) > MAX_TERMINATION_BYTES:
|
|
457
|
+
raise AssertionError("termination payload exceeds 2 KiB")
|
|
458
|
+
try:
|
|
459
|
+
_TERMINATION_LOG_PATH.write_bytes(payload)
|
|
460
|
+
except OSError:
|
|
461
|
+
pass
|
|
462
|
+
raise SystemExit(1)
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def _exact(
|
|
466
|
+
value: Mapping[str, Any],
|
|
467
|
+
allowed: set[str],
|
|
468
|
+
path: str,
|
|
469
|
+
*,
|
|
470
|
+
required: set[str] | None = None,
|
|
471
|
+
) -> None:
|
|
472
|
+
unknown = sorted(set(value) - allowed)
|
|
473
|
+
if unknown:
|
|
474
|
+
raise ValueError(f"{path}.{unknown[0]} is not allowed")
|
|
475
|
+
missing = sorted((allowed if required is None else required) - set(value))
|
|
476
|
+
if missing:
|
|
477
|
+
raise ValueError(f"{path}.{missing[0]} is required")
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def _object(value: Mapping[str, Any], key: str) -> dict[str, Any]:
|
|
481
|
+
item = value.get(key)
|
|
482
|
+
if not isinstance(item, dict):
|
|
483
|
+
raise ValueError(f"{key} must be an object")
|
|
484
|
+
return item
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _json_object(value: Mapping[str, Any], key: str, path: str) -> dict[str, Any]:
|
|
488
|
+
item = _object(value, key)
|
|
489
|
+
_validate_json(item, path)
|
|
490
|
+
return item
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def _validate_json(value: Any, path: str) -> None:
|
|
494
|
+
if isinstance(value, dict):
|
|
495
|
+
for key, item in value.items():
|
|
496
|
+
if not isinstance(key, str):
|
|
497
|
+
raise ValueError(f"{path} keys must be strings")
|
|
498
|
+
_validate_json(item, f"{path}.{key}")
|
|
499
|
+
elif isinstance(value, list):
|
|
500
|
+
for index, item in enumerate(value):
|
|
501
|
+
_validate_json(item, f"{path}[{index}]")
|
|
502
|
+
elif isinstance(value, float) and not math.isfinite(value):
|
|
503
|
+
raise ValueError(f"{path} numbers must be finite")
|
|
504
|
+
elif not isinstance(value, str | int | float | bool | type(None)):
|
|
505
|
+
raise ValueError(f"{path} must contain JSON values only")
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def _string(value: Mapping[str, Any], key: str, path: str) -> str:
|
|
509
|
+
item = value.get(key)
|
|
510
|
+
if not isinstance(item, str) or not item.strip():
|
|
511
|
+
raise ValueError(f"{path} must be a non-empty string")
|
|
512
|
+
if item != item.strip():
|
|
513
|
+
raise ValueError(f"{path} must not contain surrounding whitespace")
|
|
514
|
+
return item
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def _number(value: Any) -> float:
|
|
518
|
+
if not isinstance(value, int | float) or isinstance(value, bool):
|
|
519
|
+
raise ValueError("metric must be a finite number")
|
|
520
|
+
try:
|
|
521
|
+
number = float(value)
|
|
522
|
+
except OverflowError as exc:
|
|
523
|
+
raise ValueError("metric must be a finite number") from exc
|
|
524
|
+
if not math.isfinite(number):
|
|
525
|
+
raise ValueError("metric must be a finite number")
|
|
526
|
+
return number
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def _http_url(value: str, path: str, *, v1: bool = False) -> None:
|
|
530
|
+
parsed = urlparse(value)
|
|
531
|
+
if (
|
|
532
|
+
parsed.scheme not in {"http", "https"}
|
|
533
|
+
or not parsed.netloc
|
|
534
|
+
or parsed.username is not None
|
|
535
|
+
or parsed.password is not None
|
|
536
|
+
or parsed.query
|
|
537
|
+
or parsed.fragment
|
|
538
|
+
or v1
|
|
539
|
+
and parsed.path.rstrip("/") != "/v1"
|
|
540
|
+
):
|
|
541
|
+
raise ValueError(f"{path} must be a clean HTTP(S) URL")
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def _json_bytes(value: Any) -> bytes:
|
|
545
|
+
return json.dumps(
|
|
546
|
+
value, ensure_ascii=False, separators=(",", ":"), allow_nan=False
|
|
547
|
+
).encode()
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def deepcopy_json(value: Any) -> Any:
|
|
551
|
+
return json.loads(json.dumps(value))
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
__all__ = [
|
|
555
|
+
"EvaluationConfigError",
|
|
556
|
+
"EvaluationContext",
|
|
557
|
+
"EvaluationResult",
|
|
558
|
+
"run",
|
|
559
|
+
]
|
nufi/exceptions.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class NuFiError(Exception):
|
|
5
|
+
"""Base exception for the NuFi Python SDK."""
|
|
6
|
+
|
|
7
|
+
def __init__(
|
|
8
|
+
self,
|
|
9
|
+
message: str,
|
|
10
|
+
*,
|
|
11
|
+
status_code: int | None = None,
|
|
12
|
+
response_body: str | None = None,
|
|
13
|
+
request_id: str | None = None,
|
|
14
|
+
):
|
|
15
|
+
super().__init__(message)
|
|
16
|
+
self.status_code = status_code
|
|
17
|
+
self.response_body = response_body
|
|
18
|
+
self.request_id = request_id
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class ConfigurationError(NuFiError):
|
|
22
|
+
"""Raised when local profile or configuration data is missing or invalid."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class AuthenticationError(NuFiError):
|
|
26
|
+
"""Raised for missing, expired, invalid, or rejected credentials."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class PermissionDeniedError(NuFiError):
|
|
30
|
+
"""Raised when the API server returns 403."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class NotFoundError(NuFiError):
|
|
34
|
+
"""Raised when the API server returns 404."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ValidationError(NuFiError):
|
|
38
|
+
"""Raised when request data is invalid."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class APIError(NuFiError):
|
|
42
|
+
"""Raised for API errors that do not have a more specific mapping."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class ConnectionError(NuFiError):
|
|
46
|
+
"""Raised when the API server cannot be reached."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class DatasetUploadError(NuFiError):
|
|
50
|
+
"""Raised when a Dataset upload can be resumed after interruption."""
|
|
51
|
+
|
|
52
|
+
def __init__(self, import_id: str, upload_url: str):
|
|
53
|
+
super().__init__(
|
|
54
|
+
f"Dataset upload interrupted for import {import_id}; "
|
|
55
|
+
"call datasets.imports.upload with the same import ID and file to resume."
|
|
56
|
+
)
|
|
57
|
+
self.import_id = import_id
|
|
58
|
+
self.upload_url = upload_url
|
nufi/http_utils.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import socket
|
|
4
|
+
from collections.abc import Mapping
|
|
5
|
+
|
|
6
|
+
import httpx
|
|
7
|
+
|
|
8
|
+
LOCAL_DEV_DOMAIN_SUFFIX = ".nufi.local"
|
|
9
|
+
LOCAL_DEV_LOOPBACK_HOST = "127.0.0.1"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _default_port(scheme: str) -> int | None:
|
|
13
|
+
if scheme == "http":
|
|
14
|
+
return 80
|
|
15
|
+
if scheme == "https":
|
|
16
|
+
return 443
|
|
17
|
+
return None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _host_header(url: httpx.URL) -> str:
|
|
21
|
+
host = url.host or ""
|
|
22
|
+
port = url.port
|
|
23
|
+
if port and port != _default_port(url.scheme):
|
|
24
|
+
return f"{host}:{port}"
|
|
25
|
+
return host
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _needs_local_dev_loopback(url: httpx.URL) -> bool:
|
|
29
|
+
host = url.host
|
|
30
|
+
if url.scheme != "http" or not host or not host.endswith(LOCAL_DEV_DOMAIN_SUFFIX):
|
|
31
|
+
return False
|
|
32
|
+
try:
|
|
33
|
+
socket.getaddrinfo(host, url.port or _default_port(url.scheme) or 0)
|
|
34
|
+
except OSError:
|
|
35
|
+
return True
|
|
36
|
+
return False
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def local_dev_url_and_headers(
|
|
40
|
+
url: str,
|
|
41
|
+
headers: Mapping[str, str] | None = None,
|
|
42
|
+
) -> tuple[str, dict[str, str]]:
|
|
43
|
+
parsed = httpx.URL(url)
|
|
44
|
+
request_headers = dict(headers or {})
|
|
45
|
+
if not _needs_local_dev_loopback(parsed):
|
|
46
|
+
return url, request_headers
|
|
47
|
+
|
|
48
|
+
has_host_header = any(key.lower() == "host" for key in request_headers)
|
|
49
|
+
if not has_host_header:
|
|
50
|
+
request_headers["Host"] = _host_header(parsed)
|
|
51
|
+
return str(parsed.copy_with(host=LOCAL_DEV_LOOPBACK_HOST)), request_headers
|