nufi-python 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. nufi/__init__.py +52 -0
  2. nufi/auth.py +364 -0
  3. nufi/cli/__init__.py +0 -0
  4. nufi/cli/commands/__init__.py +1 -0
  5. nufi/cli/commands/auth.py +113 -0
  6. nufi/cli/commands/common.py +198 -0
  7. nufi/cli/commands/compiles.py +507 -0
  8. nufi/cli/commands/custom_evaluation_templates.py +206 -0
  9. nufi/cli/commands/datasets.py +697 -0
  10. nufi/cli/commands/evaluations.py +854 -0
  11. nufi/cli/commands/leaderboards.py +763 -0
  12. nufi/cli/commands/models.py +1347 -0
  13. nufi/cli/commands/monitoring.py +350 -0
  14. nufi/cli/commands/profile.py +123 -0
  15. nufi/cli/commands/projects.py +308 -0
  16. nufi/cli/commands/servings.py +306 -0
  17. nufi/cli/main.py +66 -0
  18. nufi/client.py +103 -0
  19. nufi/config.py +281 -0
  20. nufi/evaluator.py +559 -0
  21. nufi/exceptions.py +58 -0
  22. nufi/http_utils.py +51 -0
  23. nufi/models/__init__.py +185 -0
  24. nufi/models/compiles.py +79 -0
  25. nufi/models/custom_evaluation_templates.py +111 -0
  26. nufi/models/datasets.py +94 -0
  27. nufi/models/evaluations.py +253 -0
  28. nufi/models/leaderboards.py +178 -0
  29. nufi/models/monitoring.py +179 -0
  30. nufi/models/pagination.py +33 -0
  31. nufi/models/projects.py +32 -0
  32. nufi/models/registry.py +138 -0
  33. nufi/models/servings.py +27 -0
  34. nufi/py.typed +1 -0
  35. nufi/resources/__init__.py +34 -0
  36. nufi/resources/base.py +190 -0
  37. nufi/resources/compiles.py +193 -0
  38. nufi/resources/custom_evaluation_templates.py +126 -0
  39. nufi/resources/datasets.py +786 -0
  40. nufi/resources/evaluations.py +428 -0
  41. nufi/resources/leaderboards.py +413 -0
  42. nufi/resources/models.py +726 -0
  43. nufi/resources/monitoring.py +82 -0
  44. nufi/resources/projects.py +114 -0
  45. nufi/resources/servings.py +80 -0
  46. nufi/transport.py +248 -0
  47. nufi_python-1.0.0.dist-info/METADATA +228 -0
  48. nufi_python-1.0.0.dist-info/RECORD +52 -0
  49. nufi_python-1.0.0.dist-info/WHEEL +5 -0
  50. nufi_python-1.0.0.dist-info/entry_points.txt +2 -0
  51. nufi_python-1.0.0.dist-info/licenses/LICENSE +21 -0
  52. nufi_python-1.0.0.dist-info/top_level.txt +1 -0
nufi/evaluator.py ADDED
@@ -0,0 +1,559 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import math
5
+ import os
6
+ import re
7
+ from collections.abc import Callable, Iterator, Mapping
8
+ from dataclasses import dataclass
9
+ from pathlib import Path
10
+ from typing import Any
11
+ from urllib.parse import unquote, urlparse
12
+
13
+ import httpx
14
+
15
+
16
+ SCHEMA_VERSION = "nufi.evaluation.runner.v1"
17
+ SUBMISSION_COMPLETE_MARKER = "NUFI_EVAL_RESULT_SUBMITTED"
18
+ MAX_SPEC_BYTES = 256 * 1024
19
+ MAX_RESULT_BYTES = 64 * 1024
20
+ MAX_DIAGNOSTIC_BYTES = 48 * 1024
21
+ MAX_DIAGNOSTIC_ITEMS = 20
22
+ MAX_DIAGNOSTIC_VALUE_BYTES = 2 * 1024
23
+ MAX_TERMINATION_BYTES = 2 * 1024
24
+ ROLE_NAME = re.compile(r"^[a-z][a-z0-9_-]{0,62}$")
25
+ ROLE_TYPES = {"string", "integer", "number", "boolean", "array", "object"}
26
+ UNAVAILABLE_REASONS = {"runner_omitted", "upload_not_completed", "result_too_large"}
27
+ DEFAULT_DIAGNOSTICS = {
28
+ "status": "unavailable",
29
+ "unavailableReason": "runner_omitted",
30
+ "selectionPolicy": "failed_then_first_by_source_index",
31
+ "complete": False,
32
+ "omittedRowCount": 0,
33
+ "valuesTruncated": False,
34
+ "items": [],
35
+ }
36
+ FAILURE_MESSAGES = {
37
+ "invalid_runner_spec": "Runner spec is invalid.",
38
+ "invalid_evaluator_config": "Evaluator config is invalid.",
39
+ "dataset_read_failed": "Dataset read failed.",
40
+ "target_request_failed": "Target request failed.",
41
+ "invalid_result": "Evaluator result is invalid.",
42
+ "result_submission_failed": "Result submission failed.",
43
+ "evaluator_failed": "Evaluator failed.",
44
+ }
45
+ _TERMINATION_LOG_PATH = Path("/dev/termination-log")
46
+
47
+
48
+ class EvaluationConfigError(ValueError):
49
+ """Signal that evaluator-specific configuration is invalid."""
50
+
51
+
52
+ class _DatasetError(Exception):
53
+ pass
54
+
55
+
56
+ class _TargetError(Exception):
57
+ pass
58
+
59
+
60
+ @dataclass(frozen=True)
61
+ class EvaluationResult:
62
+ metrics: Mapping[str, int | float]
63
+ diagnostic_samples: Mapping[str, Any] | None = None
64
+
65
+
66
+ class _Target:
67
+ def __init__(self, spec: Mapping[str, Any]):
68
+ self.base_url = spec["baseUrl"]
69
+ self.model = spec["model"]
70
+
71
+ def request(self, method: str, path: str, **kwargs: Any) -> httpx.Response:
72
+ parsed = urlparse(path)
73
+ segments = unquote(parsed.path).split("/")
74
+ if (
75
+ not method.strip()
76
+ or parsed.scheme
77
+ or parsed.netloc
78
+ or parsed.query
79
+ or parsed.fragment
80
+ or path.startswith("/")
81
+ or "\\" in path
82
+ or any(segment in {".", ".."} for segment in segments)
83
+ or bool(kwargs.get("follow_redirects"))
84
+ ):
85
+ raise ValueError("target request path must stay under the injected /v1 URL")
86
+ url = f"{self.base_url.rstrip('/')}/{path}"
87
+ try:
88
+ response = _target_request(method, url, **kwargs)
89
+ response.raise_for_status()
90
+ return response
91
+ except (httpx.HTTPError, OSError) as exc:
92
+ raise _TargetError from exc
93
+
94
+ def chat(self, prompt: str) -> str:
95
+ try:
96
+ payload = self.request(
97
+ "POST",
98
+ "chat/completions",
99
+ json={
100
+ "model": self.model,
101
+ "messages": [{"role": "user", "content": prompt}],
102
+ },
103
+ ).json()
104
+ content = payload["choices"][0]["message"]["content"]
105
+ if not isinstance(content, str):
106
+ raise ValueError
107
+ return content
108
+ except (KeyError, IndexError, TypeError, ValueError) as exc:
109
+ raise _TargetError from exc
110
+
111
+
112
+ class EvaluationContext:
113
+ def __init__(self, spec: Mapping[str, Any]):
114
+ self._dataset = spec["dataset"]
115
+ self.config = spec["evaluator"]["config"]
116
+ self.target = _Target(spec["target"])
117
+
118
+ def rows(self) -> Iterator[dict[str, Any]]:
119
+ try:
120
+ root = Path(self._dataset["path"]).resolve(strict=True)
121
+ if not root.is_dir():
122
+ raise _DatasetError
123
+ files = []
124
+ for candidate in sorted(root.iterdir(), key=lambda path: path.name):
125
+ if candidate.suffix.lower() != ".parquet":
126
+ continue
127
+ resolved = candidate.resolve(strict=True)
128
+ try:
129
+ resolved.relative_to(root)
130
+ except ValueError as exc:
131
+ raise _DatasetError from exc
132
+ files.append(resolved)
133
+ if not files:
134
+ raise _DatasetError
135
+
136
+ import pyarrow.parquet as pq
137
+
138
+ found = False
139
+ for path in files:
140
+ for batch in pq.ParquetFile(path).iter_batches():
141
+ for row in batch.to_pylist():
142
+ if not isinstance(row, dict):
143
+ raise _DatasetError
144
+ found = True
145
+ yield row
146
+ if not found:
147
+ raise _DatasetError
148
+ except _DatasetError:
149
+ raise
150
+ except (ImportError, OSError, ValueError) as exc:
151
+ raise _DatasetError from exc
152
+
153
+ def value(self, row: Mapping[str, Any], role: str) -> Any:
154
+ binding = self._dataset["roles"][role]
155
+ field = binding["field"]
156
+ if field is None:
157
+ return None
158
+ value = row.get(field)
159
+ if binding["required"] and value is None:
160
+ raise _DatasetError
161
+ return value
162
+
163
+
164
+ def run(handler: Callable[[EvaluationContext], EvaluationResult]) -> None:
165
+ try:
166
+ spec = _load_spec(os.environ["NUFI_EVAL_SPEC_PATH"])
167
+ except (KeyError, OSError, ValueError):
168
+ _fail("invalid_runner_spec")
169
+
170
+ try:
171
+ result = handler(EvaluationContext(spec))
172
+ except EvaluationConfigError:
173
+ _fail("invalid_evaluator_config")
174
+ except _DatasetError:
175
+ _fail("dataset_read_failed")
176
+ except _TargetError:
177
+ _fail("target_request_failed")
178
+ except Exception:
179
+ _fail("evaluator_failed")
180
+
181
+ try:
182
+ envelope = _result_envelope(spec, result)
183
+ except (TypeError, ValueError):
184
+ _fail("invalid_result")
185
+
186
+ try:
187
+ token = Path(spec["result"]["tokenFile"]).read_text(encoding="utf-8").strip()
188
+ if not token or any(character.isspace() for character in token):
189
+ raise ValueError
190
+ body = json.dumps(
191
+ envelope, ensure_ascii=False, separators=(",", ":"), allow_nan=False
192
+ ).encode()
193
+ _submit_result(spec["result"]["url"], token, body)
194
+ except (OSError, ValueError, httpx.HTTPError):
195
+ _fail("result_submission_failed")
196
+
197
+ print(SUBMISSION_COMPLETE_MARKER)
198
+
199
+
200
+ def _load_spec(path: str | Path) -> dict[str, Any]:
201
+ raw = Path(path).read_bytes()
202
+ if len(raw) > MAX_SPEC_BYTES:
203
+ raise ValueError("runner spec exceeds 256 KiB")
204
+ try:
205
+ spec = json.loads(raw)
206
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
207
+ raise ValueError("runner spec must be valid UTF-8 JSON") from exc
208
+ _validate_spec(spec)
209
+ return spec
210
+
211
+
212
+ def _validate_spec(spec: Any) -> None:
213
+ if not isinstance(spec, dict):
214
+ raise ValueError("runner spec must be an object")
215
+ if len(_json_bytes(spec)) > MAX_SPEC_BYTES:
216
+ raise ValueError("runner spec exceeds 256 KiB")
217
+ _exact(
218
+ spec,
219
+ {"schemaVersion", "run", "dataset", "target", "evaluator", "result"},
220
+ "runner spec",
221
+ )
222
+ if spec["schemaVersion"] != SCHEMA_VERSION:
223
+ raise ValueError(f"schemaVersion must be {SCHEMA_VERSION}")
224
+
225
+ run_spec = _object(spec, "run")
226
+ _exact(run_spec, {"id"}, "run")
227
+ _string(run_spec, "id", "run.id")
228
+
229
+ dataset = _object(spec, "dataset")
230
+ _exact(dataset, {"path", "format", "schemaHash", "schema", "roles"}, "dataset")
231
+ if not Path(_string(dataset, "path", "dataset.path")).is_absolute():
232
+ raise ValueError("dataset.path must be absolute")
233
+ if dataset["format"] != "parquet":
234
+ raise ValueError("dataset.format must be parquet")
235
+ _string(dataset, "schemaHash", "dataset.schemaHash")
236
+ _json_object(dataset, "schema", "dataset.schema")
237
+ roles = _json_object(dataset, "roles", "dataset.roles")
238
+ if not 1 <= len(roles) <= 32:
239
+ raise ValueError("dataset.roles must contain 1 to 32 roles")
240
+ for role, binding in roles.items():
241
+ if ROLE_NAME.fullmatch(role) is None:
242
+ raise ValueError(f"invalid role name {role!r}")
243
+ if not isinstance(binding, dict):
244
+ raise ValueError(f"dataset.roles.{role} must be an object")
245
+ path = f"dataset.roles.{role}"
246
+ _exact(binding, {"field", "type", "required"}, path)
247
+ field = binding["field"]
248
+ if field is not None and (
249
+ not isinstance(field, str) or not field or field != field.strip()
250
+ ):
251
+ raise ValueError(f"{path}.field must be a non-empty string or null")
252
+ if not isinstance(binding["type"], str) or binding["type"] not in ROLE_TYPES:
253
+ raise ValueError(f"{path}.type must be a canonical Dataset type")
254
+ if not isinstance(binding["required"], bool):
255
+ raise ValueError(f"{path}.required must be a boolean")
256
+ if field is None and binding["required"]:
257
+ raise ValueError(f"{path}.field may be null only for an optional role")
258
+
259
+ target = _object(spec, "target")
260
+ _exact(target, {"protocol", "baseUrl", "model"}, "target")
261
+ if target["protocol"] != "openai-compatible":
262
+ raise ValueError("target.protocol must be openai-compatible")
263
+ _http_url(_string(target, "baseUrl", "target.baseUrl"), "target.baseUrl", v1=True)
264
+ _string(target, "model", "target.model")
265
+
266
+ evaluator = _object(spec, "evaluator")
267
+ _exact(evaluator, {"implementation", "config", "declaredMetrics"}, "evaluator")
268
+ if evaluator["implementation"] not in {"lm_eval", "custom"}:
269
+ raise ValueError("evaluator.implementation must be lm_eval or custom")
270
+ _json_object(evaluator, "config", "evaluator.config")
271
+ metrics = evaluator["declaredMetrics"]
272
+ if not isinstance(metrics, list) or not 1 <= len(metrics) <= 32:
273
+ raise ValueError("evaluator.declaredMetrics must contain 1 to 32 metrics")
274
+ seen: set[str] = set()
275
+ for index, metric in enumerate(metrics):
276
+ path = f"evaluator.declaredMetrics[{index}]"
277
+ if not isinstance(metric, dict):
278
+ raise ValueError(f"{path} must be an object")
279
+ _exact(
280
+ metric,
281
+ {"key", "higherIsBetter", "min", "max"},
282
+ path,
283
+ required={"key", "higherIsBetter"},
284
+ )
285
+ key = _string(metric, "key", f"{path}.key")
286
+ if ROLE_NAME.fullmatch(key) is None or key in seen:
287
+ raise ValueError(f"invalid or duplicate declared metric {key!r}")
288
+ seen.add(key)
289
+ if not isinstance(metric["higherIsBetter"], bool):
290
+ raise ValueError(f"{path}.higherIsBetter must be a boolean")
291
+ minimum = _number(metric["min"]) if "min" in metric else None
292
+ maximum = _number(metric["max"]) if "max" in metric else None
293
+ if minimum is not None and maximum is not None and minimum > maximum:
294
+ raise ValueError(f"{path}.min must not exceed max")
295
+
296
+ result = _object(spec, "result")
297
+ _exact(result, {"url", "tokenFile"}, "result")
298
+ _http_url(_string(result, "url", "result.url"), "result.url")
299
+ if not Path(_string(result, "tokenFile", "result.tokenFile")).is_absolute():
300
+ raise ValueError("result.tokenFile must be absolute")
301
+
302
+
303
+ def _result_envelope(
304
+ spec: Mapping[str, Any], result: EvaluationResult
305
+ ) -> dict[str, Any]:
306
+ if not isinstance(result, EvaluationResult) or not isinstance(
307
+ result.metrics, Mapping
308
+ ):
309
+ raise TypeError("handler must return EvaluationResult")
310
+ declared = {
311
+ metric["key"]: metric for metric in spec["evaluator"]["declaredMetrics"]
312
+ }
313
+ if set(result.metrics) != set(declared):
314
+ raise ValueError("result metrics must match declared metrics")
315
+ aggregates = []
316
+ for key, metric in declared.items():
317
+ value = _bounded_metric(result.metrics[key], metric)
318
+ aggregates.append(
319
+ {"key": key, "value": value, "higherIsBetter": metric["higherIsBetter"]}
320
+ )
321
+ diagnostics = _diagnostics(result.diagnostic_samples, declared)
322
+ envelope = {"aggregateMetrics": aggregates, "diagnosticSamples": diagnostics}
323
+ if len(_json_bytes(envelope)) > MAX_RESULT_BYTES:
324
+ raise ValueError("result exceeds 64 KiB")
325
+ return envelope
326
+
327
+
328
+ def _diagnostics(
329
+ raw: Mapping[str, Any] | None, declared: Mapping[str, Mapping[str, Any]]
330
+ ) -> dict[str, Any]:
331
+ if raw is None:
332
+ return deepcopy_json(DEFAULT_DIAGNOSTICS)
333
+ if not isinstance(raw, Mapping):
334
+ raise ValueError("diagnostic_samples must be a mapping")
335
+ diagnostic = dict(raw)
336
+ diagnostic.setdefault("unavailableReason", None)
337
+ diagnostic.setdefault("selectionPolicy", "")
338
+ _exact(
339
+ diagnostic,
340
+ {
341
+ "status",
342
+ "unavailableReason",
343
+ "selectionPolicy",
344
+ "complete",
345
+ "omittedRowCount",
346
+ "valuesTruncated",
347
+ "items",
348
+ },
349
+ "diagnosticSamples",
350
+ )
351
+ if (
352
+ not isinstance(diagnostic["complete"], bool)
353
+ or not isinstance(diagnostic["valuesTruncated"], bool)
354
+ or not isinstance(diagnostic["omittedRowCount"], int)
355
+ or isinstance(diagnostic["omittedRowCount"], bool)
356
+ or diagnostic["omittedRowCount"] < 0
357
+ or not isinstance(diagnostic["items"], list)
358
+ or len(diagnostic["items"]) > MAX_DIAGNOSTIC_ITEMS
359
+ ):
360
+ raise ValueError("invalid diagnosticSamples bounds")
361
+ if diagnostic["status"] == "available":
362
+ if (
363
+ diagnostic["unavailableReason"] is not None
364
+ or diagnostic["selectionPolicy"] != "failed_then_first_by_source_index"
365
+ ):
366
+ raise ValueError("invalid available diagnosticSamples")
367
+ elif diagnostic["status"] == "unavailable":
368
+ if (
369
+ diagnostic["unavailableReason"] not in UNAVAILABLE_REASONS
370
+ or diagnostic["items"]
371
+ ):
372
+ raise ValueError("invalid unavailable diagnosticSamples")
373
+ else:
374
+ raise ValueError("invalid diagnosticSamples status")
375
+ if diagnostic["complete"] and (
376
+ diagnostic["omittedRowCount"] or diagnostic["valuesTruncated"]
377
+ ):
378
+ raise ValueError("complete diagnosticSamples cannot be truncated")
379
+
380
+ items = []
381
+ for raw_item in diagnostic["items"]:
382
+ if not isinstance(raw_item, Mapping):
383
+ raise ValueError("diagnostic item must be a mapping")
384
+ item = dict(raw_item)
385
+ _exact(
386
+ item,
387
+ {"sourceIndex", "rowId", "input", "output", "reference", "scores", "error"},
388
+ "diagnosticSamples.items[]",
389
+ required={"sourceIndex", "rowId", "scores"},
390
+ )
391
+ if (
392
+ not isinstance(item["sourceIndex"], int)
393
+ or isinstance(item["sourceIndex"], bool)
394
+ or item["sourceIndex"] < 0
395
+ or not isinstance(item["rowId"], str)
396
+ or not item["rowId"].strip()
397
+ or not isinstance(item["scores"], Mapping)
398
+ ):
399
+ raise ValueError("invalid diagnostic item identity")
400
+ for name in ("input", "output", "reference", "error"):
401
+ value = item.get(name)
402
+ if value is not None and (
403
+ not isinstance(value, str)
404
+ or len(value.encode()) > MAX_DIAGNOSTIC_VALUE_BYTES
405
+ ):
406
+ raise ValueError("diagnostic value exceeds bounds")
407
+ if not set(item["scores"]).issubset(declared):
408
+ raise ValueError("diagnostic score is not declared")
409
+ scores = {
410
+ key: _bounded_metric(value, declared[key])
411
+ for key, value in item["scores"].items()
412
+ }
413
+ item["scores"] = scores
414
+ items.append(item)
415
+ diagnostic["items"] = items
416
+ if len(_json_bytes(diagnostic)) > MAX_DIAGNOSTIC_BYTES:
417
+ raise ValueError("diagnosticSamples exceeds 48 KiB")
418
+ return diagnostic
419
+
420
+
421
+ def _bounded_metric(value: Any, declaration: Mapping[str, Any]) -> float:
422
+ number = _number(value)
423
+ if "min" in declaration and number < declaration["min"]:
424
+ raise ValueError("metric is below its declared minimum")
425
+ if "max" in declaration and number > declaration["max"]:
426
+ raise ValueError("metric exceeds its declared maximum")
427
+ return number
428
+
429
+
430
+ def _submit_result(url: str, token: str, body: bytes) -> None:
431
+ headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
432
+ with _http_client() as client:
433
+ for attempt in range(2):
434
+ try:
435
+ response = client.post(url, content=body, headers=headers)
436
+ if response.status_code < 500 or attempt == 1:
437
+ response.raise_for_status()
438
+ return
439
+ except httpx.RequestError:
440
+ if attempt == 1:
441
+ raise
442
+
443
+
444
+ def _target_request(method: str, url: str, **kwargs: Any) -> httpx.Response:
445
+ return httpx.request(method, url, timeout=30, **kwargs)
446
+
447
+
448
+ def _http_client() -> httpx.Client:
449
+ return httpx.Client(timeout=30)
450
+
451
+
452
+ def _fail(code: str) -> None:
453
+ payload = json.dumps(
454
+ {"code": code, "message": FAILURE_MESSAGES[code]}, separators=(",", ":")
455
+ ).encode()
456
+ if len(payload) > MAX_TERMINATION_BYTES:
457
+ raise AssertionError("termination payload exceeds 2 KiB")
458
+ try:
459
+ _TERMINATION_LOG_PATH.write_bytes(payload)
460
+ except OSError:
461
+ pass
462
+ raise SystemExit(1)
463
+
464
+
465
+ def _exact(
466
+ value: Mapping[str, Any],
467
+ allowed: set[str],
468
+ path: str,
469
+ *,
470
+ required: set[str] | None = None,
471
+ ) -> None:
472
+ unknown = sorted(set(value) - allowed)
473
+ if unknown:
474
+ raise ValueError(f"{path}.{unknown[0]} is not allowed")
475
+ missing = sorted((allowed if required is None else required) - set(value))
476
+ if missing:
477
+ raise ValueError(f"{path}.{missing[0]} is required")
478
+
479
+
480
+ def _object(value: Mapping[str, Any], key: str) -> dict[str, Any]:
481
+ item = value.get(key)
482
+ if not isinstance(item, dict):
483
+ raise ValueError(f"{key} must be an object")
484
+ return item
485
+
486
+
487
+ def _json_object(value: Mapping[str, Any], key: str, path: str) -> dict[str, Any]:
488
+ item = _object(value, key)
489
+ _validate_json(item, path)
490
+ return item
491
+
492
+
493
+ def _validate_json(value: Any, path: str) -> None:
494
+ if isinstance(value, dict):
495
+ for key, item in value.items():
496
+ if not isinstance(key, str):
497
+ raise ValueError(f"{path} keys must be strings")
498
+ _validate_json(item, f"{path}.{key}")
499
+ elif isinstance(value, list):
500
+ for index, item in enumerate(value):
501
+ _validate_json(item, f"{path}[{index}]")
502
+ elif isinstance(value, float) and not math.isfinite(value):
503
+ raise ValueError(f"{path} numbers must be finite")
504
+ elif not isinstance(value, str | int | float | bool | type(None)):
505
+ raise ValueError(f"{path} must contain JSON values only")
506
+
507
+
508
+ def _string(value: Mapping[str, Any], key: str, path: str) -> str:
509
+ item = value.get(key)
510
+ if not isinstance(item, str) or not item.strip():
511
+ raise ValueError(f"{path} must be a non-empty string")
512
+ if item != item.strip():
513
+ raise ValueError(f"{path} must not contain surrounding whitespace")
514
+ return item
515
+
516
+
517
+ def _number(value: Any) -> float:
518
+ if not isinstance(value, int | float) or isinstance(value, bool):
519
+ raise ValueError("metric must be a finite number")
520
+ try:
521
+ number = float(value)
522
+ except OverflowError as exc:
523
+ raise ValueError("metric must be a finite number") from exc
524
+ if not math.isfinite(number):
525
+ raise ValueError("metric must be a finite number")
526
+ return number
527
+
528
+
529
+ def _http_url(value: str, path: str, *, v1: bool = False) -> None:
530
+ parsed = urlparse(value)
531
+ if (
532
+ parsed.scheme not in {"http", "https"}
533
+ or not parsed.netloc
534
+ or parsed.username is not None
535
+ or parsed.password is not None
536
+ or parsed.query
537
+ or parsed.fragment
538
+ or v1
539
+ and parsed.path.rstrip("/") != "/v1"
540
+ ):
541
+ raise ValueError(f"{path} must be a clean HTTP(S) URL")
542
+
543
+
544
+ def _json_bytes(value: Any) -> bytes:
545
+ return json.dumps(
546
+ value, ensure_ascii=False, separators=(",", ":"), allow_nan=False
547
+ ).encode()
548
+
549
+
550
+ def deepcopy_json(value: Any) -> Any:
551
+ return json.loads(json.dumps(value))
552
+
553
+
554
+ __all__ = [
555
+ "EvaluationConfigError",
556
+ "EvaluationContext",
557
+ "EvaluationResult",
558
+ "run",
559
+ ]
nufi/exceptions.py ADDED
@@ -0,0 +1,58 @@
1
+ from __future__ import annotations
2
+
3
+
4
+ class NuFiError(Exception):
5
+ """Base exception for the NuFi Python SDK."""
6
+
7
+ def __init__(
8
+ self,
9
+ message: str,
10
+ *,
11
+ status_code: int | None = None,
12
+ response_body: str | None = None,
13
+ request_id: str | None = None,
14
+ ):
15
+ super().__init__(message)
16
+ self.status_code = status_code
17
+ self.response_body = response_body
18
+ self.request_id = request_id
19
+
20
+
21
+ class ConfigurationError(NuFiError):
22
+ """Raised when local profile or configuration data is missing or invalid."""
23
+
24
+
25
+ class AuthenticationError(NuFiError):
26
+ """Raised for missing, expired, invalid, or rejected credentials."""
27
+
28
+
29
+ class PermissionDeniedError(NuFiError):
30
+ """Raised when the API server returns 403."""
31
+
32
+
33
+ class NotFoundError(NuFiError):
34
+ """Raised when the API server returns 404."""
35
+
36
+
37
+ class ValidationError(NuFiError):
38
+ """Raised when request data is invalid."""
39
+
40
+
41
+ class APIError(NuFiError):
42
+ """Raised for API errors that do not have a more specific mapping."""
43
+
44
+
45
+ class ConnectionError(NuFiError):
46
+ """Raised when the API server cannot be reached."""
47
+
48
+
49
+ class DatasetUploadError(NuFiError):
50
+ """Raised when a Dataset upload can be resumed after interruption."""
51
+
52
+ def __init__(self, import_id: str, upload_url: str):
53
+ super().__init__(
54
+ f"Dataset upload interrupted for import {import_id}; "
55
+ "call datasets.imports.upload with the same import ID and file to resume."
56
+ )
57
+ self.import_id = import_id
58
+ self.upload_url = upload_url
nufi/http_utils.py ADDED
@@ -0,0 +1,51 @@
1
+ from __future__ import annotations
2
+
3
+ import socket
4
+ from collections.abc import Mapping
5
+
6
+ import httpx
7
+
8
+ LOCAL_DEV_DOMAIN_SUFFIX = ".nufi.local"
9
+ LOCAL_DEV_LOOPBACK_HOST = "127.0.0.1"
10
+
11
+
12
+ def _default_port(scheme: str) -> int | None:
13
+ if scheme == "http":
14
+ return 80
15
+ if scheme == "https":
16
+ return 443
17
+ return None
18
+
19
+
20
+ def _host_header(url: httpx.URL) -> str:
21
+ host = url.host or ""
22
+ port = url.port
23
+ if port and port != _default_port(url.scheme):
24
+ return f"{host}:{port}"
25
+ return host
26
+
27
+
28
+ def _needs_local_dev_loopback(url: httpx.URL) -> bool:
29
+ host = url.host
30
+ if url.scheme != "http" or not host or not host.endswith(LOCAL_DEV_DOMAIN_SUFFIX):
31
+ return False
32
+ try:
33
+ socket.getaddrinfo(host, url.port or _default_port(url.scheme) or 0)
34
+ except OSError:
35
+ return True
36
+ return False
37
+
38
+
39
+ def local_dev_url_and_headers(
40
+ url: str,
41
+ headers: Mapping[str, str] | None = None,
42
+ ) -> tuple[str, dict[str, str]]:
43
+ parsed = httpx.URL(url)
44
+ request_headers = dict(headers or {})
45
+ if not _needs_local_dev_loopback(parsed):
46
+ return url, request_headers
47
+
48
+ has_host_header = any(key.lower() == "host" for key in request_headers)
49
+ if not has_host_header:
50
+ request_headers["Host"] = _host_header(parsed)
51
+ return str(parsed.copy_with(host=LOCAL_DEV_LOOPBACK_HOST)), request_headers