nufi-python 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nufi/__init__.py +52 -0
- nufi/auth.py +364 -0
- nufi/cli/__init__.py +0 -0
- nufi/cli/commands/__init__.py +1 -0
- nufi/cli/commands/auth.py +113 -0
- nufi/cli/commands/common.py +198 -0
- nufi/cli/commands/compiles.py +507 -0
- nufi/cli/commands/custom_evaluation_templates.py +206 -0
- nufi/cli/commands/datasets.py +697 -0
- nufi/cli/commands/evaluations.py +854 -0
- nufi/cli/commands/leaderboards.py +763 -0
- nufi/cli/commands/models.py +1347 -0
- nufi/cli/commands/monitoring.py +350 -0
- nufi/cli/commands/profile.py +123 -0
- nufi/cli/commands/projects.py +308 -0
- nufi/cli/commands/servings.py +306 -0
- nufi/cli/main.py +66 -0
- nufi/client.py +103 -0
- nufi/config.py +281 -0
- nufi/evaluator.py +559 -0
- nufi/exceptions.py +58 -0
- nufi/http_utils.py +51 -0
- nufi/models/__init__.py +185 -0
- nufi/models/compiles.py +79 -0
- nufi/models/custom_evaluation_templates.py +111 -0
- nufi/models/datasets.py +94 -0
- nufi/models/evaluations.py +253 -0
- nufi/models/leaderboards.py +178 -0
- nufi/models/monitoring.py +179 -0
- nufi/models/pagination.py +33 -0
- nufi/models/projects.py +32 -0
- nufi/models/registry.py +138 -0
- nufi/models/servings.py +27 -0
- nufi/py.typed +1 -0
- nufi/resources/__init__.py +34 -0
- nufi/resources/base.py +190 -0
- nufi/resources/compiles.py +193 -0
- nufi/resources/custom_evaluation_templates.py +126 -0
- nufi/resources/datasets.py +786 -0
- nufi/resources/evaluations.py +428 -0
- nufi/resources/leaderboards.py +413 -0
- nufi/resources/models.py +726 -0
- nufi/resources/monitoring.py +82 -0
- nufi/resources/projects.py +114 -0
- nufi/resources/servings.py +80 -0
- nufi/transport.py +248 -0
- nufi_python-1.0.0.dist-info/METADATA +228 -0
- nufi_python-1.0.0.dist-info/RECORD +52 -0
- nufi_python-1.0.0.dist-info/WHEEL +5 -0
- nufi_python-1.0.0.dist-info/entry_points.txt +2 -0
- nufi_python-1.0.0.dist-info/licenses/LICENSE +21 -0
- nufi_python-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import yaml
|
|
7
|
+
from pydantic import ValidationError
|
|
8
|
+
from yaml.tokens import AliasToken, AnchorToken, TagToken
|
|
9
|
+
|
|
10
|
+
from nufi.models.custom_evaluation_templates import (
|
|
11
|
+
CustomEvaluationTemplate,
|
|
12
|
+
CustomEvaluationTemplateRegistration,
|
|
13
|
+
_require_json_compatible,
|
|
14
|
+
)
|
|
15
|
+
from nufi.resources.base import BaseResource, ProjectRef, escape_path, response_json
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class _UniqueKeyLoader(yaml.SafeLoader):
|
|
19
|
+
def construct_mapping(self, node: yaml.MappingNode, deep: bool = False) -> Any:
|
|
20
|
+
seen: set[Any] = set()
|
|
21
|
+
for key_node, _ in node.value:
|
|
22
|
+
key = self.construct_object(key_node, deep=False)
|
|
23
|
+
if key in seen:
|
|
24
|
+
raise ValueError(f"duplicate YAML key: {key}")
|
|
25
|
+
seen.add(key)
|
|
26
|
+
return super().construct_mapping(node, deep=deep)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def load_strict_mapping(path: str | Path) -> dict[str, Any]:
|
|
30
|
+
path = Path(path)
|
|
31
|
+
try:
|
|
32
|
+
raw = path.read_text(encoding="utf-8")
|
|
33
|
+
for token in yaml.scan(raw):
|
|
34
|
+
if isinstance(token, (AliasToken, AnchorToken)):
|
|
35
|
+
raise ValueError("YAML aliases and anchors are not allowed")
|
|
36
|
+
if isinstance(token, TagToken):
|
|
37
|
+
raise ValueError("YAML tags are not allowed")
|
|
38
|
+
documents = list(yaml.load_all(raw, Loader=_UniqueKeyLoader))
|
|
39
|
+
except (OSError, UnicodeError, yaml.YAMLError) as exc:
|
|
40
|
+
raise ValueError(f"Could not read file '{path}': {exc}") from exc
|
|
41
|
+
if len(documents) != 1:
|
|
42
|
+
raise ValueError("file must contain exactly one document")
|
|
43
|
+
_require_json_compatible(documents[0])
|
|
44
|
+
if not isinstance(documents[0], dict):
|
|
45
|
+
raise ValueError("file must contain a mapping")
|
|
46
|
+
return documents[0]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _load_registration(path: str | Path) -> CustomEvaluationTemplateRegistration:
|
|
50
|
+
try:
|
|
51
|
+
return CustomEvaluationTemplateRegistration.model_validate(
|
|
52
|
+
load_strict_mapping(path)
|
|
53
|
+
)
|
|
54
|
+
except ValidationError as exc:
|
|
55
|
+
raise ValueError(f"invalid registration file: {exc}") from exc
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class CustomEvaluationTemplatesResource(BaseResource):
|
|
59
|
+
@staticmethod
|
|
60
|
+
def load_file(path: str | Path) -> CustomEvaluationTemplateRegistration:
|
|
61
|
+
return _load_registration(path)
|
|
62
|
+
|
|
63
|
+
@staticmethod
|
|
64
|
+
def _payload(
|
|
65
|
+
registration: CustomEvaluationTemplateRegistration | dict[str, Any],
|
|
66
|
+
) -> dict[str, Any]:
|
|
67
|
+
value = CustomEvaluationTemplateRegistration.model_validate(registration)
|
|
68
|
+
return value.model_dump(mode="json", by_alias=True, exclude_none=True)
|
|
69
|
+
|
|
70
|
+
def register(
|
|
71
|
+
self,
|
|
72
|
+
registration: CustomEvaluationTemplateRegistration | dict[str, Any],
|
|
73
|
+
*,
|
|
74
|
+
project: ProjectRef | None = None,
|
|
75
|
+
) -> CustomEvaluationTemplate:
|
|
76
|
+
project_id = self._resolve_project_id(project)
|
|
77
|
+
response = self._transport.post(
|
|
78
|
+
f"/api/v1/projects/{project_id}/custom-evaluation-templates",
|
|
79
|
+
json=self._payload(registration),
|
|
80
|
+
)
|
|
81
|
+
return CustomEvaluationTemplate.model_validate(response_json(response))
|
|
82
|
+
|
|
83
|
+
def register_file(
|
|
84
|
+
self, path: str | Path, *, project: ProjectRef | None = None
|
|
85
|
+
) -> CustomEvaluationTemplate:
|
|
86
|
+
return self.register(self.load_file(path), project=project)
|
|
87
|
+
|
|
88
|
+
def list(
|
|
89
|
+
self,
|
|
90
|
+
*,
|
|
91
|
+
project: ProjectRef | None = None,
|
|
92
|
+
archived: bool | None = None,
|
|
93
|
+
) -> list[CustomEvaluationTemplate]:
|
|
94
|
+
project_id = self._resolve_project_id(project)
|
|
95
|
+
response = self._transport.get(
|
|
96
|
+
f"/api/v1/projects/{project_id}/custom-evaluation-templates",
|
|
97
|
+
params={} if archived is None else {"archived": archived},
|
|
98
|
+
)
|
|
99
|
+
return [
|
|
100
|
+
CustomEvaluationTemplate.model_validate(item)
|
|
101
|
+
for item in response_json(response).get("items", [])
|
|
102
|
+
]
|
|
103
|
+
|
|
104
|
+
def get(
|
|
105
|
+
self, template_id: str, *, project: ProjectRef | None = None
|
|
106
|
+
) -> CustomEvaluationTemplate:
|
|
107
|
+
project_id = self._resolve_project_id(project)
|
|
108
|
+
response = self._transport.get(
|
|
109
|
+
f"/api/v1/projects/{project_id}/custom-evaluation-templates/"
|
|
110
|
+
f"{escape_path(template_id)}"
|
|
111
|
+
)
|
|
112
|
+
return CustomEvaluationTemplate.model_validate(response_json(response))
|
|
113
|
+
|
|
114
|
+
def archive(self, template_id: str, *, project: ProjectRef | None = None) -> None:
|
|
115
|
+
project_id = self._resolve_project_id(project)
|
|
116
|
+
self._transport.post(
|
|
117
|
+
f"/api/v1/projects/{project_id}/custom-evaluation-templates/"
|
|
118
|
+
f"{escape_path(template_id)}/archive"
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
def restore(self, template_id: str, *, project: ProjectRef | None = None) -> None:
|
|
122
|
+
project_id = self._resolve_project_id(project)
|
|
123
|
+
self._transport.post(
|
|
124
|
+
f"/api/v1/projects/{project_id}/custom-evaluation-templates/"
|
|
125
|
+
f"{escape_path(template_id)}/restore"
|
|
126
|
+
)
|