nufi-python 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nufi/__init__.py +52 -0
- nufi/auth.py +364 -0
- nufi/cli/__init__.py +0 -0
- nufi/cli/commands/__init__.py +1 -0
- nufi/cli/commands/auth.py +113 -0
- nufi/cli/commands/common.py +198 -0
- nufi/cli/commands/compiles.py +507 -0
- nufi/cli/commands/custom_evaluation_templates.py +206 -0
- nufi/cli/commands/datasets.py +697 -0
- nufi/cli/commands/evaluations.py +854 -0
- nufi/cli/commands/leaderboards.py +763 -0
- nufi/cli/commands/models.py +1347 -0
- nufi/cli/commands/monitoring.py +350 -0
- nufi/cli/commands/profile.py +123 -0
- nufi/cli/commands/projects.py +308 -0
- nufi/cli/commands/servings.py +306 -0
- nufi/cli/main.py +66 -0
- nufi/client.py +103 -0
- nufi/config.py +281 -0
- nufi/evaluator.py +559 -0
- nufi/exceptions.py +58 -0
- nufi/http_utils.py +51 -0
- nufi/models/__init__.py +185 -0
- nufi/models/compiles.py +79 -0
- nufi/models/custom_evaluation_templates.py +111 -0
- nufi/models/datasets.py +94 -0
- nufi/models/evaluations.py +253 -0
- nufi/models/leaderboards.py +178 -0
- nufi/models/monitoring.py +179 -0
- nufi/models/pagination.py +33 -0
- nufi/models/projects.py +32 -0
- nufi/models/registry.py +138 -0
- nufi/models/servings.py +27 -0
- nufi/py.typed +1 -0
- nufi/resources/__init__.py +34 -0
- nufi/resources/base.py +190 -0
- nufi/resources/compiles.py +193 -0
- nufi/resources/custom_evaluation_templates.py +126 -0
- nufi/resources/datasets.py +786 -0
- nufi/resources/evaluations.py +428 -0
- nufi/resources/leaderboards.py +413 -0
- nufi/resources/models.py +726 -0
- nufi/resources/monitoring.py +82 -0
- nufi/resources/projects.py +114 -0
- nufi/resources/servings.py +80 -0
- nufi/transport.py +248 -0
- nufi_python-1.0.0.dist-info/METADATA +228 -0
- nufi_python-1.0.0.dist-info/RECORD +52 -0
- nufi_python-1.0.0.dist-info/WHEEL +5 -0
- nufi_python-1.0.0.dist-info/entry_points.txt +2 -0
- nufi_python-1.0.0.dist-info/licenses/LICENSE +21 -0
- nufi_python-1.0.0.dist-info/top_level.txt +1 -0
nufi/client.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
|
|
7
|
+
from nufi import auth, config
|
|
8
|
+
from nufi.resources import (
|
|
9
|
+
CompilesResource,
|
|
10
|
+
DatasetsResource,
|
|
11
|
+
EvaluationsResource,
|
|
12
|
+
LeaderboardsResource,
|
|
13
|
+
MonitoringResource,
|
|
14
|
+
ModelsResource,
|
|
15
|
+
ProjectsResource,
|
|
16
|
+
ServingsResource,
|
|
17
|
+
)
|
|
18
|
+
from nufi.transport import SyncTransport
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class NuFi:
|
|
22
|
+
"""Synchronous NuFi client with an optional default project."""
|
|
23
|
+
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
*,
|
|
27
|
+
base_url: str,
|
|
28
|
+
access_token: str | None = None,
|
|
29
|
+
project: str | int | None = None,
|
|
30
|
+
timeout: float = 30.0,
|
|
31
|
+
transport: SyncTransport | None = None,
|
|
32
|
+
http_transport: httpx.BaseTransport | None = None,
|
|
33
|
+
access_token_provider: Callable[[], str] | None = None,
|
|
34
|
+
refresh_access_token: Callable[[], str] | None = None,
|
|
35
|
+
):
|
|
36
|
+
self.base_url = base_url.rstrip("/")
|
|
37
|
+
self.project = project
|
|
38
|
+
self.transport = transport or SyncTransport(
|
|
39
|
+
self.base_url,
|
|
40
|
+
access_token=access_token,
|
|
41
|
+
access_token_provider=access_token_provider,
|
|
42
|
+
refresh_access_token=refresh_access_token,
|
|
43
|
+
timeout=timeout,
|
|
44
|
+
transport=http_transport,
|
|
45
|
+
)
|
|
46
|
+
resource_args = (self.transport, project)
|
|
47
|
+
self.projects = ProjectsResource(*resource_args)
|
|
48
|
+
self.models = ModelsResource(*resource_args)
|
|
49
|
+
self.servings = ServingsResource(*resource_args)
|
|
50
|
+
self.compiles = CompilesResource(*resource_args)
|
|
51
|
+
self.monitoring = MonitoringResource(*resource_args)
|
|
52
|
+
self.datasets = DatasetsResource(*resource_args)
|
|
53
|
+
self.evaluations = EvaluationsResource(*resource_args)
|
|
54
|
+
self.leaderboards = LeaderboardsResource(*resource_args)
|
|
55
|
+
|
|
56
|
+
@classmethod
|
|
57
|
+
def from_profile(
|
|
58
|
+
cls,
|
|
59
|
+
profile_name: str | None = None,
|
|
60
|
+
*,
|
|
61
|
+
config_store: config.ConfigStore | None = None,
|
|
62
|
+
token_store: auth.TokenStore | None = None,
|
|
63
|
+
now: int | Callable[[], int] | None = None,
|
|
64
|
+
timeout: float = 30.0,
|
|
65
|
+
transport: httpx.BaseTransport | None = None,
|
|
66
|
+
) -> "NuFi":
|
|
67
|
+
"""Create a refresh-capable client from a saved profile snapshot."""
|
|
68
|
+
config_store = config_store or config.ConfigStore()
|
|
69
|
+
resolved_name, profile = config_store.resolve_profile(profile_name)
|
|
70
|
+
access_token = auth.get_valid_access_token(
|
|
71
|
+
resolved_name,
|
|
72
|
+
profile,
|
|
73
|
+
token_store=token_store,
|
|
74
|
+
now=now,
|
|
75
|
+
)
|
|
76
|
+
return cls(
|
|
77
|
+
base_url=profile.resolved_api_url,
|
|
78
|
+
access_token=access_token,
|
|
79
|
+
project=profile.active_project,
|
|
80
|
+
timeout=timeout,
|
|
81
|
+
http_transport=transport,
|
|
82
|
+
access_token_provider=lambda: auth.get_valid_access_token(
|
|
83
|
+
resolved_name,
|
|
84
|
+
profile,
|
|
85
|
+
token_store=token_store,
|
|
86
|
+
now=now,
|
|
87
|
+
),
|
|
88
|
+
refresh_access_token=lambda: auth.refresh_access_token(
|
|
89
|
+
resolved_name,
|
|
90
|
+
profile,
|
|
91
|
+
token_store=token_store,
|
|
92
|
+
now=now,
|
|
93
|
+
),
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
def close(self) -> None:
|
|
97
|
+
self.transport.close()
|
|
98
|
+
|
|
99
|
+
def __enter__(self) -> "NuFi":
|
|
100
|
+
return self
|
|
101
|
+
|
|
102
|
+
def __exit__(self, *args: object) -> None:
|
|
103
|
+
self.close()
|
nufi/config.py
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Literal
|
|
7
|
+
from urllib.parse import urlsplit
|
|
8
|
+
|
|
9
|
+
import typer
|
|
10
|
+
import yaml
|
|
11
|
+
from pydantic import BaseModel, Field, field_validator
|
|
12
|
+
|
|
13
|
+
from nufi.exceptions import ConfigurationError
|
|
14
|
+
|
|
15
|
+
APP_NAME = "nufi"
|
|
16
|
+
DEFAULT_BASE_DOMAIN = "nufi.local"
|
|
17
|
+
DEFAULT_PROJECT = "public-space"
|
|
18
|
+
DEFAULT_REALM = "super-llm"
|
|
19
|
+
DEFAULT_CLIENT_ID = "cli-sdk-client"
|
|
20
|
+
PROFILE_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def is_valid_profile_name(name: str) -> bool:
|
|
24
|
+
return bool(PROFILE_NAME_RE.fullmatch(name))
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def validate_profile_name(name: str) -> None:
|
|
28
|
+
if not is_valid_profile_name(name):
|
|
29
|
+
raise ConfigurationError(
|
|
30
|
+
"Invalid profile name. Use letters, numbers, dot, underscore, or dash."
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def get_config_dir() -> Path:
|
|
35
|
+
if config_path := os.environ.get("NUFI_CONFIG_PATH"):
|
|
36
|
+
return Path(config_path).expanduser().parent
|
|
37
|
+
return Path(typer.get_app_dir(APP_NAME))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def get_config_path() -> Path:
|
|
41
|
+
if config_path := os.environ.get("NUFI_CONFIG_PATH"):
|
|
42
|
+
return Path(config_path).expanduser()
|
|
43
|
+
return get_config_dir() / "config.yaml"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class Profile(BaseModel):
|
|
47
|
+
base_domain: str | None = None
|
|
48
|
+
protocol: Literal["http", "https"] = "https"
|
|
49
|
+
active_project: str = DEFAULT_PROJECT
|
|
50
|
+
realm: str = DEFAULT_REALM
|
|
51
|
+
client_id: str = DEFAULT_CLIENT_ID
|
|
52
|
+
api_url: str | None = None
|
|
53
|
+
keycloak_issuer: str | None = None
|
|
54
|
+
|
|
55
|
+
@field_validator(
|
|
56
|
+
"base_domain",
|
|
57
|
+
"active_project",
|
|
58
|
+
"api_url",
|
|
59
|
+
"keycloak_issuer",
|
|
60
|
+
mode="before",
|
|
61
|
+
)
|
|
62
|
+
@classmethod
|
|
63
|
+
def _clean_optional_string(cls, value: object) -> object:
|
|
64
|
+
if isinstance(value, str):
|
|
65
|
+
value = value.strip()
|
|
66
|
+
return value or None
|
|
67
|
+
return value
|
|
68
|
+
|
|
69
|
+
def model_post_init(self, __context: object) -> None:
|
|
70
|
+
for field, value in (
|
|
71
|
+
("api_url", self.api_url),
|
|
72
|
+
("keycloak_issuer", self.keycloak_issuer),
|
|
73
|
+
):
|
|
74
|
+
if value is not None:
|
|
75
|
+
_validate_url_override(field, value)
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def resolved_api_url(self) -> str:
|
|
79
|
+
if self.api_url:
|
|
80
|
+
url = self.api_url.rstrip("/")
|
|
81
|
+
elif not self.base_domain:
|
|
82
|
+
raise ConfigurationError("Profile requires base_domain or api_url.")
|
|
83
|
+
else:
|
|
84
|
+
url = f"{self.protocol}://api.{self.base_domain}"
|
|
85
|
+
_validate_url_override("api_url", url)
|
|
86
|
+
return url
|
|
87
|
+
|
|
88
|
+
@property
|
|
89
|
+
def resolved_keycloak_issuer(self) -> str:
|
|
90
|
+
if self.keycloak_issuer:
|
|
91
|
+
url = self.keycloak_issuer.rstrip("/")
|
|
92
|
+
elif not self.base_domain:
|
|
93
|
+
raise ConfigurationError("Profile requires base_domain or keycloak_issuer.")
|
|
94
|
+
else:
|
|
95
|
+
url = f"{self.protocol}://keycloak.{self.base_domain}/realms/{self.realm}"
|
|
96
|
+
_validate_url_override("keycloak_issuer", url)
|
|
97
|
+
return url
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _validate_url_override(field: str, value: str) -> None:
|
|
101
|
+
try:
|
|
102
|
+
parsed = urlsplit(value)
|
|
103
|
+
hostname = parsed.hostname
|
|
104
|
+
parsed.port
|
|
105
|
+
except ValueError as exc:
|
|
106
|
+
raise ConfigurationError(
|
|
107
|
+
f"{field} must be an absolute HTTP or HTTPS URL without "
|
|
108
|
+
"userinfo, query, or fragment."
|
|
109
|
+
) from exc
|
|
110
|
+
if (
|
|
111
|
+
parsed.scheme not in {"http", "https"}
|
|
112
|
+
or not parsed.netloc
|
|
113
|
+
or not hostname
|
|
114
|
+
or parsed.username is not None
|
|
115
|
+
or parsed.password is not None
|
|
116
|
+
or parsed.query
|
|
117
|
+
or parsed.fragment
|
|
118
|
+
):
|
|
119
|
+
raise ConfigurationError(
|
|
120
|
+
f"{field} must be an absolute HTTP or HTTPS URL without "
|
|
121
|
+
"userinfo, query, or fragment."
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class StoredProfile(BaseModel):
|
|
126
|
+
name: str
|
|
127
|
+
profile: Profile = Field(exclude=True)
|
|
128
|
+
|
|
129
|
+
@property
|
|
130
|
+
def resolved_api_url(self) -> str:
|
|
131
|
+
return self.profile.resolved_api_url
|
|
132
|
+
|
|
133
|
+
@property
|
|
134
|
+
def resolved_keycloak_issuer(self) -> str:
|
|
135
|
+
return self.profile.resolved_keycloak_issuer
|
|
136
|
+
|
|
137
|
+
@property
|
|
138
|
+
def active_project(self) -> str:
|
|
139
|
+
return self.profile.active_project
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class ConfigData(BaseModel):
|
|
143
|
+
active_profile: str | None = None
|
|
144
|
+
profiles: dict[str, Profile] = Field(default_factory=dict)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class ConfigStore:
|
|
148
|
+
def __init__(self, path: Path | None = None):
|
|
149
|
+
self.path = path or get_config_path()
|
|
150
|
+
|
|
151
|
+
def load(self) -> ConfigData:
|
|
152
|
+
if not self.path.exists():
|
|
153
|
+
return ConfigData()
|
|
154
|
+
raw = yaml.safe_load(self.path.read_text()) or {}
|
|
155
|
+
return ConfigData.model_validate(raw)
|
|
156
|
+
|
|
157
|
+
def save(self, data: ConfigData) -> None:
|
|
158
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
159
|
+
self.path.write_text(
|
|
160
|
+
yaml.safe_dump(data.model_dump(exclude_none=True), sort_keys=False)
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
def upsert_profile(self, name: str, **values: object) -> Profile:
|
|
164
|
+
validate_profile_name(name)
|
|
165
|
+
data = self.load()
|
|
166
|
+
profile = Profile(**values)
|
|
167
|
+
profile.resolved_api_url
|
|
168
|
+
profile.resolved_keycloak_issuer
|
|
169
|
+
data.profiles[name] = profile
|
|
170
|
+
if data.active_profile is None:
|
|
171
|
+
data.active_profile = name
|
|
172
|
+
self.save(data)
|
|
173
|
+
return profile
|
|
174
|
+
|
|
175
|
+
def set_active_profile(self, name: str) -> None:
|
|
176
|
+
validate_profile_name(name)
|
|
177
|
+
data = self.load()
|
|
178
|
+
if name not in data.profiles:
|
|
179
|
+
raise ConfigurationError(f"Profile '{name}' was not found.")
|
|
180
|
+
data.active_profile = name
|
|
181
|
+
self.save(data)
|
|
182
|
+
|
|
183
|
+
def set_active_project(
|
|
184
|
+
self, project: str, *, profile_name: str | None = None
|
|
185
|
+
) -> str:
|
|
186
|
+
project = project.strip()
|
|
187
|
+
if not project:
|
|
188
|
+
raise ConfigurationError("Project name must not be empty.")
|
|
189
|
+
data = self.load()
|
|
190
|
+
resolved_name = profile_name or data.active_profile
|
|
191
|
+
if not resolved_name:
|
|
192
|
+
raise ConfigurationError("No active profile is configured.")
|
|
193
|
+
try:
|
|
194
|
+
profile = data.profiles[resolved_name]
|
|
195
|
+
except KeyError as exc:
|
|
196
|
+
raise ConfigurationError(
|
|
197
|
+
f"Profile '{resolved_name}' was not found."
|
|
198
|
+
) from exc
|
|
199
|
+
profile.active_project = project
|
|
200
|
+
data.profiles[resolved_name] = profile
|
|
201
|
+
self.save(data)
|
|
202
|
+
return project
|
|
203
|
+
|
|
204
|
+
def get_active_profile_name(self) -> str:
|
|
205
|
+
data = self.load()
|
|
206
|
+
if not data.active_profile:
|
|
207
|
+
raise ConfigurationError("No active profile is configured.")
|
|
208
|
+
return data.active_profile
|
|
209
|
+
|
|
210
|
+
def get_profile(self, name: str) -> Profile:
|
|
211
|
+
validate_profile_name(name)
|
|
212
|
+
data = self.load()
|
|
213
|
+
try:
|
|
214
|
+
return data.profiles[name]
|
|
215
|
+
except KeyError as exc:
|
|
216
|
+
raise ConfigurationError(f"Profile '{name}' was not found.") from exc
|
|
217
|
+
|
|
218
|
+
def resolve_profile(self, name: str | None = None) -> tuple[str, Profile]:
|
|
219
|
+
resolved_name = name or self.get_active_profile_name()
|
|
220
|
+
return resolved_name, self.get_profile(resolved_name)
|
|
221
|
+
|
|
222
|
+
def resolve_project(
|
|
223
|
+
self,
|
|
224
|
+
project: str | None = None,
|
|
225
|
+
*,
|
|
226
|
+
profile_name: str | None = None,
|
|
227
|
+
) -> str:
|
|
228
|
+
if project:
|
|
229
|
+
return project
|
|
230
|
+
_, profile = self.resolve_profile(profile_name)
|
|
231
|
+
if not profile.active_project:
|
|
232
|
+
raise ConfigurationError(
|
|
233
|
+
"No project is configured. Run 'nufi project use <project>' "
|
|
234
|
+
"or pass '--project'."
|
|
235
|
+
)
|
|
236
|
+
return profile.active_project
|
|
237
|
+
|
|
238
|
+
def list_profiles(self) -> list[StoredProfile]:
|
|
239
|
+
data = self.load()
|
|
240
|
+
return [
|
|
241
|
+
StoredProfile(name=name, profile=profile)
|
|
242
|
+
for name, profile in data.profiles.items()
|
|
243
|
+
]
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def get_store() -> ConfigStore:
|
|
247
|
+
return ConfigStore()
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def create_profile(name: str, **values: object) -> Profile:
|
|
251
|
+
return get_store().upsert_profile(name, **values)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def use_profile(name: str) -> None:
|
|
255
|
+
get_store().set_active_profile(name)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def use_project(project: str, *, profile_name: str | None = None) -> str:
|
|
259
|
+
return get_store().set_active_project(project, profile_name=profile_name)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def get_active_profile_name() -> str:
|
|
263
|
+
return get_store().get_active_profile_name()
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def get_profile(name: str) -> Profile:
|
|
267
|
+
return get_store().get_profile(name)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def resolve_profile(name: str | None = None) -> tuple[str, Profile]:
|
|
271
|
+
return get_store().resolve_profile(name)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def resolve_project(
|
|
275
|
+
project: str | None = None, *, profile_name: str | None = None
|
|
276
|
+
) -> str:
|
|
277
|
+
return get_store().resolve_project(project, profile_name=profile_name)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def list_profiles() -> list[StoredProfile]:
|
|
281
|
+
return get_store().list_profiles()
|