nufi-python 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nufi/__init__.py +52 -0
- nufi/auth.py +364 -0
- nufi/cli/__init__.py +0 -0
- nufi/cli/commands/__init__.py +1 -0
- nufi/cli/commands/auth.py +113 -0
- nufi/cli/commands/common.py +198 -0
- nufi/cli/commands/compiles.py +507 -0
- nufi/cli/commands/custom_evaluation_templates.py +206 -0
- nufi/cli/commands/datasets.py +697 -0
- nufi/cli/commands/evaluations.py +854 -0
- nufi/cli/commands/leaderboards.py +763 -0
- nufi/cli/commands/models.py +1347 -0
- nufi/cli/commands/monitoring.py +350 -0
- nufi/cli/commands/profile.py +123 -0
- nufi/cli/commands/projects.py +308 -0
- nufi/cli/commands/servings.py +306 -0
- nufi/cli/main.py +66 -0
- nufi/client.py +103 -0
- nufi/config.py +281 -0
- nufi/evaluator.py +559 -0
- nufi/exceptions.py +58 -0
- nufi/http_utils.py +51 -0
- nufi/models/__init__.py +185 -0
- nufi/models/compiles.py +79 -0
- nufi/models/custom_evaluation_templates.py +111 -0
- nufi/models/datasets.py +94 -0
- nufi/models/evaluations.py +253 -0
- nufi/models/leaderboards.py +178 -0
- nufi/models/monitoring.py +179 -0
- nufi/models/pagination.py +33 -0
- nufi/models/projects.py +32 -0
- nufi/models/registry.py +138 -0
- nufi/models/servings.py +27 -0
- nufi/py.typed +1 -0
- nufi/resources/__init__.py +34 -0
- nufi/resources/base.py +190 -0
- nufi/resources/compiles.py +193 -0
- nufi/resources/custom_evaluation_templates.py +126 -0
- nufi/resources/datasets.py +786 -0
- nufi/resources/evaluations.py +428 -0
- nufi/resources/leaderboards.py +413 -0
- nufi/resources/models.py +726 -0
- nufi/resources/monitoring.py +82 -0
- nufi/resources/projects.py +114 -0
- nufi/resources/servings.py +80 -0
- nufi/transport.py +248 -0
- nufi_python-1.0.0.dist-info/METADATA +228 -0
- nufi_python-1.0.0.dist-info/RECORD +52 -0
- nufi_python-1.0.0.dist-info/WHEEL +5 -0
- nufi_python-1.0.0.dist-info/entry_points.txt +2 -0
- nufi_python-1.0.0.dist-info/licenses/LICENSE +21 -0
- nufi_python-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from typing import Any, Literal
|
|
5
|
+
|
|
6
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
|
7
|
+
|
|
8
|
+
from nufi.models.custom_evaluation_templates import CustomEvaluationTemplateProjection
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class EvaluationMethodTemplate(BaseModel):
|
|
12
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
13
|
+
|
|
14
|
+
type: str
|
|
15
|
+
display_name: str = Field(alias="displayName")
|
|
16
|
+
evaluator: str | None = None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class EvaluationMethodV2Template(BaseModel):
|
|
20
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
21
|
+
|
|
22
|
+
name: str
|
|
23
|
+
display_name: str = Field(alias="displayName")
|
|
24
|
+
output_type: str = Field(alias="outputType")
|
|
25
|
+
dataset_mapping: list[dict[str, Any]] = Field(alias="datasetMapping")
|
|
26
|
+
metrics: list[str]
|
|
27
|
+
default_native_config: dict[str, Any] = Field(alias="defaultNativeConfig")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class EvaluationMethodV2Metadata(BaseModel):
|
|
31
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
32
|
+
|
|
33
|
+
schema_version: str = Field(alias="schemaVersion")
|
|
34
|
+
runner_type: str = Field(alias="runnerType")
|
|
35
|
+
templates: list[EvaluationMethodV2Template]
|
|
36
|
+
metrics_by_output_type: dict[str, list[str]] = Field(alias="metricsByOutputType")
|
|
37
|
+
filters: list[str]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class EvaluationTemplateRole(BaseModel):
|
|
41
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
42
|
+
|
|
43
|
+
name: str
|
|
44
|
+
type: str
|
|
45
|
+
required: bool
|
|
46
|
+
allows_missing_or_null: bool = Field(alias="allowsMissingOrNull")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class EvaluationTemplateMetric(BaseModel):
|
|
50
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
51
|
+
|
|
52
|
+
key: str
|
|
53
|
+
display_name: str = Field(alias="displayName")
|
|
54
|
+
higher_is_better: bool = Field(alias="higherIsBetter")
|
|
55
|
+
min: float | None = None
|
|
56
|
+
max: float | None = None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class EvaluationCatalogTemplate(BaseModel):
|
|
60
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
61
|
+
|
|
62
|
+
id: str
|
|
63
|
+
source: Literal["nufi", "custom"]
|
|
64
|
+
name: str
|
|
65
|
+
version: str | None
|
|
66
|
+
display_name: str = Field(alias="displayName")
|
|
67
|
+
description: str | None
|
|
68
|
+
image: str | None
|
|
69
|
+
image_policy: Literal["pinned", "mutable"] | None = Field(alias="imagePolicy")
|
|
70
|
+
dataset_roles: list[EvaluationTemplateRole] = Field(alias="datasetRoles")
|
|
71
|
+
metrics: list[EvaluationTemplateMetric]
|
|
72
|
+
default_native_config: dict[str, Any] | None = Field(alias="defaultNativeConfig")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class EvaluationMethodTemplates(BaseModel):
|
|
76
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
77
|
+
|
|
78
|
+
method_v2: EvaluationMethodV2Metadata = Field(alias="methodV2")
|
|
79
|
+
templates: list[EvaluationCatalogTemplate] = Field(default_factory=list)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class EvaluationMethodPreview(BaseModel):
|
|
83
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
84
|
+
|
|
85
|
+
config: dict[str, Any]
|
|
86
|
+
source_schema_hash: str = Field(alias="sourceSchemaHash")
|
|
87
|
+
template_id: str | None = Field(default=None, alias="templateId")
|
|
88
|
+
custom_template: CustomEvaluationTemplateProjection | None = Field(
|
|
89
|
+
default=None, alias="customTemplate"
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class EvaluationMethod(BaseModel):
|
|
94
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
95
|
+
|
|
96
|
+
id: str
|
|
97
|
+
project_id: str = Field(alias="projectId")
|
|
98
|
+
dataset_id: str = Field(alias="datasetId")
|
|
99
|
+
dataset_version_id: str | None = Field(default=None, alias="datasetVersionId")
|
|
100
|
+
name: str
|
|
101
|
+
type: str
|
|
102
|
+
evaluator: str
|
|
103
|
+
status: str
|
|
104
|
+
config: dict[str, Any]
|
|
105
|
+
criteria_config: dict[str, Any] | None = Field(default=None, alias="criteriaConfig")
|
|
106
|
+
method_config: dict[str, Any] | None = Field(default=None, alias="methodConfig")
|
|
107
|
+
default_runtime: dict[str, Any] | None = Field(alias="defaultRuntime")
|
|
108
|
+
source_schema_hash: str = Field(alias="sourceSchemaHash")
|
|
109
|
+
source_schema_version: str = Field(alias="sourceSchemaVersion")
|
|
110
|
+
description: str
|
|
111
|
+
template: EvaluationMethodTemplate
|
|
112
|
+
runner: dict[str, Any]
|
|
113
|
+
template_id: str | None = Field(default=None, alias="templateId")
|
|
114
|
+
template_source: str | None = Field(default=None, alias="templateSource")
|
|
115
|
+
custom_template: CustomEvaluationTemplateProjection | None = Field(
|
|
116
|
+
default=None, alias="customTemplate"
|
|
117
|
+
)
|
|
118
|
+
archived_at: datetime | None = Field(default=None, alias="archivedAt")
|
|
119
|
+
archived_by: str | None = Field(default=None, alias="archivedBy")
|
|
120
|
+
archive_reason: str | None = Field(default=None, alias="archiveReason")
|
|
121
|
+
created_at: datetime = Field(alias="createdAt")
|
|
122
|
+
updated_at: datetime = Field(alias="updatedAt")
|
|
123
|
+
|
|
124
|
+
@field_validator("config")
|
|
125
|
+
@classmethod
|
|
126
|
+
def validate_v2_config(cls, config: dict[str, Any]) -> dict[str, Any]:
|
|
127
|
+
supported = {
|
|
128
|
+
"nufi.evaluation.method.v2",
|
|
129
|
+
"nufi.evaluation.method.custom.v1",
|
|
130
|
+
}
|
|
131
|
+
if config.get("schemaVersion") not in supported:
|
|
132
|
+
raise ValueError("config.schemaVersion is not supported")
|
|
133
|
+
return config
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class EvaluationMethodListResponse(BaseModel):
|
|
137
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
138
|
+
|
|
139
|
+
items: list[EvaluationMethod]
|
|
140
|
+
total: int
|
|
141
|
+
page: int
|
|
142
|
+
limit: int
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class EvaluationTargetResources(BaseModel):
|
|
146
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
147
|
+
|
|
148
|
+
cpu: str
|
|
149
|
+
memory: str
|
|
150
|
+
accelerator_count: int = Field(alias="acceleratorCount")
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class EvaluationAggregateMetric(BaseModel):
|
|
154
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
155
|
+
|
|
156
|
+
key: str
|
|
157
|
+
value: float
|
|
158
|
+
higher_is_better: bool = Field(alias="higherIsBetter")
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class EvaluationDiagnosticSample(BaseModel):
|
|
162
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
163
|
+
|
|
164
|
+
source_index: int = Field(alias="sourceIndex")
|
|
165
|
+
row_id: str = Field(alias="rowId")
|
|
166
|
+
input: str | None = None
|
|
167
|
+
output: str | None = None
|
|
168
|
+
reference: str | None = None
|
|
169
|
+
scores: dict[str, float | bool]
|
|
170
|
+
error: str | None = None
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
class EvaluationDiagnosticSamples(BaseModel):
|
|
174
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
175
|
+
|
|
176
|
+
status: str
|
|
177
|
+
unavailable_reason: str | None = Field(default=None, alias="unavailableReason")
|
|
178
|
+
selection_policy: str = Field(alias="selectionPolicy")
|
|
179
|
+
complete: bool
|
|
180
|
+
omitted_row_count: int = Field(alias="omittedRowCount")
|
|
181
|
+
values_truncated: bool = Field(alias="valuesTruncated")
|
|
182
|
+
items: list[EvaluationDiagnosticSample]
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
class EvaluationRun(BaseModel):
|
|
186
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
187
|
+
|
|
188
|
+
id: str
|
|
189
|
+
project_id: str = Field(alias="projectId")
|
|
190
|
+
evaluation_type: str | None = Field(default=None, alias="evaluationType")
|
|
191
|
+
dataset_id: str | None = Field(alias="datasetId")
|
|
192
|
+
evaluation_method_id: str | None = Field(default=None, alias="evaluationTaskId")
|
|
193
|
+
dataset_version_id: str | None = Field(default=None, alias="datasetVersionId")
|
|
194
|
+
evaluation_snapshot: dict[str, Any] | None = Field(
|
|
195
|
+
default=None, alias="evaluationSnapshot"
|
|
196
|
+
)
|
|
197
|
+
model_artifact_id: int = Field(alias="modelArtifactId")
|
|
198
|
+
model_artifact: dict[str, Any] | None = Field(default=None, alias="modelArtifact")
|
|
199
|
+
evaluator: str | None = None
|
|
200
|
+
runner_type: str = Field(alias="runnerType", min_length=1)
|
|
201
|
+
runner_config_schema_version: str = Field(
|
|
202
|
+
alias="runnerConfigSchemaVersion", min_length=1
|
|
203
|
+
)
|
|
204
|
+
runner_image: str | None = Field(default=None, alias="runnerImage")
|
|
205
|
+
runner_version: str | None = Field(default=None, alias="runnerVersion")
|
|
206
|
+
template_source: str | None = Field(default=None, alias="templateSource")
|
|
207
|
+
image_policy: str | None = Field(default=None, alias="imagePolicy")
|
|
208
|
+
resolved_image_id: str | None = Field(default=None, alias="resolvedImageId")
|
|
209
|
+
runner_config: dict[str, Any] | None = Field(default=None, alias="runnerConfig")
|
|
210
|
+
runner_snapshot: dict[str, Any] | None = Field(default=None, alias="runnerSnapshot")
|
|
211
|
+
target_resources: EvaluationTargetResources | None = Field(
|
|
212
|
+
default=None, alias="targetResources"
|
|
213
|
+
)
|
|
214
|
+
status: str
|
|
215
|
+
metrics: dict[str, Any] | None = None
|
|
216
|
+
metric_meta: dict[str, Any] | None = Field(default=None, alias="metricMeta")
|
|
217
|
+
runtime_metrics: dict[str, Any] | None = Field(default=None, alias="runtimeMetrics")
|
|
218
|
+
generated_evaluator_config: dict[str, Any] | None = Field(
|
|
219
|
+
default=None, alias="generatedEvaluatorConfig"
|
|
220
|
+
)
|
|
221
|
+
result_metrics: dict[str, Any] | None = Field(default=None, alias="resultMetrics")
|
|
222
|
+
result_artifacts: dict[str, Any] | None = Field(
|
|
223
|
+
default=None, alias="resultArtifacts"
|
|
224
|
+
)
|
|
225
|
+
aggregate_metrics: list[EvaluationAggregateMetric] | None = Field(
|
|
226
|
+
default=None, alias="aggregateMetrics"
|
|
227
|
+
)
|
|
228
|
+
diagnostic_samples: EvaluationDiagnosticSamples | None = Field(
|
|
229
|
+
default=None, alias="diagnosticSamples"
|
|
230
|
+
)
|
|
231
|
+
result: dict[str, Any] | None = None
|
|
232
|
+
raw_evaluator_result_path: str | None = Field(
|
|
233
|
+
default=None, alias="rawEvaluatorResultPath"
|
|
234
|
+
)
|
|
235
|
+
result_artifact_path: str | None = Field(default=None, alias="resultArtifactPath")
|
|
236
|
+
error_message: str | None = Field(default=None, alias="errorMessage")
|
|
237
|
+
job_name: str | None = Field(default=None, alias="jobName")
|
|
238
|
+
target_deploy_name: str | None = Field(default=None, alias="targetDeployName")
|
|
239
|
+
created_by: str = Field(alias="createdBy")
|
|
240
|
+
created_at: datetime = Field(alias="createdAt")
|
|
241
|
+
completed_at: datetime | None = Field(default=None, alias="completedAt")
|
|
242
|
+
cancel_requested_at: datetime | None = Field(
|
|
243
|
+
default=None, alias="cancelRequestedAt"
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
class EvaluationRunListResponse(BaseModel):
|
|
248
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
249
|
+
|
|
250
|
+
items: list[EvaluationRun]
|
|
251
|
+
total: int
|
|
252
|
+
page: int
|
|
253
|
+
limit: int
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class LeaderboardModel(BaseModel):
|
|
10
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class LeaderboardMetricRef(LeaderboardModel):
|
|
14
|
+
source: str
|
|
15
|
+
name: str
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class LeaderboardGateCondition(LeaderboardModel):
|
|
19
|
+
metric: LeaderboardMetricRef
|
|
20
|
+
operator: str
|
|
21
|
+
value: float
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class LeaderboardGatePolicy(LeaderboardModel):
|
|
25
|
+
all: list[LeaderboardGateCondition] = Field(default_factory=list)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class LeaderboardRankingRule(LeaderboardModel):
|
|
29
|
+
metric: LeaderboardMetricRef
|
|
30
|
+
direction: str
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class LeaderboardRankingPolicy(LeaderboardModel):
|
|
34
|
+
primary: LeaderboardRankingRule
|
|
35
|
+
tie_breakers: list[LeaderboardRankingRule] = Field(
|
|
36
|
+
default_factory=list, alias="tieBreakers"
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class LeaderboardRequestedResources(LeaderboardModel):
|
|
41
|
+
cpu: str
|
|
42
|
+
memory: str
|
|
43
|
+
accelerator_count: int = Field(alias="acceleratorCount")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class LeaderboardEvaluationMethodSummary(LeaderboardModel):
|
|
47
|
+
id: str
|
|
48
|
+
name: str
|
|
49
|
+
status: str
|
|
50
|
+
dataset_id: str = Field(alias="datasetId")
|
|
51
|
+
dataset_version_id: str = Field(alias="datasetVersionId")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class LeaderboardSubmissionCounts(LeaderboardModel):
|
|
55
|
+
total: int
|
|
56
|
+
qualified: int
|
|
57
|
+
disqualified: int
|
|
58
|
+
incomparable: int
|
|
59
|
+
not_evaluated: int = Field(alias="notEvaluated")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class Leaderboard(LeaderboardModel):
|
|
63
|
+
id: str
|
|
64
|
+
project_id: str = Field(alias="projectId")
|
|
65
|
+
name: str
|
|
66
|
+
description: str
|
|
67
|
+
evaluation_method_id: str = Field(alias="evaluationMethodId")
|
|
68
|
+
evaluation_method: LeaderboardEvaluationMethodSummary = Field(
|
|
69
|
+
alias="evaluationMethod"
|
|
70
|
+
)
|
|
71
|
+
runner_config: dict[str, Any] | None = Field(alias="runnerConfig")
|
|
72
|
+
gate_policy: LeaderboardGatePolicy = Field(alias="gatePolicy")
|
|
73
|
+
ranking_policy: LeaderboardRankingPolicy = Field(alias="rankingPolicy")
|
|
74
|
+
resource_defaults: LeaderboardRequestedResources = Field(alias="resourceDefaults")
|
|
75
|
+
status: str
|
|
76
|
+
submission_counts: LeaderboardSubmissionCounts | None = Field(
|
|
77
|
+
default=None, alias="submissionCounts"
|
|
78
|
+
)
|
|
79
|
+
created_by: str = Field(alias="createdBy")
|
|
80
|
+
updated_by: str = Field(alias="updatedBy")
|
|
81
|
+
created_at: datetime = Field(alias="createdAt")
|
|
82
|
+
updated_at: datetime = Field(alias="updatedAt")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class LeaderboardArtifact(LeaderboardModel):
|
|
86
|
+
id: int
|
|
87
|
+
model_name: str | None = Field(default=None, alias="modelName")
|
|
88
|
+
version: str | None = None
|
|
89
|
+
name: str
|
|
90
|
+
platform: str
|
|
91
|
+
source_type: str | None = Field(default=None, alias="sourceType")
|
|
92
|
+
compilation_run_id: int | None = Field(default=None, alias="compilationRunId")
|
|
93
|
+
source_optimizer_run_id: int | None = Field(
|
|
94
|
+
default=None, alias="sourceOptimizerRunId"
|
|
95
|
+
)
|
|
96
|
+
source_optimizer_trial_id: int | None = Field(
|
|
97
|
+
default=None, alias="sourceOptimizerTrialId"
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class LeaderboardRun(LeaderboardModel):
|
|
102
|
+
evaluation_run_id: str = Field(alias="evaluationRunId")
|
|
103
|
+
attempt_number: int = Field(alias="attemptNumber")
|
|
104
|
+
status: str
|
|
105
|
+
error_message: str | None = Field(default=None, alias="errorMessage")
|
|
106
|
+
created_at: datetime = Field(alias="createdAt")
|
|
107
|
+
completed_at: datetime | None = Field(default=None, alias="completedAt")
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class LeaderboardQualificationReason(LeaderboardModel):
|
|
111
|
+
code: str
|
|
112
|
+
metric: LeaderboardMetricRef
|
|
113
|
+
operator: str | None = None
|
|
114
|
+
threshold: float | None = None
|
|
115
|
+
actual: float | None = None
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class LeaderboardSubmission(LeaderboardModel):
|
|
119
|
+
id: str
|
|
120
|
+
project_id: str = Field(alias="projectId")
|
|
121
|
+
leaderboard_id: str = Field(alias="leaderboardId")
|
|
122
|
+
model_artifact_id: int = Field(alias="modelArtifactId")
|
|
123
|
+
artifact: LeaderboardArtifact
|
|
124
|
+
requested_resources: LeaderboardRequestedResources = Field(
|
|
125
|
+
alias="requestedResources"
|
|
126
|
+
)
|
|
127
|
+
qualification: str
|
|
128
|
+
qualification_reasons: list[LeaderboardQualificationReason] = Field(
|
|
129
|
+
alias="qualificationReasons"
|
|
130
|
+
)
|
|
131
|
+
latest_run: LeaderboardRun | None = Field(alias="latestRun")
|
|
132
|
+
scoring_run: LeaderboardRun | None = Field(alias="scoringRun")
|
|
133
|
+
aggregate_metrics: dict[str, Any] = Field(alias="aggregateMetrics")
|
|
134
|
+
runtime_metrics: dict[str, Any] = Field(alias="runtimeMetrics")
|
|
135
|
+
rank: int | None
|
|
136
|
+
created_by: str = Field(alias="createdBy")
|
|
137
|
+
created_at: datetime = Field(alias="createdAt")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class LeaderboardEntry(LeaderboardModel):
|
|
141
|
+
rank: int
|
|
142
|
+
submission_id: str = Field(alias="submissionId")
|
|
143
|
+
evaluation_method_id: str = Field(alias="evaluationMethodId")
|
|
144
|
+
evaluation_run_id: str = Field(alias="evaluationRunId")
|
|
145
|
+
model_artifact_id: int = Field(alias="modelArtifactId")
|
|
146
|
+
artifact: LeaderboardArtifact
|
|
147
|
+
requested_resources: LeaderboardRequestedResources = Field(
|
|
148
|
+
alias="requestedResources"
|
|
149
|
+
)
|
|
150
|
+
aggregate_metrics: dict[str, Any] = Field(alias="aggregateMetrics")
|
|
151
|
+
runtime_metrics: dict[str, Any] = Field(alias="runtimeMetrics")
|
|
152
|
+
completed_at: datetime | None = Field(default=None, alias="completedAt")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class LeaderboardListResponse(LeaderboardModel):
|
|
156
|
+
items: list[Leaderboard]
|
|
157
|
+
total: int
|
|
158
|
+
page: int
|
|
159
|
+
limit: int
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class LeaderboardSubmissionListResponse(LeaderboardModel):
|
|
163
|
+
items: list[LeaderboardSubmission]
|
|
164
|
+
total: int
|
|
165
|
+
page: int
|
|
166
|
+
limit: int
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
class LeaderboardRunListResponse(LeaderboardModel):
|
|
170
|
+
items: list[LeaderboardRun]
|
|
171
|
+
total: int
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
class LeaderboardEntryListResponse(LeaderboardModel):
|
|
175
|
+
items: list[LeaderboardEntry]
|
|
176
|
+
total: int
|
|
177
|
+
page: int
|
|
178
|
+
limit: int
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class ResourceUsage(BaseModel):
|
|
7
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
8
|
+
|
|
9
|
+
usage: float = 0
|
|
10
|
+
total: float = 0
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ClusterOverview(BaseModel):
|
|
14
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
15
|
+
|
|
16
|
+
cpu: ResourceUsage = Field(default_factory=ResourceUsage)
|
|
17
|
+
memory: ResourceUsage = Field(default_factory=ResourceUsage)
|
|
18
|
+
disk: ResourceUsage = Field(default_factory=ResourceUsage)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class ClusterTimeseriesData(BaseModel):
|
|
22
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
23
|
+
|
|
24
|
+
timestamps: list[str] = Field(default_factory=list)
|
|
25
|
+
cpu: list[float] = Field(default_factory=list)
|
|
26
|
+
memory: list[float] = Field(default_factory=list)
|
|
27
|
+
disk: list[float] = Field(default_factory=list)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ClusterMonitoring(BaseModel):
|
|
31
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
32
|
+
|
|
33
|
+
cluster_overview: ClusterOverview = Field(
|
|
34
|
+
default_factory=ClusterOverview, alias="clusterOverview"
|
|
35
|
+
)
|
|
36
|
+
cluster_timeseries: ClusterTimeseriesData = Field(
|
|
37
|
+
default_factory=ClusterTimeseriesData, alias="clusterTimeseries"
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class MonitoringDeviceInfo(BaseModel):
|
|
42
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
43
|
+
|
|
44
|
+
key: str
|
|
45
|
+
label: str
|
|
46
|
+
vendor: str
|
|
47
|
+
mem_label: str = Field(alias="memLabel")
|
|
48
|
+
resource_key: str = Field(alias="resourceKey")
|
|
49
|
+
total: int
|
|
50
|
+
in_use: int = Field(alias="inUse")
|
|
51
|
+
free: int
|
|
52
|
+
avg_utilization: float = Field(alias="avgUtilization")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class DeviceSummary(BaseModel):
|
|
56
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
57
|
+
|
|
58
|
+
total: int = 0
|
|
59
|
+
in_use: int = Field(default=0, alias="inUse")
|
|
60
|
+
free: int = 0
|
|
61
|
+
avg_utilization: float = Field(default=0, alias="avgUtilization")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class MonitoringDeviceAllocation(BaseModel):
|
|
65
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
66
|
+
|
|
67
|
+
resource_type: str = Field(alias="resourceType")
|
|
68
|
+
resource_name: str = Field(alias="resourceName")
|
|
69
|
+
project_name: str = Field(alias="projectName")
|
|
70
|
+
pod_name: str = Field(alias="podName")
|
|
71
|
+
namespace: str
|
|
72
|
+
owner: str | None = None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class MonitoringDevice(BaseModel):
|
|
76
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
77
|
+
|
|
78
|
+
index: str
|
|
79
|
+
status: str
|
|
80
|
+
usage: float = 0
|
|
81
|
+
temp: float = 0
|
|
82
|
+
power: float = 0
|
|
83
|
+
memory: float = 0
|
|
84
|
+
memory_used: float = Field(default=0, alias="memoryUsed")
|
|
85
|
+
memory_total: float = Field(default=0, alias="memoryTotal")
|
|
86
|
+
pod_name: str = Field(default="", alias="podName")
|
|
87
|
+
pod_namespace: str = Field(default="", alias="podNamespace")
|
|
88
|
+
allocation: MonitoringDeviceAllocation | None = None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class MonitoringNode(BaseModel):
|
|
92
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
93
|
+
|
|
94
|
+
name: str
|
|
95
|
+
cpu: float = 0
|
|
96
|
+
memory: float = 0
|
|
97
|
+
disk: float = 0
|
|
98
|
+
device_usage: float = Field(default=0, alias="deviceUsage")
|
|
99
|
+
devices_in_use: int = Field(default=0, alias="devicesInUse")
|
|
100
|
+
devices_total: int = Field(default=0, alias="devicesTotal")
|
|
101
|
+
devices: list[MonitoringDevice] = Field(default_factory=list)
|
|
102
|
+
network_rx: float | None = Field(default=None, alias="networkRx")
|
|
103
|
+
network_tx: float | None = Field(default=None, alias="networkTx")
|
|
104
|
+
network_unit: str | None = Field(default=None, alias="networkUnit")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class DeviceNodesResponse(BaseModel):
|
|
108
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
109
|
+
|
|
110
|
+
device_summary: DeviceSummary = Field(
|
|
111
|
+
default_factory=DeviceSummary, alias="deviceSummary"
|
|
112
|
+
)
|
|
113
|
+
nodes: list[MonitoringNode] = Field(default_factory=list)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class NodeTimeseriesData(BaseModel):
|
|
117
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
118
|
+
|
|
119
|
+
usage: list[float] = Field(default_factory=list)
|
|
120
|
+
temp: list[float] = Field(default_factory=list)
|
|
121
|
+
power: list[float] = Field(default_factory=list)
|
|
122
|
+
memory: list[float] = Field(default_factory=list)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class TimeseriesData(BaseModel):
|
|
126
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
127
|
+
|
|
128
|
+
timestamps: list[str] = Field(default_factory=list)
|
|
129
|
+
nodes: dict[str, NodeTimeseriesData] = Field(default_factory=dict)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class NodeResourceMetrics(BaseModel):
|
|
133
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
134
|
+
|
|
135
|
+
cpu: list[float] = Field(default_factory=list)
|
|
136
|
+
memory: list[float] = Field(default_factory=list)
|
|
137
|
+
disk: list[float] = Field(default_factory=list)
|
|
138
|
+
network_rx: list[float] = Field(default_factory=list, alias="networkRx")
|
|
139
|
+
network_tx: list[float] = Field(default_factory=list, alias="networkTx")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class NodeResourceTimeseriesData(BaseModel):
|
|
143
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
144
|
+
|
|
145
|
+
timestamps: list[str] = Field(default_factory=list)
|
|
146
|
+
nodes: dict[str, NodeResourceMetrics] = Field(default_factory=dict)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class DeviceTimeseriesResponse(BaseModel):
|
|
150
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
151
|
+
|
|
152
|
+
timeseries: TimeseriesData = Field(default_factory=TimeseriesData)
|
|
153
|
+
node_resource_timeseries: NodeResourceTimeseriesData = Field(
|
|
154
|
+
default_factory=NodeResourceTimeseriesData, alias="nodeResourceTimeseries"
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
class NodeTimeseriesResponse(BaseModel):
|
|
159
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
160
|
+
|
|
161
|
+
node_name: str = Field(alias="nodeName")
|
|
162
|
+
timestamps: list[str] = Field(default_factory=list)
|
|
163
|
+
cpu: list[float] = Field(default_factory=list)
|
|
164
|
+
memory: list[float] = Field(default_factory=list)
|
|
165
|
+
disk: list[float] = Field(default_factory=list)
|
|
166
|
+
network_rx: list[float] = Field(default_factory=list, alias="networkRx")
|
|
167
|
+
network_tx: list[float] = Field(default_factory=list, alias="networkTx")
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class SingleDeviceTimeseriesResponse(BaseModel):
|
|
171
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
172
|
+
|
|
173
|
+
node_name: str = Field(alias="nodeName")
|
|
174
|
+
device_index: str = Field(alias="deviceIndex")
|
|
175
|
+
timestamps: list[str] = Field(default_factory=list)
|
|
176
|
+
usage: list[float] = Field(default_factory=list)
|
|
177
|
+
memory: list[float] = Field(default_factory=list)
|
|
178
|
+
temperature: list[float] = Field(default_factory=list)
|
|
179
|
+
power: list[float] = Field(default_factory=list)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Generic, TypeVar
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, ConfigDict, model_validator
|
|
6
|
+
|
|
7
|
+
T = TypeVar("T")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Page(BaseModel, Generic[T]):
|
|
11
|
+
"""One server page with its pagination metadata."""
|
|
12
|
+
|
|
13
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
14
|
+
|
|
15
|
+
items: list[T]
|
|
16
|
+
total: int
|
|
17
|
+
page: int
|
|
18
|
+
limit: int
|
|
19
|
+
|
|
20
|
+
@model_validator(mode="before")
|
|
21
|
+
@classmethod
|
|
22
|
+
def _normalize_envelope(cls, value: object) -> object:
|
|
23
|
+
if not isinstance(value, dict):
|
|
24
|
+
return value
|
|
25
|
+
normalized = dict(value)
|
|
26
|
+
if "items" not in normalized:
|
|
27
|
+
for key in ("models", "versions", "artifacts", "jobs"):
|
|
28
|
+
if key in normalized:
|
|
29
|
+
normalized["items"] = normalized[key]
|
|
30
|
+
break
|
|
31
|
+
if "total" not in normalized and "totalCount" in normalized:
|
|
32
|
+
normalized["total"] = normalized["totalCount"]
|
|
33
|
+
return normalized
|
nufi/models/projects.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from typing import Literal
|
|
5
|
+
|
|
6
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Project(BaseModel):
|
|
10
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
11
|
+
|
|
12
|
+
id: int
|
|
13
|
+
name: str
|
|
14
|
+
created_at: datetime = Field(alias="createdAt")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ProjectAccessToken(BaseModel):
|
|
18
|
+
model_config = ConfigDict(extra="ignore", populate_by_name=True)
|
|
19
|
+
|
|
20
|
+
id: str
|
|
21
|
+
project_id: int = Field(alias="projectId")
|
|
22
|
+
name: str
|
|
23
|
+
created_by: str = Field(alias="createdBy")
|
|
24
|
+
created_by_name: str = Field(alias="createdByName")
|
|
25
|
+
created_at: datetime = Field(alias="createdAt")
|
|
26
|
+
expires_at: datetime = Field(alias="expiresAt")
|
|
27
|
+
revoked_at: datetime | None = Field(default=None, alias="revokedAt")
|
|
28
|
+
status: Literal["active", "expired", "revoked"]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ProjectAccessTokenCreateResponse(ProjectAccessToken):
|
|
32
|
+
token: str
|