nufi-python 1.1.3__tar.gz → 1.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. nufi_python-1.1.4/PKG-INFO +171 -0
  2. nufi_python-1.1.4/README.md +151 -0
  3. {nufi_python-1.1.3 → nufi_python-1.1.4}/pyproject.toml +1 -4
  4. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/__init__.py +22 -2
  5. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/cli/commands/common.py +45 -3
  6. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/cli/commands/datasets.py +32 -20
  7. nufi_python-1.1.4/src/nufi/cli/commands/evaluation_providers.py +290 -0
  8. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/cli/commands/evaluations.py +258 -245
  9. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/cli/commands/leaderboards.py +50 -34
  10. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/cli/commands/models.py +94 -70
  11. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/cli/commands/monitoring.py +69 -17
  12. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/cli/commands/servings.py +2 -0
  13. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/cli/main.py +2 -2
  14. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/exceptions.py +31 -0
  15. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/models/__init__.py +64 -40
  16. nufi_python-1.1.4/src/nufi/models/evaluation_providers.py +1034 -0
  17. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/models/evaluations.py +45 -149
  18. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/models/leaderboards.py +19 -9
  19. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/models/monitoring.py +63 -18
  20. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/models/registry.py +0 -8
  21. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/models/servings.py +2 -1
  22. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/resources/__init__.py +2 -2
  23. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/resources/base.py +38 -14
  24. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/resources/datasets.py +133 -40
  25. nufi_python-1.1.4/src/nufi/resources/evaluation_providers.py +200 -0
  26. nufi_python-1.1.4/src/nufi/resources/evaluations.py +637 -0
  27. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/resources/leaderboards.py +54 -9
  28. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/resources/models.py +1 -11
  29. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/resources/projects.py +2 -10
  30. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/resources/servings.py +6 -0
  31. nufi_python-1.1.4/src/nufi_python.egg-info/PKG-INFO +171 -0
  32. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi_python.egg-info/SOURCES.txt +3 -4
  33. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi_python.egg-info/requires.txt +0 -3
  34. nufi_python-1.1.3/PKG-INFO +0 -274
  35. nufi_python-1.1.3/README.md +0 -252
  36. nufi_python-1.1.3/src/nufi/cli/commands/custom_evaluation_templates.py +0 -214
  37. nufi_python-1.1.3/src/nufi/evaluator.py +0 -679
  38. nufi_python-1.1.3/src/nufi/models/custom_evaluation_templates.py +0 -324
  39. nufi_python-1.1.3/src/nufi/resources/custom_evaluation_templates.py +0 -126
  40. nufi_python-1.1.3/src/nufi/resources/evaluations.py +0 -423
  41. nufi_python-1.1.3/src/nufi_python.egg-info/PKG-INFO +0 -274
  42. {nufi_python-1.1.3 → nufi_python-1.1.4}/LICENSE +0 -0
  43. {nufi_python-1.1.3 → nufi_python-1.1.4}/setup.cfg +0 -0
  44. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/auth.py +0 -0
  45. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/cli/__init__.py +0 -0
  46. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/cli/commands/__init__.py +0 -0
  47. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/cli/commands/auth.py +0 -0
  48. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/cli/commands/compiles.py +0 -0
  49. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/cli/commands/doctor.py +0 -0
  50. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/cli/commands/profile.py +0 -0
  51. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/cli/commands/projects.py +0 -0
  52. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/client.py +0 -0
  53. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/config.py +0 -0
  54. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/http_utils.py +0 -0
  55. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/models/compiles.py +0 -0
  56. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/models/datasets.py +0 -0
  57. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/models/pagination.py +0 -0
  58. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/models/projects.py +0 -0
  59. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/py.typed +0 -0
  60. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/resources/compiles.py +0 -0
  61. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/resources/monitoring.py +0 -0
  62. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi/transport.py +0 -0
  63. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi_python.egg-info/dependency_links.txt +0 -0
  64. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi_python.egg-info/entry_points.txt +0 -0
  65. {nufi_python-1.1.3 → nufi_python-1.1.4}/src/nufi_python.egg-info/top_level.txt +0 -0
@@ -0,0 +1,171 @@
1
+ Metadata-Version: 2.4
2
+ Name: nufi-python
3
+ Version: 1.1.4
4
+ Summary: Python SDK and CLI for the NuFi platform.
5
+ Author: Dudaji Inc.
6
+ License-Expression: MIT
7
+ Classifier: Programming Language :: Python :: 3
8
+ Classifier: Operating System :: OS Independent
9
+ Classifier: Typing :: Typed
10
+ Requires-Python: >=3.11
11
+ Description-Content-Type: text/markdown
12
+ License-File: LICENSE
13
+ Requires-Dist: httpx>=0.25.0
14
+ Requires-Dist: PyYAML>=6.0
15
+ Requires-Dist: pydantic>=2.0.0
16
+ Requires-Dist: tuspy==1.1.0
17
+ Requires-Dist: typer>=0.16.0
18
+ Requires-Dist: rich>=13.0.0
19
+ Dynamic: license-file
20
+
21
+ # NuFi Python SDK/CLI
22
+
23
+ `nufi-python` is the Python SDK and CLI for a NuFi installation. It requires Python
24
+ 3.11 or newer and installs the `nufi` Python package and `nufi` command.
25
+
26
+ ## Installation
27
+
28
+ ```bash
29
+ python -m pip install nufi-python
30
+ nufi --version
31
+ nufi --help
32
+ ```
33
+
34
+ ## Configure access
35
+
36
+ You need a reachable NuFi installation, a user account, and access to a Project.
37
+ Ask your NuFi Administrator for the base domain and project name, then replace
38
+ `nufi.example` and `team-a` below:
39
+
40
+ ```bash
41
+ nufi profile create production --base-domain nufi.example
42
+ nufi profile use production
43
+ nufi login
44
+ nufi project use team-a
45
+ nufi model list
46
+ ```
47
+
48
+ `login` uses the device flow. Add `--no-browser` on a headless host and open the
49
+ printed verification URL elsewhere. A successful `model list`, even when empty,
50
+ confirms access to the selected Project. The SDK also uses this saved Profile.
51
+ See [installation and authentication](https://docs.nufi.me/dev/docs/next/sdk-cli/installation-and-authentication)
52
+ for other connection settings.
53
+
54
+ ## CLI quickstart
55
+
56
+ These examples assume a registered `qwen2` Model Version `v1` with an `original`
57
+ artifact, a compatible accelerator, and an existing Evaluation Method named
58
+ `METHOD_NAME`. Replace those names with your resources. The Leaderboard example
59
+ also assumes that the Method reports `exact_match` and that `quality` is a new
60
+ Leaderboard name. For model import and serving from scratch, follow the
61
+ [serving quickstart](https://docs.nufi.me/dev/docs/next/sdk-cli/quickstart).
62
+
63
+ Compile for RNGD when you need a compiled artifact:
64
+
65
+ ```bash
66
+ nufi compile create qwen2 --version v1 --wait
67
+ ```
68
+
69
+ Evaluate the original artifact, or submit it to a Leaderboard that starts its own
70
+ Evaluation Run:
71
+
72
+ ```bash
73
+ nufi evaluation run create METHOD_NAME --model qwen2 \
74
+ --model-version v1 --artifact original --wait
75
+ nufi leaderboard create quality --method METHOD_NAME \
76
+ --rank-metric exact_match --rank-direction desc
77
+ nufi leaderboard submission create quality --model qwen2 \
78
+ --model-version v1 --artifact original --wait
79
+ nufi leaderboard entries quality
80
+ ```
81
+
82
+ Compile succeeds at `Registered`; an Evaluation Run or a Submission's latest Run
83
+ succeeds at `succeeded`. The wait commands return a non-zero exit code for failure,
84
+ cancellation where supported, or timeout. A timeout stops local waiting and leaves
85
+ the server operation running. Resume with the printed name or ID:
86
+
87
+ ```bash
88
+ nufi compile wait COMPILE_NAME_OR_ID
89
+ nufi evaluation run wait RUN_ID
90
+ nufi leaderboard submission wait quality SUBMISSION_ID
91
+ ```
92
+
93
+ Use `--help` on a command for its options and `--output json` for structured
94
+ results. Evaluation detail JSON can include sensitive diagnostic inputs and
95
+ outputs; table output shows summaries. See the
96
+ [Evaluation reference](https://docs.nufi.me/dev/docs/next/sdk-cli/cli-reference/evaluation)
97
+ for Method creation, Provider selection, and result inspection.
98
+
99
+ ## Python SDK quickstart
100
+
101
+ After configuring the Profile above, use a context manager to close the client.
102
+ Replace `METHOD_ID` with an Evaluation Method UUID, `9` with your Model Artifact's
103
+ numeric ID, and `LEADERBOARD_ID` with the UUID of an existing Leaderboard such as
104
+ `quality` above. CLI `get` commands with `--output json` expose these IDs.
105
+
106
+ ```python
107
+ from nufi import NuFi
108
+
109
+ with NuFi.from_profile() as client:
110
+ compile_run = client.compiles.create(model_name="qwen2", version="v1")
111
+ compiled = client.compiles.wait(compile_run.id)
112
+ if compiled.phase != "Registered" or compiled.registration_error:
113
+ raise RuntimeError(compiled.error_message or compiled.registration_error or "Compile failed")
114
+
115
+ run = client.evaluations.create(
116
+ evaluation_method_id="METHOD_ID",
117
+ model_artifact_id=9,
118
+ cpu="4",
119
+ memory="8Gi",
120
+ accelerator_count=1,
121
+ )
122
+ evaluated = client.evaluations.wait(run.id)
123
+ if evaluated.status != "succeeded":
124
+ raise RuntimeError(f"Evaluation {run.id} ended in {evaluated.status}")
125
+ print(evaluated.aggregate_metrics)
126
+
127
+ submission = client.leaderboards.submit("LEADERBOARD_ID", model_artifact_id=9)
128
+ completed = client.leaderboards.wait("LEADERBOARD_ID", submission.id)
129
+ print(completed.latest_run.status)
130
+ ```
131
+
132
+ SDK `wait()` methods return terminal objects, including failed or cancelled Runs;
133
+ check their status before using results. Request errors raise
134
+ `nufi.exceptions.NuFiError` subclasses, and a wait timeout raises `TimeoutError`.
135
+ Save the returned IDs to resume waiting without creating another operation.
136
+ See the
137
+ [Python Evaluation reference](https://docs.nufi.me/dev/docs/next/sdk-cli/python-reference/evaluation)
138
+ for Method creation and Dataset replacement.
139
+
140
+ ## Dataset import and profiling
141
+
142
+ Dataset Import and Dataset Profiling complete independently. With an existing
143
+ `mmlu` Dataset and a project storage PVC named `DATASET_PVC`, import a version and
144
+ wait for its Profile before previewing or creating a Method that uses its fields:
145
+
146
+ ```bash
147
+ nufi dataset version create mmlu mmlu-aa-test --source huggingface \
148
+ --repository cais/mmlu --subset abstract_algebra --split test \
149
+ --storage-pvc DATASET_PVC --wait
150
+ nufi dataset version profile mmlu mmlu-aa-test --wait --output json
151
+ ```
152
+
153
+ The Hugging Face import requires cluster access to the source. Continue only when
154
+ the Profile is `ready`. In Python, use the import result's `dataset_version_id`
155
+ after checking that the import completed and supplied a version:
156
+
157
+ ```python
158
+ from nufi import NuFi
159
+
160
+ with NuFi.from_profile() as client:
161
+ profile = client.datasets.versions.wait_for_profile("DATASET_ID", "VERSION_ID")
162
+ if profile.status != "ready":
163
+ raise RuntimeError(profile.error or "Dataset Profiling failed")
164
+ ```
165
+
166
+ See the [Dataset reference](https://docs.nufi.me/dev/docs/next/sdk-cli/python-reference/dataset)
167
+ for import, upload, and recovery, and the
168
+ [Evaluation Method guide](https://docs.nufi.me/dev/docs/next/sdk-cli/cli-reference/evaluation#nufi-evaluation-method-create)
169
+ for canonical Dataset preparation and custom Dataset bindings.
170
+
171
+ The linked references describe the current development version of NuFi.
@@ -0,0 +1,151 @@
1
+ # NuFi Python SDK/CLI
2
+
3
+ `nufi-python` is the Python SDK and CLI for a NuFi installation. It requires Python
4
+ 3.11 or newer and installs the `nufi` Python package and `nufi` command.
5
+
6
+ ## Installation
7
+
8
+ ```bash
9
+ python -m pip install nufi-python
10
+ nufi --version
11
+ nufi --help
12
+ ```
13
+
14
+ ## Configure access
15
+
16
+ You need a reachable NuFi installation, a user account, and access to a Project.
17
+ Ask your NuFi Administrator for the base domain and project name, then replace
18
+ `nufi.example` and `team-a` below:
19
+
20
+ ```bash
21
+ nufi profile create production --base-domain nufi.example
22
+ nufi profile use production
23
+ nufi login
24
+ nufi project use team-a
25
+ nufi model list
26
+ ```
27
+
28
+ `login` uses the device flow. Add `--no-browser` on a headless host and open the
29
+ printed verification URL elsewhere. A successful `model list`, even when empty,
30
+ confirms access to the selected Project. The SDK also uses this saved Profile.
31
+ See [installation and authentication](https://docs.nufi.me/dev/docs/next/sdk-cli/installation-and-authentication)
32
+ for other connection settings.
33
+
34
+ ## CLI quickstart
35
+
36
+ These examples assume a registered `qwen2` Model Version `v1` with an `original`
37
+ artifact, a compatible accelerator, and an existing Evaluation Method named
38
+ `METHOD_NAME`. Replace those names with your resources. The Leaderboard example
39
+ also assumes that the Method reports `exact_match` and that `quality` is a new
40
+ Leaderboard name. For model import and serving from scratch, follow the
41
+ [serving quickstart](https://docs.nufi.me/dev/docs/next/sdk-cli/quickstart).
42
+
43
+ Compile for RNGD when you need a compiled artifact:
44
+
45
+ ```bash
46
+ nufi compile create qwen2 --version v1 --wait
47
+ ```
48
+
49
+ Evaluate the original artifact, or submit it to a Leaderboard that starts its own
50
+ Evaluation Run:
51
+
52
+ ```bash
53
+ nufi evaluation run create METHOD_NAME --model qwen2 \
54
+ --model-version v1 --artifact original --wait
55
+ nufi leaderboard create quality --method METHOD_NAME \
56
+ --rank-metric exact_match --rank-direction desc
57
+ nufi leaderboard submission create quality --model qwen2 \
58
+ --model-version v1 --artifact original --wait
59
+ nufi leaderboard entries quality
60
+ ```
61
+
62
+ Compile succeeds at `Registered`; an Evaluation Run or a Submission's latest Run
63
+ succeeds at `succeeded`. The wait commands return a non-zero exit code for failure,
64
+ cancellation where supported, or timeout. A timeout stops local waiting and leaves
65
+ the server operation running. Resume with the printed name or ID:
66
+
67
+ ```bash
68
+ nufi compile wait COMPILE_NAME_OR_ID
69
+ nufi evaluation run wait RUN_ID
70
+ nufi leaderboard submission wait quality SUBMISSION_ID
71
+ ```
72
+
73
+ Use `--help` on a command for its options and `--output json` for structured
74
+ results. Evaluation detail JSON can include sensitive diagnostic inputs and
75
+ outputs; table output shows summaries. See the
76
+ [Evaluation reference](https://docs.nufi.me/dev/docs/next/sdk-cli/cli-reference/evaluation)
77
+ for Method creation, Provider selection, and result inspection.
78
+
79
+ ## Python SDK quickstart
80
+
81
+ After configuring the Profile above, use a context manager to close the client.
82
+ Replace `METHOD_ID` with an Evaluation Method UUID, `9` with your Model Artifact's
83
+ numeric ID, and `LEADERBOARD_ID` with the UUID of an existing Leaderboard such as
84
+ `quality` above. CLI `get` commands with `--output json` expose these IDs.
85
+
86
+ ```python
87
+ from nufi import NuFi
88
+
89
+ with NuFi.from_profile() as client:
90
+ compile_run = client.compiles.create(model_name="qwen2", version="v1")
91
+ compiled = client.compiles.wait(compile_run.id)
92
+ if compiled.phase != "Registered" or compiled.registration_error:
93
+ raise RuntimeError(compiled.error_message or compiled.registration_error or "Compile failed")
94
+
95
+ run = client.evaluations.create(
96
+ evaluation_method_id="METHOD_ID",
97
+ model_artifact_id=9,
98
+ cpu="4",
99
+ memory="8Gi",
100
+ accelerator_count=1,
101
+ )
102
+ evaluated = client.evaluations.wait(run.id)
103
+ if evaluated.status != "succeeded":
104
+ raise RuntimeError(f"Evaluation {run.id} ended in {evaluated.status}")
105
+ print(evaluated.aggregate_metrics)
106
+
107
+ submission = client.leaderboards.submit("LEADERBOARD_ID", model_artifact_id=9)
108
+ completed = client.leaderboards.wait("LEADERBOARD_ID", submission.id)
109
+ print(completed.latest_run.status)
110
+ ```
111
+
112
+ SDK `wait()` methods return terminal objects, including failed or cancelled Runs;
113
+ check their status before using results. Request errors raise
114
+ `nufi.exceptions.NuFiError` subclasses, and a wait timeout raises `TimeoutError`.
115
+ Save the returned IDs to resume waiting without creating another operation.
116
+ See the
117
+ [Python Evaluation reference](https://docs.nufi.me/dev/docs/next/sdk-cli/python-reference/evaluation)
118
+ for Method creation and Dataset replacement.
119
+
120
+ ## Dataset import and profiling
121
+
122
+ Dataset Import and Dataset Profiling complete independently. With an existing
123
+ `mmlu` Dataset and a project storage PVC named `DATASET_PVC`, import a version and
124
+ wait for its Profile before previewing or creating a Method that uses its fields:
125
+
126
+ ```bash
127
+ nufi dataset version create mmlu mmlu-aa-test --source huggingface \
128
+ --repository cais/mmlu --subset abstract_algebra --split test \
129
+ --storage-pvc DATASET_PVC --wait
130
+ nufi dataset version profile mmlu mmlu-aa-test --wait --output json
131
+ ```
132
+
133
+ The Hugging Face import requires cluster access to the source. Continue only when
134
+ the Profile is `ready`. In Python, use the import result's `dataset_version_id`
135
+ after checking that the import completed and supplied a version:
136
+
137
+ ```python
138
+ from nufi import NuFi
139
+
140
+ with NuFi.from_profile() as client:
141
+ profile = client.datasets.versions.wait_for_profile("DATASET_ID", "VERSION_ID")
142
+ if profile.status != "ready":
143
+ raise RuntimeError(profile.error or "Dataset Profiling failed")
144
+ ```
145
+
146
+ See the [Dataset reference](https://docs.nufi.me/dev/docs/next/sdk-cli/python-reference/dataset)
147
+ for import, upload, and recovery, and the
148
+ [Evaluation Method guide](https://docs.nufi.me/dev/docs/next/sdk-cli/cli-reference/evaluation#nufi-evaluation-method-create)
149
+ for canonical Dataset preparation and custom Dataset bindings.
150
+
151
+ The linked references describe the current development version of NuFi.
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "nufi-python"
3
- version = "1.1.3"
3
+ version = "1.1.4"
4
4
  authors = [
5
5
  { name = "Dudaji Inc." },
6
6
  ]
@@ -22,9 +22,6 @@ dependencies = [
22
22
  "rich>=13.0.0",
23
23
  ]
24
24
 
25
- [project.optional-dependencies]
26
- evaluator = ["pyarrow>=24.0.0"]
27
-
28
25
  [build-system]
29
26
  requires = ["setuptools>=61.0"]
30
27
  build-backend = "setuptools.build_meta"
@@ -6,8 +6,18 @@ from nufi.models import (
6
6
  CompileRun,
7
7
  CompileRunSummary,
8
8
  DatasetVersionProfile,
9
+ EvaluationProvider,
10
+ EvaluationProviderCatalog,
9
11
  EvaluationMethod,
10
- EvaluationMethodTemplates,
12
+ EvaluationMethodPreview,
13
+ EvaluationMethodRequest,
14
+ EvaluationMethodCreateInput,
15
+ EvaluationProviderOutputMetric,
16
+ EvaluationDatasetPreparation,
17
+ EvaluationMethodSummary,
18
+ EvaluationProviderRelease,
19
+ EvaluationProviderReleaseManifest,
20
+ EvaluationProviderReleaseSummary,
11
21
  EvaluationRun,
12
22
  EvaluationRunSummary,
13
23
  HuggingFaceMapping,
@@ -36,8 +46,18 @@ __all__ = [
36
46
  "CompileRun",
37
47
  "CompileRunSummary",
38
48
  "DatasetVersionProfile",
49
+ "EvaluationProvider",
50
+ "EvaluationProviderCatalog",
39
51
  "EvaluationMethod",
40
- "EvaluationMethodTemplates",
52
+ "EvaluationMethodPreview",
53
+ "EvaluationMethodRequest",
54
+ "EvaluationMethodCreateInput",
55
+ "EvaluationProviderOutputMetric",
56
+ "EvaluationDatasetPreparation",
57
+ "EvaluationMethodSummary",
58
+ "EvaluationProviderRelease",
59
+ "EvaluationProviderReleaseManifest",
60
+ "EvaluationProviderReleaseSummary",
41
61
  "EvaluationRun",
42
62
  "EvaluationRunSummary",
43
63
  "HuggingFaceMapping",
@@ -6,10 +6,11 @@ import os
6
6
  from collections.abc import Callable
7
7
  from enum import Enum
8
8
  from typing import Any
9
+ from uuid import UUID
9
10
 
10
11
  import typer
11
12
  from rich.console import Console
12
- from rich.table import Table
13
+ from rich.table import Column, Table
13
14
  from rich.text import Text
14
15
 
15
16
  from nufi import NuFi
@@ -123,12 +124,52 @@ def print_table(
123
124
  width: int | None = None,
124
125
  show_lines: bool = False,
125
126
  ) -> None:
126
- table = Table(*columns, width=width, show_lines=show_lines)
127
+ column_values = [
128
+ [row[index] for row in rows if str(row[index])] for index in range(len(columns))
129
+ ]
130
+ uuid_columns = {
131
+ index
132
+ for index, values in enumerate(column_values)
133
+ if any(_is_uuid(value) for value in values)
134
+ }
135
+ uuid_only_columns = {
136
+ index
137
+ for index in uuid_columns
138
+ if all(_is_uuid(value) for value in column_values[index])
139
+ }
140
+ if uuid_columns:
141
+ minimum_width = (
142
+ sum(
143
+ 36 if index in uuid_columns else len(column)
144
+ for index, column in enumerate(columns)
145
+ )
146
+ + 3 * len(columns)
147
+ + 1
148
+ )
149
+ width = max(width or console.width, minimum_width)
150
+ table = Table(
151
+ *(
152
+ Column(column, min_width=36, no_wrap=index in uuid_only_columns)
153
+ if index in uuid_columns
154
+ else Column(column)
155
+ for index, column in enumerate(columns)
156
+ ),
157
+ width=width,
158
+ show_lines=show_lines,
159
+ )
127
160
  for row in rows:
128
161
  table.add_row(*(Text(str(value)) for value in row))
129
162
  console.print(table, width=width, crop=False)
130
163
 
131
164
 
165
+ def _is_uuid(value: object) -> bool:
166
+ text = str(value)
167
+ try:
168
+ return str(UUID(text)) == text.lower()
169
+ except ValueError:
170
+ return False
171
+
172
+
132
173
  def print_page_table(
133
174
  page: Page[Any],
134
175
  columns: list[str],
@@ -202,4 +243,5 @@ def handle_cli_error(exc: Exception, *, verbose: bool) -> None:
202
243
  else:
203
244
  detail = str(exc)
204
245
  error_console.print(f"Error: {detail}", style="bold red", markup=False)
205
- raise typer.Exit(code=124 if isinstance(exc, TimeoutError) else 1) from exc
246
+ timed_out = isinstance(exc, TimeoutError) or isinstance(exc.__cause__, TimeoutError)
247
+ raise typer.Exit(code=124 if timed_out else 1) from exc
@@ -43,26 +43,24 @@ def _is_uuid(value: str) -> bool:
43
43
  return True
44
44
 
45
45
 
46
+ def _get_dataset(client, dataset: str, *, project: str | int):
47
+ try:
48
+ return client.datasets.get(dataset, project=project)
49
+ except ValidationError as exc:
50
+ message = str(exc)
51
+ if message.startswith("Multiple datasets named "):
52
+ raise ValidationError(message.replace("datasets", "Datasets", 1)) from None
53
+ raise
54
+
55
+
46
56
  def _resolve_dataset_id(client, dataset: str, *, project: str | int) -> str:
47
- if _is_uuid(dataset):
48
- return dataset
49
- return common.resolve_named_resource(
50
- client.datasets.iter_all(project=project),
51
- dataset,
52
- resource="Dataset",
53
- ).id
57
+ return _get_dataset(client, dataset, project=project).id
54
58
 
55
59
 
56
60
  def _resolve_version_id(
57
61
  client, dataset_id: str, version: str, *, project: str | int
58
62
  ) -> str:
59
- if _is_uuid(version):
60
- return version
61
- return common.resolve_named_resource(
62
- client.datasets.versions.iter_all(dataset_id, project=project),
63
- version,
64
- resource="Dataset Version",
65
- ).id
63
+ return client.datasets.versions.get(dataset_id, version, project=project).id
66
64
 
67
65
 
68
66
  def _project_ref(
@@ -251,8 +249,7 @@ def get_dataset(
251
249
  try:
252
250
  resolved_project = _project_ref(project, project_id, profile_name)
253
251
  client = common.make_client(profile_name)
254
- dataset_id = _resolve_dataset_id(client, dataset, project=resolved_project)
255
- result = client.datasets.get(dataset_id, project=resolved_project)
252
+ result = _get_dataset(client, dataset, project=resolved_project)
256
253
  except Exception as exc:
257
254
  common.handle_cli_error(exc, verbose=verbose)
258
255
  finally:
@@ -288,6 +285,11 @@ def create_dataset(
288
285
  profile_name: str | None = typer.Option(
289
286
  None, "--profile", help="CLI Profile name."
290
287
  ),
288
+ output: common.OutputChoice = typer.Option(
289
+ common.OutputChoice.table,
290
+ "--output",
291
+ help="Output format.",
292
+ ),
291
293
  verbose: bool = typer.Option(False, "--verbose", help="Show detailed errors."),
292
294
  ):
293
295
  client = None
@@ -305,6 +307,10 @@ def create_dataset(
305
307
  if client is not None:
306
308
  client.close()
307
309
 
310
+ if output == common.OutputChoice.json:
311
+ common.print_json(dataset)
312
+ return
313
+
308
314
  common.console.print(f"[bold green]Dataset created:[/] {escape(dataset.name)}")
309
315
 
310
316
 
@@ -494,11 +500,8 @@ def get_version(
494
500
  resolved_project = _project_ref(project, project_id, profile_name)
495
501
  client = common.make_client(profile_name)
496
502
  dataset_id = _resolve_dataset_id(client, dataset, project=resolved_project)
497
- version_id = _resolve_version_id(
498
- client, dataset_id, version, project=resolved_project
499
- )
500
503
  result = client.datasets.versions.get(
501
- dataset_id, version_id, project=resolved_project
504
+ dataset_id, version, project=resolved_project
502
505
  )
503
506
  except Exception as exc:
504
507
  common.handle_cli_error(exc, verbose=verbose)
@@ -625,6 +628,11 @@ def create_version(
625
628
  profile_name: str | None = typer.Option(
626
629
  None, "--profile", help="CLI Profile name."
627
630
  ),
631
+ output: common.OutputChoice = typer.Option(
632
+ common.OutputChoice.table,
633
+ "--output",
634
+ help="Output format.",
635
+ ),
628
636
  verbose: bool = typer.Option(False, "--verbose", help="Show detailed errors."),
629
637
  ):
630
638
  client = None
@@ -702,6 +710,10 @@ def create_version(
702
710
  if client is not None:
703
711
  client.close()
704
712
 
713
+ if output == common.OutputChoice.json:
714
+ common.print_json(result)
715
+ return
716
+
705
717
  if source == "pvc":
706
718
  common.console.print(f"[bold green]Dataset Version created:[/] {escape(name)}")
707
719
  return