hf-msft-foundry 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hf_msft_foundry-0.1.0/LICENSE +25 -0
- hf_msft_foundry-0.1.0/PKG-INFO +173 -0
- hf_msft_foundry-0.1.0/README.md +156 -0
- hf_msft_foundry-0.1.0/pyproject.toml +54 -0
- hf_msft_foundry-0.1.0/pyproject.toml.orig +49 -0
- hf_msft_foundry-0.1.0/src/hf_msft_foundry/__init__.py +0 -0
- hf_msft_foundry-0.1.0/src/hf_msft_foundry/accounts.py +147 -0
- hf_msft_foundry-0.1.0/src/hf_msft_foundry/azure.py +47 -0
- hf_msft_foundry-0.1.0/src/hf_msft_foundry/cli.py +223 -0
- hf_msft_foundry-0.1.0/src/hf_msft_foundry/client.py +81 -0
- hf_msft_foundry-0.1.0/src/hf_msft_foundry/deployments.py +151 -0
- hf_msft_foundry-0.1.0/src/hf_msft_foundry/errors.py +16 -0
- hf_msft_foundry-0.1.0/src/hf_msft_foundry/pricing.py +38 -0
- hf_msft_foundry-0.1.0/src/hf_msft_foundry/py.typed +0 -0
- hf_msft_foundry-0.1.0/src/hf_msft_foundry/registry.py +158 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
Copyright (c) 2026-present Alvaro Bartolome <alvarobartt@gmail.com>
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any
|
|
4
|
+
person obtaining a copy of this software and associated
|
|
5
|
+
documentation files (the "Software"), to deal in the
|
|
6
|
+
Software without restriction, including without
|
|
7
|
+
limitation the rights to use, copy, modify, merge,
|
|
8
|
+
publish, distribute, sublicense, and/or sell copies of
|
|
9
|
+
the Software, and to permit persons to whom the Software
|
|
10
|
+
is furnished to do so, subject to the following
|
|
11
|
+
conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice
|
|
14
|
+
shall be included in all copies or substantial portions
|
|
15
|
+
of the Software.
|
|
16
|
+
|
|
17
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
|
|
18
|
+
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
|
|
19
|
+
TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
|
|
20
|
+
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
|
|
21
|
+
SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
|
22
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
23
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
|
|
24
|
+
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
25
|
+
DEALINGS IN THE SOFTWARE.
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hf-msft-foundry
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Deploy open models from the Hugging Face collection on Microsoft Foundry
|
|
5
|
+
Author: Alvaro Bartolome
|
|
6
|
+
Author-email: Alvaro Bartolome <36760800+alvarobartt@users.noreply.github.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: azure-identity>=1.25.1
|
|
10
|
+
Requires-Dist: azure-mgmt-cognitiveservices==15.1.0b1
|
|
11
|
+
Requires-Dist: requests>=2.32.0
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Project-URL: Homepage, https://github.com/alvarobartt/hf-msft-foundry
|
|
14
|
+
Project-URL: Repository, https://github.com/alvarobartt/hf-msft-foundry
|
|
15
|
+
Project-URL: Issues, https://github.com/alvarobartt/hf-msft-foundry/issues
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
# `hf-msft-foundry`
|
|
19
|
+
|
|
20
|
+
> [!WARNING]
|
|
21
|
+
> `hf-msft-foundry` is an experimental and independent open source project, not affiliated with or endorsed by Microsoft. Its interface may change without notice.
|
|
22
|
+
|
|
23
|
+
`hf-msft-foundry` is an extension for the [`hf` CLI](https://huggingface.co/docs/huggingface_hub/guides/cli) to deploy open models from the Hugging Face collection on [Microsoft Foundry](https://ai.azure.com), on managed compute within your own Azure subscription, powered by engines such as vLLM on NVIDIA A100, H100 and H200, or AMD Instinct MI300X.
|
|
24
|
+
|
|
25
|
+
It lets you browse the models and their deployment templates, check capacity and quota, and create, scale and delete deployments, all from the terminal and without having to go through the Microsoft Foundry portal.
|
|
26
|
+
|
|
27
|
+
## Installation
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
hf extensions install alvarobartt/hf-msft-foundry
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Alternatively, it can also be installed as a standalone CLI from [PyPI](https://pypi.org/project/hf-msft-foundry/), and used as `hf-msft-foundry` instead of `hf msft-foundry`:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
uv tool install hf-msft-foundry
|
|
37
|
+
# or
|
|
38
|
+
pip install hf-msft-foundry
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
It requires Python 3.10 or higher.
|
|
42
|
+
|
|
43
|
+
## Authentication
|
|
44
|
+
|
|
45
|
+
`hf-msft-foundry` authenticates with [`DefaultAzureCredential`](https://learn.microsoft.com/azure/developer/python/sdk/authentication/credential-chains#defaultazurecredential-overview), so either log in with the Azure CLI via `az login`, or set the environment variables for a service principal or a managed identity.
|
|
46
|
+
|
|
47
|
+
The subscription defaults to the one selected in the Azure CLI, and the Microsoft Foundry resource defaults to the only one in the subscription. If there's more than one, the commands fail listing them, so set them as follows:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
export AZURE_SUBSCRIPTION_ID=...
|
|
51
|
+
export FOUNDRY_ACCOUNT_NAME=...
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Or pass `--subscription-id` and `--account` to any command. If you don't have a Microsoft Foundry resource yet, you can create one with:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
hf msft-foundry accounts create my-foundry --resource-group my-foundry-rg --location eastus
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Getting started
|
|
61
|
+
|
|
62
|
+
Some models to get started with, on different sizes:
|
|
63
|
+
|
|
64
|
+
| Model | Size | Default deployment template |
|
|
65
|
+
|---|---|---|
|
|
66
|
+
| [`zai-org/GLM-5.3-Flash`](https://huggingface.co/zai-org/GLM-5.3-Flash) | 321B | 4 x AMD Instinct MI300X at 512K context |
|
|
67
|
+
| [`nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8`](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8) | 120B (12B active) | 2 x NVIDIA H100 at 256K context |
|
|
68
|
+
| [`google/gemma-4-26B-A4B-it`](https://huggingface.co/google/gemma-4-26B-A4B-it) | 26B (4B active) | 1 x NVIDIA A100 at 256K context |
|
|
69
|
+
|
|
70
|
+
Check whether a model is available, and which deployment templates i.e., the ready-to-use configurations for each instance and context length, can be used with it and at which price:
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
hf msft-foundry models --search "gemma 4"
|
|
74
|
+
hf msft-foundry models google/gemma-4-26B-A4B-it
|
|
75
|
+
hf msft-foundry models google/gemma-4-26B-A4B-it deployment-templates
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
```console
|
|
79
|
+
NAME ACCELERATOR_TYPE ACCELERATOR_COUNT MAX_MODEL_LEN ENVIRONMENT PRICE_PER_HOUR DEFAULT
|
|
80
|
+
google--gemma-4-26b-a4b-it--256k-amd-mi300x MI300_192GB 1 262144 hf-vllm-v0-33-0-amd 7.91 False
|
|
81
|
+
google--gemma-4-26b-a4b-it--256k-nvidia-a100 A100_80GB 1 262144 hf-vllm-v0-33-0-nvidia 3.95 True
|
|
82
|
+
google--gemma-4-26b-a4b-it--256k-nvidia-h100 H100_80GB 1 262144 hf-vllm-v0-33-0-nvidia 7.91 False
|
|
83
|
+
google--gemma-4-26b-a4b-it--256k-nvidia-h200 H200_141GB 1 262144 hf-vllm-v0-33-0-nvidia 9.1 False
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
The price is the on-demand list price in USD per hour and instance, from the [Azure Retail Prices API](https://learn.microsoft.com/rest/api/cost-management/retail-prices/azure-retail-prices). Both commands use the latest version of the model, unless `--version` is provided.
|
|
87
|
+
|
|
88
|
+
Check that your subscription has enough quota, and that there's capacity available, for the accelerators of the template you want to use:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
hf msft-foundry quota
|
|
92
|
+
hf msft-foundry capacity
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Then deploy it with its default deployment template, or with another one via `--deployment-template`, either with its full name or its suffix:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
hf msft-foundry deploy google/gemma-4-26B-A4B-it
|
|
99
|
+
hf msft-foundry deploy google/gemma-4-26B-A4B-it --deployment-template 256k-nvidia-h100
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
It waits until the deployment is ready, which can take 20+ minutes, unless `--no-wait` is provided. The deployment name defaults to `<author>--<model>--<7-digit timestamp>` e.g., `google--gemma-4-26b-a4b-it--5083872`, so that it doesn't collide with other deployments, and can be set with `--name` instead, using only letters, digits, underscores and hyphens.
|
|
103
|
+
|
|
104
|
+
Once deployed, you can send OpenAI-compatible Chat Completions requests to it, setting the deployment name as the `model`, with the endpoint and API key of your Microsoft Foundry resource:
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
export OPENAI_BASE_URL=$(hf msft-foundry accounts info -o json | jq -r .openai_base_url)
|
|
108
|
+
export OPENAI_API_KEY=$(hf msft-foundry accounts keys -o json | jq -r .key1)
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
from openai import OpenAI
|
|
113
|
+
|
|
114
|
+
client = OpenAI()
|
|
115
|
+
|
|
116
|
+
completion = client.chat.completions.create(
|
|
117
|
+
model="google--gemma-4-26b-a4b-it--5083872",
|
|
118
|
+
messages=[{"role": "user", "content": "What's the capital of France?"}],
|
|
119
|
+
max_completion_tokens=1024,
|
|
120
|
+
)
|
|
121
|
+
print(completion.choices[0].message.content)
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
> [!NOTE]
|
|
125
|
+
> Deployments can't reach the internet, so send images as base64-encoded data URIs rather than as URLs.
|
|
126
|
+
|
|
127
|
+
Finally, delete the deployment to stop paying for it:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
hf msft-foundry deployments delete google--gemma-4-26b-a4b-it--5083872 --wait
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
## Commands
|
|
134
|
+
|
|
135
|
+
| Command | Description |
|
|
136
|
+
|---|---|
|
|
137
|
+
| `models [--search]` | List the models in the Hugging Face collection |
|
|
138
|
+
| `models <model> [info] [--version]` | Show a model, its task, modalities and deployment templates |
|
|
139
|
+
| `models <model> deployment-templates [--version]` | List the deployment templates of a model, with their accelerators, context length and price |
|
|
140
|
+
| `deploy <model> [--version] [--deployment-template] [--name] [--instances] [--no-wait]` | Deploy a model |
|
|
141
|
+
| `deployments list` | List the deployments |
|
|
142
|
+
| `deployments status <name>` | Show a deployment |
|
|
143
|
+
| `deployments wait <name>` | Wait until a deployment succeeds |
|
|
144
|
+
| `deployments scale <name> --instances <n>` | Change the number of instances |
|
|
145
|
+
| `deployments delete <name> [--wait]` | Delete a deployment |
|
|
146
|
+
| `accounts list` | List the Microsoft Foundry resources |
|
|
147
|
+
| `accounts info` | Show the Microsoft Foundry resource and its endpoint |
|
|
148
|
+
| `accounts keys` | Show the API keys of the Microsoft Foundry resource |
|
|
149
|
+
| `accounts create <name> --resource-group --location` | Create a Microsoft Foundry resource and project |
|
|
150
|
+
| `capacity` | Show the available accelerators |
|
|
151
|
+
| `quota [--location]` | Show the managed compute quota |
|
|
152
|
+
|
|
153
|
+
All the commands accept `-o json` for machine-readable output, including errors, which makes `hf-msft-foundry` friendly to scripts and agents.
|
|
154
|
+
|
|
155
|
+
## Agent skill
|
|
156
|
+
|
|
157
|
+
The [`skills/hf-msft-foundry/SKILL.md`](https://github.com/alvarobartt/hf-msft-foundry/blob/main/skills/hf-msft-foundry/SKILL.md) skill teaches agents how to use `hf-msft-foundry` to find, deploy, use and delete models on Microsoft Foundry.
|
|
158
|
+
|
|
159
|
+
## Development
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
uv sync --group lint --group test
|
|
163
|
+
uv run pre-commit install
|
|
164
|
+
uv run pytest
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
The tests mock the Azure APIs, so they don't need an Azure subscription nor create any resource.
|
|
168
|
+
|
|
169
|
+
To release a new version, bump it with `uv version --bump <major|minor|patch>`, commit it, and push a matching `v<version>` tag, which publishes it to PyPI via [trusted publishing](https://docs.pypi.org/trusted-publishers/).
|
|
170
|
+
|
|
171
|
+
## License
|
|
172
|
+
|
|
173
|
+
MIT
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# `hf-msft-foundry`
|
|
2
|
+
|
|
3
|
+
> [!WARNING]
|
|
4
|
+
> `hf-msft-foundry` is an experimental and independent open source project, not affiliated with or endorsed by Microsoft. Its interface may change without notice.
|
|
5
|
+
|
|
6
|
+
`hf-msft-foundry` is an extension for the [`hf` CLI](https://huggingface.co/docs/huggingface_hub/guides/cli) to deploy open models from the Hugging Face collection on [Microsoft Foundry](https://ai.azure.com), on managed compute within your own Azure subscription, powered by engines such as vLLM on NVIDIA A100, H100 and H200, or AMD Instinct MI300X.
|
|
7
|
+
|
|
8
|
+
It lets you browse the models and their deployment templates, check capacity and quota, and create, scale and delete deployments, all from the terminal and without having to go through the Microsoft Foundry portal.
|
|
9
|
+
|
|
10
|
+
## Installation
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
hf extensions install alvarobartt/hf-msft-foundry
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
Alternatively, it can also be installed as a standalone CLI from [PyPI](https://pypi.org/project/hf-msft-foundry/), and used as `hf-msft-foundry` instead of `hf msft-foundry`:
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
uv tool install hf-msft-foundry
|
|
20
|
+
# or
|
|
21
|
+
pip install hf-msft-foundry
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
It requires Python 3.10 or higher.
|
|
25
|
+
|
|
26
|
+
## Authentication
|
|
27
|
+
|
|
28
|
+
`hf-msft-foundry` authenticates with [`DefaultAzureCredential`](https://learn.microsoft.com/azure/developer/python/sdk/authentication/credential-chains#defaultazurecredential-overview), so either log in with the Azure CLI via `az login`, or set the environment variables for a service principal or a managed identity.
|
|
29
|
+
|
|
30
|
+
The subscription defaults to the one selected in the Azure CLI, and the Microsoft Foundry resource defaults to the only one in the subscription. If there's more than one, the commands fail listing them, so set them as follows:
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
export AZURE_SUBSCRIPTION_ID=...
|
|
34
|
+
export FOUNDRY_ACCOUNT_NAME=...
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Or pass `--subscription-id` and `--account` to any command. If you don't have a Microsoft Foundry resource yet, you can create one with:
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
hf msft-foundry accounts create my-foundry --resource-group my-foundry-rg --location eastus
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Getting started
|
|
44
|
+
|
|
45
|
+
Some models to get started with, on different sizes:
|
|
46
|
+
|
|
47
|
+
| Model | Size | Default deployment template |
|
|
48
|
+
|---|---|---|
|
|
49
|
+
| [`zai-org/GLM-5.3-Flash`](https://huggingface.co/zai-org/GLM-5.3-Flash) | 321B | 4 x AMD Instinct MI300X at 512K context |
|
|
50
|
+
| [`nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8`](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8) | 120B (12B active) | 2 x NVIDIA H100 at 256K context |
|
|
51
|
+
| [`google/gemma-4-26B-A4B-it`](https://huggingface.co/google/gemma-4-26B-A4B-it) | 26B (4B active) | 1 x NVIDIA A100 at 256K context |
|
|
52
|
+
|
|
53
|
+
Check whether a model is available, and which deployment templates i.e., the ready-to-use configurations for each instance and context length, can be used with it and at which price:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
hf msft-foundry models --search "gemma 4"
|
|
57
|
+
hf msft-foundry models google/gemma-4-26B-A4B-it
|
|
58
|
+
hf msft-foundry models google/gemma-4-26B-A4B-it deployment-templates
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
```console
|
|
62
|
+
NAME ACCELERATOR_TYPE ACCELERATOR_COUNT MAX_MODEL_LEN ENVIRONMENT PRICE_PER_HOUR DEFAULT
|
|
63
|
+
google--gemma-4-26b-a4b-it--256k-amd-mi300x MI300_192GB 1 262144 hf-vllm-v0-33-0-amd 7.91 False
|
|
64
|
+
google--gemma-4-26b-a4b-it--256k-nvidia-a100 A100_80GB 1 262144 hf-vllm-v0-33-0-nvidia 3.95 True
|
|
65
|
+
google--gemma-4-26b-a4b-it--256k-nvidia-h100 H100_80GB 1 262144 hf-vllm-v0-33-0-nvidia 7.91 False
|
|
66
|
+
google--gemma-4-26b-a4b-it--256k-nvidia-h200 H200_141GB 1 262144 hf-vllm-v0-33-0-nvidia 9.1 False
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
The price is the on-demand list price in USD per hour and instance, from the [Azure Retail Prices API](https://learn.microsoft.com/rest/api/cost-management/retail-prices/azure-retail-prices). Both commands use the latest version of the model, unless `--version` is provided.
|
|
70
|
+
|
|
71
|
+
Check that your subscription has enough quota, and that there's capacity available, for the accelerators of the template you want to use:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
hf msft-foundry quota
|
|
75
|
+
hf msft-foundry capacity
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Then deploy it with its default deployment template, or with another one via `--deployment-template`, either with its full name or its suffix:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
hf msft-foundry deploy google/gemma-4-26B-A4B-it
|
|
82
|
+
hf msft-foundry deploy google/gemma-4-26B-A4B-it --deployment-template 256k-nvidia-h100
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
It waits until the deployment is ready, which can take 20+ minutes, unless `--no-wait` is provided. The deployment name defaults to `<author>--<model>--<7-digit timestamp>` e.g., `google--gemma-4-26b-a4b-it--5083872`, so that it doesn't collide with other deployments, and can be set with `--name` instead, using only letters, digits, underscores and hyphens.
|
|
86
|
+
|
|
87
|
+
Once deployed, you can send OpenAI-compatible Chat Completions requests to it, setting the deployment name as the `model`, with the endpoint and API key of your Microsoft Foundry resource:
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
export OPENAI_BASE_URL=$(hf msft-foundry accounts info -o json | jq -r .openai_base_url)
|
|
91
|
+
export OPENAI_API_KEY=$(hf msft-foundry accounts keys -o json | jq -r .key1)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
from openai import OpenAI
|
|
96
|
+
|
|
97
|
+
client = OpenAI()
|
|
98
|
+
|
|
99
|
+
completion = client.chat.completions.create(
|
|
100
|
+
model="google--gemma-4-26b-a4b-it--5083872",
|
|
101
|
+
messages=[{"role": "user", "content": "What's the capital of France?"}],
|
|
102
|
+
max_completion_tokens=1024,
|
|
103
|
+
)
|
|
104
|
+
print(completion.choices[0].message.content)
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
> [!NOTE]
|
|
108
|
+
> Deployments can't reach the internet, so send images as base64-encoded data URIs rather than as URLs.
|
|
109
|
+
|
|
110
|
+
Finally, delete the deployment to stop paying for it:
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
hf msft-foundry deployments delete google--gemma-4-26b-a4b-it--5083872 --wait
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## Commands
|
|
117
|
+
|
|
118
|
+
| Command | Description |
|
|
119
|
+
|---|---|
|
|
120
|
+
| `models [--search]` | List the models in the Hugging Face collection |
|
|
121
|
+
| `models <model> [info] [--version]` | Show a model, its task, modalities and deployment templates |
|
|
122
|
+
| `models <model> deployment-templates [--version]` | List the deployment templates of a model, with their accelerators, context length and price |
|
|
123
|
+
| `deploy <model> [--version] [--deployment-template] [--name] [--instances] [--no-wait]` | Deploy a model |
|
|
124
|
+
| `deployments list` | List the deployments |
|
|
125
|
+
| `deployments status <name>` | Show a deployment |
|
|
126
|
+
| `deployments wait <name>` | Wait until a deployment succeeds |
|
|
127
|
+
| `deployments scale <name> --instances <n>` | Change the number of instances |
|
|
128
|
+
| `deployments delete <name> [--wait]` | Delete a deployment |
|
|
129
|
+
| `accounts list` | List the Microsoft Foundry resources |
|
|
130
|
+
| `accounts info` | Show the Microsoft Foundry resource and its endpoint |
|
|
131
|
+
| `accounts keys` | Show the API keys of the Microsoft Foundry resource |
|
|
132
|
+
| `accounts create <name> --resource-group --location` | Create a Microsoft Foundry resource and project |
|
|
133
|
+
| `capacity` | Show the available accelerators |
|
|
134
|
+
| `quota [--location]` | Show the managed compute quota |
|
|
135
|
+
|
|
136
|
+
All the commands accept `-o json` for machine-readable output, including errors, which makes `hf-msft-foundry` friendly to scripts and agents.
|
|
137
|
+
|
|
138
|
+
## Agent skill
|
|
139
|
+
|
|
140
|
+
The [`skills/hf-msft-foundry/SKILL.md`](https://github.com/alvarobartt/hf-msft-foundry/blob/main/skills/hf-msft-foundry/SKILL.md) skill teaches agents how to use `hf-msft-foundry` to find, deploy, use and delete models on Microsoft Foundry.
|
|
141
|
+
|
|
142
|
+
## Development
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
uv sync --group lint --group test
|
|
146
|
+
uv run pre-commit install
|
|
147
|
+
uv run pytest
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
The tests mock the Azure APIs, so they don't need an Azure subscription nor create any resource.
|
|
151
|
+
|
|
152
|
+
To release a new version, bump it with `uv version --bump <major|minor|patch>`, commit it, and push a matching `v<version>` tag, which publishes it to PyPI via [trusted publishing](https://docs.pypi.org/trusted-publishers/).
|
|
153
|
+
|
|
154
|
+
## License
|
|
155
|
+
|
|
156
|
+
MIT
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "hf-msft-foundry"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Deploy open models from the Hugging Face collection on Microsoft Foundry"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
requires-python = ">=3.10"
|
|
9
|
+
dependencies = [
|
|
10
|
+
"azure-identity>=1.25.1",
|
|
11
|
+
"azure-mgmt-cognitiveservices==15.1.0b1",
|
|
12
|
+
"requests>=2.32.0",
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
[[project.authors]]
|
|
16
|
+
name = "Alvaro Bartolome"
|
|
17
|
+
email = "36760800+alvarobartt@users.noreply.github.com"
|
|
18
|
+
|
|
19
|
+
[project.urls]
|
|
20
|
+
Homepage = "https://github.com/alvarobartt/hf-msft-foundry"
|
|
21
|
+
Repository = "https://github.com/alvarobartt/hf-msft-foundry"
|
|
22
|
+
Issues = "https://github.com/alvarobartt/hf-msft-foundry/issues"
|
|
23
|
+
|
|
24
|
+
[project.scripts]
|
|
25
|
+
hf-msft-foundry = "hf_msft_foundry.cli:main"
|
|
26
|
+
|
|
27
|
+
[[tool.uv.index]]
|
|
28
|
+
url = "https://pypi.org/simple"
|
|
29
|
+
default = true
|
|
30
|
+
|
|
31
|
+
[tool.ruff]
|
|
32
|
+
line-length = 119
|
|
33
|
+
|
|
34
|
+
[tool.ruff.lint]
|
|
35
|
+
select = [
|
|
36
|
+
"E",
|
|
37
|
+
"F",
|
|
38
|
+
"I",
|
|
39
|
+
"UP",
|
|
40
|
+
"B",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
[build-system]
|
|
44
|
+
requires = ["uv_build>=0.9.5,<0.11.0"]
|
|
45
|
+
build-backend = "uv_build"
|
|
46
|
+
|
|
47
|
+
[dependency-groups]
|
|
48
|
+
lint = [
|
|
49
|
+
"pre-commit>=4.5.1",
|
|
50
|
+
"ruff>=0.14.10",
|
|
51
|
+
"ty>=0.0.1a32",
|
|
52
|
+
"types-requests>=2.32.0",
|
|
53
|
+
]
|
|
54
|
+
test = ["pytest>=8.4.0"]
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "hf-msft-foundry"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Deploy open models from the Hugging Face collection on Microsoft Foundry"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
authors = [
|
|
7
|
+
{ name = "Alvaro Bartolome", email = "36760800+alvarobartt@users.noreply.github.com" }
|
|
8
|
+
]
|
|
9
|
+
license = "MIT"
|
|
10
|
+
license-files = ["LICENSE"]
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
dependencies = [
|
|
13
|
+
"azure-identity>=1.25.1",
|
|
14
|
+
"azure-mgmt-cognitiveservices==15.1.0b1",
|
|
15
|
+
"requests>=2.32.0",
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
[project.urls]
|
|
19
|
+
Homepage = "https://github.com/alvarobartt/hf-msft-foundry"
|
|
20
|
+
Repository = "https://github.com/alvarobartt/hf-msft-foundry"
|
|
21
|
+
Issues = "https://github.com/alvarobartt/hf-msft-foundry/issues"
|
|
22
|
+
|
|
23
|
+
[project.scripts]
|
|
24
|
+
hf-msft-foundry = "hf_msft_foundry.cli:main"
|
|
25
|
+
|
|
26
|
+
[[tool.uv.index]]
|
|
27
|
+
url = "https://pypi.org/simple"
|
|
28
|
+
default = true
|
|
29
|
+
|
|
30
|
+
[build-system]
|
|
31
|
+
requires = ["uv_build>=0.9.5,<0.11.0"]
|
|
32
|
+
build-backend = "uv_build"
|
|
33
|
+
|
|
34
|
+
[dependency-groups]
|
|
35
|
+
lint = [
|
|
36
|
+
"pre-commit>=4.5.1",
|
|
37
|
+
"ruff>=0.14.10",
|
|
38
|
+
"ty>=0.0.1a32",
|
|
39
|
+
"types-requests>=2.32.0",
|
|
40
|
+
]
|
|
41
|
+
test = [
|
|
42
|
+
"pytest>=8.4.0",
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
[tool.ruff]
|
|
46
|
+
line-length = 119
|
|
47
|
+
|
|
48
|
+
[tool.ruff.lint]
|
|
49
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
File without changes
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from azure.mgmt.cognitiveservices import CognitiveServicesManagementClient
|
|
6
|
+
from azure.mgmt.cognitiveservices.models import (
|
|
7
|
+
Account as AccountResource,
|
|
8
|
+
)
|
|
9
|
+
from azure.mgmt.cognitiveservices.models import (
|
|
10
|
+
AccountProperties,
|
|
11
|
+
Identity,
|
|
12
|
+
Project,
|
|
13
|
+
ProjectProperties,
|
|
14
|
+
Sku,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
from hf_msft_foundry.azure import ARM, AzureClient, azure_errors
|
|
18
|
+
from hf_msft_foundry.errors import NotFoundError
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger("hf_msft_foundry")
|
|
21
|
+
|
|
22
|
+
OFFER = "GlobalManagedCompute"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class Account:
|
|
27
|
+
subscription_id: str
|
|
28
|
+
resource_group: str
|
|
29
|
+
name: str
|
|
30
|
+
location: str
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def endpoint(self) -> str:
|
|
34
|
+
return f"https://{self.name}.services.ai.azure.com"
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def openai_base_url(self) -> str:
|
|
38
|
+
return f"{self.endpoint}/openai/v1"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def resource_group(resource_id: str) -> str:
|
|
42
|
+
return resource_id.split("/resourceGroups/", 1)[1].split("/", 1)[0]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Accounts:
|
|
46
|
+
def __init__(
|
|
47
|
+
self, management: CognitiveServicesManagementClient, azure: AzureClient, subscription_id: str
|
|
48
|
+
) -> None:
|
|
49
|
+
self.management = management
|
|
50
|
+
self.azure = azure
|
|
51
|
+
self.subscription_id = subscription_id
|
|
52
|
+
|
|
53
|
+
def all(self) -> list[Account]:
|
|
54
|
+
with azure_errors(f"Subscription {self.subscription_id} not found"):
|
|
55
|
+
return [
|
|
56
|
+
Account(self.subscription_id, resource_group(item.id or ""), item.name or "", item.location or "")
|
|
57
|
+
for item in self.management.accounts.list()
|
|
58
|
+
if item.kind == "AIServices"
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
def get(self, name: str | None = None) -> Account:
|
|
62
|
+
accounts = self.all()
|
|
63
|
+
if name:
|
|
64
|
+
for account in accounts:
|
|
65
|
+
if account.name == name:
|
|
66
|
+
return account
|
|
67
|
+
raise NotFoundError(
|
|
68
|
+
f"Microsoft Foundry resource '{name}' not found in subscription {self.subscription_id}"
|
|
69
|
+
)
|
|
70
|
+
if len(accounts) == 1:
|
|
71
|
+
return accounts[0]
|
|
72
|
+
if not accounts:
|
|
73
|
+
raise NotFoundError(
|
|
74
|
+
f"No Microsoft Foundry resources found in subscription {self.subscription_id}, "
|
|
75
|
+
"create one with `hf msft-foundry accounts create`"
|
|
76
|
+
)
|
|
77
|
+
names = ", ".join(account.name for account in accounts)
|
|
78
|
+
raise ValueError(f"Found {len(accounts)} Microsoft Foundry resources, set --account to one of: {names}")
|
|
79
|
+
|
|
80
|
+
def create(self, name: str, resource_group: str, location: str, project: str | None = None) -> Account:
|
|
81
|
+
logger.info("Creating resource group '%s' in %s", resource_group, location)
|
|
82
|
+
self.azure.request(
|
|
83
|
+
"PUT",
|
|
84
|
+
f"{ARM}/subscriptions/{self.subscription_id}/resourcegroups/{resource_group}",
|
|
85
|
+
params={"api-version": "2021-04-01"},
|
|
86
|
+
json={"location": location},
|
|
87
|
+
)
|
|
88
|
+
logger.info("Creating Microsoft Foundry resource '%s'", name)
|
|
89
|
+
with azure_errors(f"Resource group '{resource_group}' not found"):
|
|
90
|
+
self.management.accounts.begin_create(
|
|
91
|
+
resource_group,
|
|
92
|
+
name,
|
|
93
|
+
AccountResource(
|
|
94
|
+
location=location,
|
|
95
|
+
kind="AIServices",
|
|
96
|
+
sku=Sku(name="S0"),
|
|
97
|
+
identity=Identity(type="SystemAssigned"),
|
|
98
|
+
properties=AccountProperties(
|
|
99
|
+
custom_sub_domain_name=name,
|
|
100
|
+
allow_project_management=True,
|
|
101
|
+
public_network_access="Enabled",
|
|
102
|
+
),
|
|
103
|
+
),
|
|
104
|
+
).result()
|
|
105
|
+
project = project or f"{name}-project"
|
|
106
|
+
logger.info("Creating Microsoft Foundry project '%s'", project)
|
|
107
|
+
self.management.projects.begin_create(
|
|
108
|
+
resource_group,
|
|
109
|
+
name,
|
|
110
|
+
project,
|
|
111
|
+
Project(location=location, identity=Identity(type="SystemAssigned"), properties=ProjectProperties()),
|
|
112
|
+
).result()
|
|
113
|
+
return Account(self.subscription_id, resource_group, name, location)
|
|
114
|
+
|
|
115
|
+
def keys(self, account: Account) -> dict[str, str]:
|
|
116
|
+
with azure_errors(f"Microsoft Foundry resource '{account.name}' not found"):
|
|
117
|
+
keys = self.management.accounts.list_keys(account.resource_group, account.name)
|
|
118
|
+
return {"key1": keys.key1 or "", "key2": keys.key2 or ""}
|
|
119
|
+
|
|
120
|
+
def capacity(self) -> list[dict[str, Any]]:
|
|
121
|
+
with azure_errors(f"Subscription {self.subscription_id} not found"):
|
|
122
|
+
items = [item.as_dict() for item in self.management.managed_compute_capacities.list(offer=OFFER)]
|
|
123
|
+
return [
|
|
124
|
+
{
|
|
125
|
+
"accelerator_type": item["properties"]["acceleratorType"],
|
|
126
|
+
"available": item["properties"]["availableAccelerators"],
|
|
127
|
+
"largest_deployment": {
|
|
128
|
+
size["modelInstanceAcceleratorCount"]: size["largestDeploymentCapacity"]
|
|
129
|
+
for size in item["properties"].get("deploymentSizeCapacities", [])
|
|
130
|
+
},
|
|
131
|
+
}
|
|
132
|
+
for item in items
|
|
133
|
+
if item["properties"].get("offerScope") == "Global"
|
|
134
|
+
]
|
|
135
|
+
|
|
136
|
+
def quota(self, location: str) -> list[dict[str, Any]]:
|
|
137
|
+
with azure_errors(f"Location '{location}' not found"):
|
|
138
|
+
items = [item.as_dict() for item in self.management.managed_compute_usages_operation_group.list(location)]
|
|
139
|
+
return [
|
|
140
|
+
{
|
|
141
|
+
"accelerator_type": item["name"]["value"].rsplit(".", 1)[-1],
|
|
142
|
+
"used": item["currentValue"],
|
|
143
|
+
"limit": item["limit"],
|
|
144
|
+
}
|
|
145
|
+
for item in items
|
|
146
|
+
if item.get("offerScope") == "Global"
|
|
147
|
+
]
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from collections.abc import Generator
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import requests
|
|
6
|
+
from azure.core.credentials import TokenCredential
|
|
7
|
+
from azure.core.exceptions import HttpResponseError, ResourceNotFoundError
|
|
8
|
+
from azure.identity import DefaultAzureCredential
|
|
9
|
+
|
|
10
|
+
from hf_msft_foundry.errors import AzureRequestError, NotFoundError
|
|
11
|
+
|
|
12
|
+
ARM = "https://management.azure.com"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@contextmanager
|
|
16
|
+
def azure_errors(not_found: str) -> Generator[None]:
|
|
17
|
+
try:
|
|
18
|
+
yield
|
|
19
|
+
except ResourceNotFoundError as error:
|
|
20
|
+
raise NotFoundError(not_found) from error
|
|
21
|
+
except HttpResponseError as error:
|
|
22
|
+
raise AzureRequestError(error.status_code or 500, error.message) from error
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class AzureClient:
|
|
26
|
+
def __init__(self, credential: TokenCredential | None = None, session: requests.Session | None = None) -> None:
|
|
27
|
+
self.credential = credential or DefaultAzureCredential(exclude_interactive_browser_credential=True)
|
|
28
|
+
self.session = session or requests.Session()
|
|
29
|
+
|
|
30
|
+
def request(self, method: str, url: str, **kwargs: Any) -> Any:
|
|
31
|
+
token = self.credential.get_token(f"{ARM}/.default").token
|
|
32
|
+
headers = {"Authorization": f"Bearer {token}"}
|
|
33
|
+
response = self.session.request(method, url, headers=headers, timeout=120, **kwargs)
|
|
34
|
+
if response.status_code == 404:
|
|
35
|
+
raise NotFoundError(f"{url.split('?')[0]} does not exist")
|
|
36
|
+
if not response.ok:
|
|
37
|
+
raise AzureRequestError(response.status_code, response.text or response.reason)
|
|
38
|
+
return response.json() if response.content else {}
|
|
39
|
+
|
|
40
|
+
def paginate(self, url: str, params: dict[str, str] | None = None) -> list[dict[str, Any]]:
|
|
41
|
+
items: list[dict[str, Any]] = []
|
|
42
|
+
next_url: str | None = url
|
|
43
|
+
while next_url:
|
|
44
|
+
data = self.request("GET", next_url, params=params)
|
|
45
|
+
items.extend(data.get("value", []))
|
|
46
|
+
next_url, params = data.get("nextLink"), None
|
|
47
|
+
return items
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
import sys
|
|
5
|
+
from dataclasses import asdict, is_dataclass
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from azure.core.exceptions import ClientAuthenticationError
|
|
9
|
+
|
|
10
|
+
from hf_msft_foundry.client import Foundry
|
|
11
|
+
from hf_msft_foundry.errors import FoundryError
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger("hf_msft_foundry")
|
|
14
|
+
|
|
15
|
+
TEMPLATE_COLUMNS = [
|
|
16
|
+
"name",
|
|
17
|
+
"accelerator_type",
|
|
18
|
+
"accelerator_count",
|
|
19
|
+
"max_model_len",
|
|
20
|
+
"environment",
|
|
21
|
+
"price_per_hour",
|
|
22
|
+
"default",
|
|
23
|
+
]
|
|
24
|
+
DEPLOYMENT_COLUMNS = ["name", "state", "model", "template", "accelerator_type", "instances"]
|
|
25
|
+
EXPORTED_PROPERTIES = ("uri", "url", "endpoint", "openai_base_url")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def serialize(data: Any) -> Any:
|
|
29
|
+
if isinstance(data, list):
|
|
30
|
+
return [serialize(item) for item in data]
|
|
31
|
+
if is_dataclass(data) and not isinstance(data, type):
|
|
32
|
+
values = asdict(data)
|
|
33
|
+
for prop in EXPORTED_PROPERTIES:
|
|
34
|
+
if isinstance(getattr(type(data), prop, None), property):
|
|
35
|
+
values[prop] = getattr(data, prop)
|
|
36
|
+
return values
|
|
37
|
+
return data
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def table(rows: list[dict[str, Any]], columns: list[str]) -> str:
|
|
41
|
+
cells = [["" if row.get(c) is None else str(row.get(c)) for c in columns] for row in rows]
|
|
42
|
+
widths = [max(len(c), *(len(r[i]) for r in cells)) for i, c in enumerate(columns)]
|
|
43
|
+
lines = [[c.upper() for c in columns], *cells]
|
|
44
|
+
return "\n".join(" ".join(v.ljust(w) for v, w in zip(line, widths, strict=True)).rstrip() for line in lines)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def render(data: Any, columns: list[str] | None) -> str:
|
|
48
|
+
if isinstance(data, dict):
|
|
49
|
+
width = max(len(k) for k in data)
|
|
50
|
+
return "\n".join(
|
|
51
|
+
f"{k.ljust(width)} {', '.join(map(str, v)) if isinstance(v, list) else ('' if v is None else v)}"
|
|
52
|
+
for k, v in data.items()
|
|
53
|
+
)
|
|
54
|
+
if columns:
|
|
55
|
+
return table(data, columns)
|
|
56
|
+
return "\n".join(map(str, data))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def emit(args: argparse.Namespace, data: Any, columns: list[str] | None = None, empty: str = "") -> None:
|
|
60
|
+
data = serialize(data)
|
|
61
|
+
if args.output == "json":
|
|
62
|
+
sys.stdout.write(json.dumps(data, indent=2) + "\n")
|
|
63
|
+
elif not data:
|
|
64
|
+
logger.warning(empty)
|
|
65
|
+
else:
|
|
66
|
+
sys.stdout.write(render(data, columns) + "\n")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def run(args: argparse.Namespace) -> None:
|
|
70
|
+
foundry = Foundry(subscription_id=args.subscription_id, account_name=args.account)
|
|
71
|
+
if args.command == "models" and args.model_id is not None and args.search:
|
|
72
|
+
raise ValueError("Use either a model ID or --search, not both")
|
|
73
|
+
action = getattr(args, "action", None)
|
|
74
|
+
if args.command == "models" and args.model_id is not None:
|
|
75
|
+
action = action or "info"
|
|
76
|
+
match args.command, action:
|
|
77
|
+
case "models", None if args.model_id is None:
|
|
78
|
+
scope = f"match '{args.search}'" if args.search else "are available"
|
|
79
|
+
empty = f"No models {scope} in the Hugging Face collection on Microsoft Foundry"
|
|
80
|
+
emit(args, foundry.registry.models(args.search), empty=empty)
|
|
81
|
+
case "models", "info":
|
|
82
|
+
emit(args, foundry.registry.model(args.model_id, args.version))
|
|
83
|
+
case "models", "deployment-templates":
|
|
84
|
+
model = foundry.registry.model(args.model_id, args.version)
|
|
85
|
+
empty = f"{args.model_id} has no deployment templates"
|
|
86
|
+
emit(args, foundry.registry.templates(model, foundry.prices), TEMPLATE_COLUMNS, empty)
|
|
87
|
+
case "deploy", _:
|
|
88
|
+
deployment = foundry.deploy(
|
|
89
|
+
args.model_id, args.deployment_template, args.name, args.instances, args.version
|
|
90
|
+
)
|
|
91
|
+
emit(args, deployment if args.no_wait else foundry.deployments.wait(deployment.name))
|
|
92
|
+
case "deployments", "list":
|
|
93
|
+
empty = f"No deployments in the Microsoft Foundry resource '{foundry.account.name}'"
|
|
94
|
+
emit(args, foundry.deployments.all(), DEPLOYMENT_COLUMNS, empty)
|
|
95
|
+
case "deployments", "status":
|
|
96
|
+
emit(args, foundry.deployments.get(args.name))
|
|
97
|
+
case "deployments", "wait":
|
|
98
|
+
emit(args, foundry.deployments.wait(args.name))
|
|
99
|
+
case "deployments", "scale":
|
|
100
|
+
deployment = foundry.deployments.update(args.name, args.instances)
|
|
101
|
+
emit(args, deployment if args.no_wait else foundry.deployments.wait(deployment.name))
|
|
102
|
+
case "deployments", "delete":
|
|
103
|
+
foundry.deployments.delete(args.name)
|
|
104
|
+
if args.wait:
|
|
105
|
+
foundry.deployments.wait_deleted(args.name)
|
|
106
|
+
case "accounts", "list":
|
|
107
|
+
empty = f"No Microsoft Foundry resources found in subscription {foundry.subscription_id}"
|
|
108
|
+
emit(args, foundry.accounts.all(), ["name", "resource_group", "location", "endpoint"], empty)
|
|
109
|
+
case "accounts", "info":
|
|
110
|
+
emit(args, foundry.account)
|
|
111
|
+
case "accounts", "create":
|
|
112
|
+
emit(args, foundry.accounts.create(args.name, args.resource_group, args.location, args.project))
|
|
113
|
+
case "accounts", "keys":
|
|
114
|
+
emit(args, foundry.accounts.keys(foundry.account))
|
|
115
|
+
case "capacity", _:
|
|
116
|
+
empty = f"No managed compute capacity available in subscription {foundry.subscription_id}"
|
|
117
|
+
emit(args, foundry.accounts.capacity(), ["accelerator_type", "available", "largest_deployment"], empty)
|
|
118
|
+
case "quota", _:
|
|
119
|
+
location = args.location or foundry.account.location
|
|
120
|
+
empty = f"No managed compute quota in {location} for subscription {foundry.subscription_id}"
|
|
121
|
+
emit(args, foundry.accounts.quota(location), ["accelerator_type", "used", "limit"], empty)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def parser() -> argparse.ArgumentParser:
|
|
125
|
+
common = argparse.ArgumentParser(add_help=False)
|
|
126
|
+
common.add_argument("--subscription-id", help="Azure subscription ID, defaults to AZURE_SUBSCRIPTION_ID")
|
|
127
|
+
common.add_argument("--account", help="Microsoft Foundry resource name, defaults to FOUNDRY_ACCOUNT_NAME")
|
|
128
|
+
common.add_argument(
|
|
129
|
+
"-o", "--output", choices=["table", "json"], default="table", help="Output format, json also applies to errors"
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
root = argparse.ArgumentParser(
|
|
133
|
+
prog="hf msft-foundry",
|
|
134
|
+
description="Deploy open models from the Hugging Face collection on Microsoft Foundry",
|
|
135
|
+
)
|
|
136
|
+
commands = root.add_subparsers(dest="command", required=True)
|
|
137
|
+
|
|
138
|
+
def group(name: str, help: str) -> argparse._SubParsersAction:
|
|
139
|
+
return commands.add_parser(name, help=help).add_subparsers(dest="action", required=True)
|
|
140
|
+
|
|
141
|
+
models = commands.add_parser(
|
|
142
|
+
"models",
|
|
143
|
+
parents=[common],
|
|
144
|
+
help="Browse the Hugging Face collection on Microsoft Foundry",
|
|
145
|
+
description="List the models, or show a model and its deployment templates, e.g. "
|
|
146
|
+
"`hf msft-foundry models google/gemma-4-31B-it deployment-templates`",
|
|
147
|
+
)
|
|
148
|
+
models.add_argument("model_id", nargs="?", help="Hub ID, e.g. google/gemma-4-31B-it, lists the models if omitted")
|
|
149
|
+
models.add_argument("action", nargs="?", choices=["info", "deployment-templates"], help="Defaults to info")
|
|
150
|
+
models.add_argument("--search", help="Filter the models by name, e.g. 'gemma 4'")
|
|
151
|
+
models.add_argument("--version", help="Model version, defaults to the latest")
|
|
152
|
+
|
|
153
|
+
deploy = commands.add_parser("deploy", parents=[common], help="Deploy a model")
|
|
154
|
+
deploy.add_argument("model_id", help="Hub ID, e.g. google/gemma-4-31B-it")
|
|
155
|
+
deploy.add_argument("--version", help="Model version, defaults to the latest")
|
|
156
|
+
deploy.add_argument(
|
|
157
|
+
"--deployment-template",
|
|
158
|
+
"--template",
|
|
159
|
+
dest="deployment_template",
|
|
160
|
+
help="Deployment template name (or its suffix, e.g. 256k-nvidia-h100), defaults to the model's default",
|
|
161
|
+
)
|
|
162
|
+
deploy.add_argument(
|
|
163
|
+
"--name",
|
|
164
|
+
help="Deployment name, i.e. the model to set in requests, defaults to <author>--<model>--<7-digit timestamp>",
|
|
165
|
+
)
|
|
166
|
+
deploy.add_argument("--instances", type=int, default=1, help="Number of instances")
|
|
167
|
+
deploy.add_argument("--no-wait", action="store_true", help="Return without waiting for the deployment")
|
|
168
|
+
|
|
169
|
+
deployments = group("deployments", "Manage the deployments")
|
|
170
|
+
deployments.add_parser("list", parents=[common], help="List the deployments")
|
|
171
|
+
for action, help in [
|
|
172
|
+
("status", "Show a deployment"),
|
|
173
|
+
("wait", "Wait until a deployment succeeds"),
|
|
174
|
+
("scale", "Change the number of instances"),
|
|
175
|
+
("delete", "Delete a deployment"),
|
|
176
|
+
]:
|
|
177
|
+
sub = deployments.add_parser(action, parents=[common], help=help)
|
|
178
|
+
sub.add_argument("name")
|
|
179
|
+
if action == "scale":
|
|
180
|
+
sub.add_argument("--instances", type=int, required=True)
|
|
181
|
+
sub.add_argument("--no-wait", action="store_true")
|
|
182
|
+
if action == "delete":
|
|
183
|
+
sub.add_argument("--wait", action="store_true", help="Wait until the deployment is deleted")
|
|
184
|
+
|
|
185
|
+
accounts = group("accounts", "Manage the Microsoft Foundry resources")
|
|
186
|
+
accounts.add_parser("list", parents=[common], help="List the Microsoft Foundry resources")
|
|
187
|
+
accounts.add_parser("info", parents=[common], help="Show the Microsoft Foundry resource and its endpoint")
|
|
188
|
+
accounts.add_parser("keys", parents=[common], help="Show the API keys of the Microsoft Foundry resource")
|
|
189
|
+
create = accounts.add_parser("create", parents=[common], help="Create a Microsoft Foundry resource")
|
|
190
|
+
create.add_argument("name")
|
|
191
|
+
create.add_argument("--resource-group", required=True, help="Created if it doesn't exist")
|
|
192
|
+
create.add_argument("--location", required=True, help="Azure region, e.g. eastus")
|
|
193
|
+
create.add_argument("--project", help="Project name, defaults to <name>-project")
|
|
194
|
+
|
|
195
|
+
commands.add_parser("capacity", parents=[common], help="Show the available accelerators")
|
|
196
|
+
quota = commands.add_parser("quota", parents=[common], help="Show the managed compute quota")
|
|
197
|
+
quota.add_argument("--location", help="Azure region, defaults to the Microsoft Foundry resource's")
|
|
198
|
+
|
|
199
|
+
return root
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def main() -> int:
|
|
203
|
+
logging.basicConfig(level=logging.WARNING, format="%(levelname)s %(message)s")
|
|
204
|
+
logger.setLevel(logging.INFO)
|
|
205
|
+
args = parser().parse_args()
|
|
206
|
+
try:
|
|
207
|
+
run(args)
|
|
208
|
+
except (FoundryError, ValueError, FileNotFoundError, ClientAuthenticationError) as error:
|
|
209
|
+
message = error.message if isinstance(error, ClientAuthenticationError) else str(error)
|
|
210
|
+
if isinstance(error, ClientAuthenticationError):
|
|
211
|
+
message = f"Azure authentication failed, run `az login` first: {message}"
|
|
212
|
+
if getattr(args, "output", "table") == "json":
|
|
213
|
+
sys.stderr.write(json.dumps({"error": type(error).__name__, "message": message}) + "\n")
|
|
214
|
+
else:
|
|
215
|
+
logger.error(message)
|
|
216
|
+
return 1
|
|
217
|
+
except KeyboardInterrupt:
|
|
218
|
+
return 130
|
|
219
|
+
return 0
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
if __name__ == "__main__":
|
|
223
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import subprocess
|
|
3
|
+
from functools import cached_property
|
|
4
|
+
|
|
5
|
+
import requests
|
|
6
|
+
from azure.core.credentials import TokenCredential
|
|
7
|
+
from azure.mgmt.cognitiveservices import CognitiveServicesManagementClient
|
|
8
|
+
|
|
9
|
+
from hf_msft_foundry.accounts import Account, Accounts
|
|
10
|
+
from hf_msft_foundry.azure import ARM, AzureClient
|
|
11
|
+
from hf_msft_foundry.deployments import Deployment, Deployments
|
|
12
|
+
from hf_msft_foundry.pricing import Prices
|
|
13
|
+
from hf_msft_foundry.registry import Registry
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Foundry:
|
|
17
|
+
def __init__(
|
|
18
|
+
self,
|
|
19
|
+
subscription_id: str | None = None,
|
|
20
|
+
account_name: str | None = None,
|
|
21
|
+
credential: TokenCredential | None = None,
|
|
22
|
+
session: requests.Session | None = None,
|
|
23
|
+
management: CognitiveServicesManagementClient | None = None,
|
|
24
|
+
) -> None:
|
|
25
|
+
self.azure = AzureClient(credential=credential, session=session)
|
|
26
|
+
self.registry = Registry(self.azure)
|
|
27
|
+
self._subscription_id = subscription_id or os.getenv("AZURE_SUBSCRIPTION_ID")
|
|
28
|
+
self._account_name = account_name or os.getenv("FOUNDRY_ACCOUNT_NAME")
|
|
29
|
+
self._management = management
|
|
30
|
+
|
|
31
|
+
@cached_property
|
|
32
|
+
def subscription_id(self) -> str:
|
|
33
|
+
if self._subscription_id:
|
|
34
|
+
return self._subscription_id
|
|
35
|
+
try:
|
|
36
|
+
result = subprocess.run(
|
|
37
|
+
["az", "account", "show", "--query", "id", "-o", "tsv"], capture_output=True, text=True
|
|
38
|
+
)
|
|
39
|
+
if result.returncode == 0 and result.stdout.strip():
|
|
40
|
+
return result.stdout.strip()
|
|
41
|
+
except FileNotFoundError:
|
|
42
|
+
pass
|
|
43
|
+
subscriptions = self.azure.paginate(f"{ARM}/subscriptions", {"api-version": "2022-12-01"})
|
|
44
|
+
if len(subscriptions) != 1:
|
|
45
|
+
raise ValueError(
|
|
46
|
+
f"Found {len(subscriptions)} Azure subscriptions, set --subscription-id or AZURE_SUBSCRIPTION_ID"
|
|
47
|
+
)
|
|
48
|
+
return subscriptions[0]["subscriptionId"]
|
|
49
|
+
|
|
50
|
+
@cached_property
|
|
51
|
+
def management(self) -> CognitiveServicesManagementClient:
|
|
52
|
+
return self._management or CognitiveServicesManagementClient(self.azure.credential, self.subscription_id)
|
|
53
|
+
|
|
54
|
+
@cached_property
|
|
55
|
+
def accounts(self) -> Accounts:
|
|
56
|
+
return Accounts(self.management, self.azure, self.subscription_id)
|
|
57
|
+
|
|
58
|
+
@cached_property
|
|
59
|
+
def account(self) -> Account:
|
|
60
|
+
return self.accounts.get(self._account_name)
|
|
61
|
+
|
|
62
|
+
@cached_property
|
|
63
|
+
def deployments(self) -> Deployments:
|
|
64
|
+
return Deployments(self.management, self.account)
|
|
65
|
+
|
|
66
|
+
@cached_property
|
|
67
|
+
def prices(self) -> Prices:
|
|
68
|
+
return Prices(self.azure.session)
|
|
69
|
+
|
|
70
|
+
def deploy(
|
|
71
|
+
self,
|
|
72
|
+
model_id: str,
|
|
73
|
+
template: str | None = None,
|
|
74
|
+
name: str | None = None,
|
|
75
|
+
instances: int = 1,
|
|
76
|
+
version: str | None = None,
|
|
77
|
+
) -> Deployment:
|
|
78
|
+
model = self.registry.model(model_id, version)
|
|
79
|
+
selected = self.registry.resolve_template(model, template)
|
|
80
|
+
selected.price_per_hour = self.prices.hourly(selected.accelerator_type, selected.accelerator_count, instances)
|
|
81
|
+
return self.deployments.create(model, selected, name, instances)
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import re
|
|
3
|
+
import time
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from azure.mgmt.cognitiveservices import CognitiveServicesManagementClient
|
|
8
|
+
from azure.mgmt.cognitiveservices.models import (
|
|
9
|
+
ManagedComputeDeployment,
|
|
10
|
+
ManagedComputeDeploymentProperties,
|
|
11
|
+
PatchResourceSku,
|
|
12
|
+
Sku,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
from hf_msft_foundry.accounts import Account
|
|
16
|
+
from hf_msft_foundry.azure import azure_errors
|
|
17
|
+
from hf_msft_foundry.errors import DeploymentFailedError, NotFoundError
|
|
18
|
+
from hf_msft_foundry.registry import Model, Template, template_name
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger("hf_msft_foundry")
|
|
21
|
+
|
|
22
|
+
SKU = "GlobalManagedCompute"
|
|
23
|
+
NAME_PATTERN = re.compile(r"[A-Za-z0-9_-]{2,64}")
|
|
24
|
+
PENDING_STATES = {"Creating", "Accepted", "Updating", "Running"}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def default_deployment_name(model: str, now: float | None = None) -> str:
|
|
28
|
+
suffix = f"{int(time.time() if now is None else now) % 10**7:07d}"
|
|
29
|
+
name = re.sub(r"[^a-z0-9-]", "-", model.lower())
|
|
30
|
+
return f"{name[: 64 - len(suffix) - 2].rstrip('-')}--{suffix}"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class Deployment:
|
|
35
|
+
name: str
|
|
36
|
+
state: str | None
|
|
37
|
+
model: str | None
|
|
38
|
+
template: str | None
|
|
39
|
+
accelerator_type: str | None
|
|
40
|
+
instances: int | None
|
|
41
|
+
|
|
42
|
+
@classmethod
|
|
43
|
+
def from_arm(cls, item: dict[str, Any]) -> "Deployment":
|
|
44
|
+
properties = item.get("properties", {})
|
|
45
|
+
model = properties.get("model", "")
|
|
46
|
+
template = properties.get("deploymentTemplate", "")
|
|
47
|
+
return cls(
|
|
48
|
+
name=item["name"],
|
|
49
|
+
state=properties.get("provisioningState"),
|
|
50
|
+
model=model.split("/models/", 1)[-1].split("/", 1)[0] if model else None,
|
|
51
|
+
template=template_name(template) if template else None,
|
|
52
|
+
accelerator_type=properties.get("acceleratorType"),
|
|
53
|
+
instances=(item.get("sku") or {}).get("capacity"),
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class Deployments:
|
|
58
|
+
def __init__(self, management: CognitiveServicesManagementClient, account: Account) -> None:
|
|
59
|
+
self.operations = management.managed_compute_deployments
|
|
60
|
+
self.account = account
|
|
61
|
+
|
|
62
|
+
def not_found(self, name: str) -> str:
|
|
63
|
+
return f"Deployment '{name}' not found in the Microsoft Foundry resource '{self.account.name}'"
|
|
64
|
+
|
|
65
|
+
def all(self) -> list[Deployment]:
|
|
66
|
+
with azure_errors(f"Microsoft Foundry resource '{self.account.name}' not found"):
|
|
67
|
+
items = list(self.operations.list(self.account.resource_group, self.account.name))
|
|
68
|
+
return [Deployment.from_arm(item.as_dict()) for item in items]
|
|
69
|
+
|
|
70
|
+
def get(self, name: str) -> Deployment:
|
|
71
|
+
with azure_errors(self.not_found(name)):
|
|
72
|
+
item = self.operations.get(self.account.resource_group, self.account.name, name)
|
|
73
|
+
return Deployment.from_arm(item.as_dict())
|
|
74
|
+
|
|
75
|
+
def create(self, model: Model, template: Template, name: str | None = None, instances: int = 1) -> Deployment:
|
|
76
|
+
name = name or default_deployment_name(model.name)
|
|
77
|
+
if not NAME_PATTERN.fullmatch(name):
|
|
78
|
+
raise ValueError(
|
|
79
|
+
f"Invalid deployment name '{name}', use 2 to 64 letters, digits, underscores or hyphens (no dots)"
|
|
80
|
+
)
|
|
81
|
+
price = f", ${template.price_per_hour}/hour" if template.price_per_hour is not None else ""
|
|
82
|
+
logger.info(
|
|
83
|
+
"Deploying %s (version %s) as '%s' with %s (%s x %s x %s instance(s)%s)",
|
|
84
|
+
model.hub_id or model.name,
|
|
85
|
+
model.version,
|
|
86
|
+
name,
|
|
87
|
+
template.name,
|
|
88
|
+
template.accelerator_count,
|
|
89
|
+
template.accelerator_type,
|
|
90
|
+
instances,
|
|
91
|
+
price,
|
|
92
|
+
)
|
|
93
|
+
with azure_errors(f"Microsoft Foundry resource '{self.account.name}' not found"):
|
|
94
|
+
self.operations.begin_create_or_update(
|
|
95
|
+
self.account.resource_group,
|
|
96
|
+
self.account.name,
|
|
97
|
+
name,
|
|
98
|
+
ManagedComputeDeployment(
|
|
99
|
+
sku=Sku(name=SKU, capacity=instances),
|
|
100
|
+
properties=ManagedComputeDeploymentProperties(
|
|
101
|
+
model=model.uri,
|
|
102
|
+
deployment_template=template.uri,
|
|
103
|
+
accelerator_type=template.accelerator_type,
|
|
104
|
+
version_upgrade_option="OnceNewDefaultVersionAvailable",
|
|
105
|
+
),
|
|
106
|
+
),
|
|
107
|
+
)
|
|
108
|
+
return self.get(name)
|
|
109
|
+
|
|
110
|
+
def update(self, name: str, instances: int) -> Deployment:
|
|
111
|
+
logger.info("Scaling deployment '%s' to %s instance(s)", name, instances)
|
|
112
|
+
with azure_errors(self.not_found(name)):
|
|
113
|
+
self.operations.begin_update(
|
|
114
|
+
self.account.resource_group,
|
|
115
|
+
self.account.name,
|
|
116
|
+
name,
|
|
117
|
+
PatchResourceSku(sku=Sku(name=SKU, capacity=instances)),
|
|
118
|
+
)
|
|
119
|
+
return self.get(name)
|
|
120
|
+
|
|
121
|
+
def delete(self, name: str) -> None:
|
|
122
|
+
self.get(name)
|
|
123
|
+
with azure_errors(self.not_found(name)):
|
|
124
|
+
self.operations.begin_delete(self.account.resource_group, self.account.name, name)
|
|
125
|
+
logger.info("Deleting deployment '%s'", name)
|
|
126
|
+
|
|
127
|
+
def wait(self, name: str, interval: int = 30) -> Deployment:
|
|
128
|
+
start = time.monotonic()
|
|
129
|
+
while True:
|
|
130
|
+
deployment = self.get(name)
|
|
131
|
+
elapsed = time.monotonic() - start
|
|
132
|
+
if deployment.state == "Succeeded":
|
|
133
|
+
logger.info(
|
|
134
|
+
"Deployment '%s' succeeded after %.0fs, use '%s' as the model in requests", name, elapsed, name
|
|
135
|
+
)
|
|
136
|
+
return deployment
|
|
137
|
+
if deployment.state not in PENDING_STATES:
|
|
138
|
+
raise DeploymentFailedError(f"Deployment '{name}' ended in state {deployment.state}")
|
|
139
|
+
logger.info("Deployment '%s' is %s (%.0fs)", name, deployment.state, elapsed)
|
|
140
|
+
time.sleep(interval)
|
|
141
|
+
|
|
142
|
+
def wait_deleted(self, name: str, interval: int = 15) -> None:
|
|
143
|
+
start = time.monotonic()
|
|
144
|
+
while True:
|
|
145
|
+
try:
|
|
146
|
+
deployment = self.get(name)
|
|
147
|
+
except NotFoundError:
|
|
148
|
+
logger.info("Deployment '%s' deleted after %.0fs", name, time.monotonic() - start)
|
|
149
|
+
return
|
|
150
|
+
logger.info("Deployment '%s' is %s (%.0fs)", name, deployment.state, time.monotonic() - start)
|
|
151
|
+
time.sleep(interval)
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
class FoundryError(Exception):
|
|
2
|
+
pass
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class AzureRequestError(FoundryError):
|
|
6
|
+
def __init__(self, status_code: int, message: str) -> None:
|
|
7
|
+
super().__init__(f"Azure returned {status_code}: {message}")
|
|
8
|
+
self.status_code = status_code
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class NotFoundError(FoundryError, LookupError):
|
|
12
|
+
pass
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class DeploymentFailedError(FoundryError):
|
|
16
|
+
pass
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from functools import cached_property
|
|
2
|
+
|
|
3
|
+
import requests
|
|
4
|
+
|
|
5
|
+
PRICES_URL = "https://prices.azure.com/api/retail/prices"
|
|
6
|
+
PRICES_FILTER = (
|
|
7
|
+
"serviceName eq 'Foundry Models' and productName eq 'Managed Compute' "
|
|
8
|
+
"and armRegionName eq '{region}' and type eq 'Consumption'"
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Prices:
|
|
13
|
+
def __init__(self, session: requests.Session, region: str = "eastus") -> None:
|
|
14
|
+
self.session = session
|
|
15
|
+
self.region = region
|
|
16
|
+
|
|
17
|
+
@cached_property
|
|
18
|
+
def per_accelerator_hour(self) -> dict[str, float]:
|
|
19
|
+
prices: dict[str, float] = {}
|
|
20
|
+
url: str | None = PRICES_URL
|
|
21
|
+
params: dict[str, str] | None = {"$filter": PRICES_FILTER.format(region=self.region)}
|
|
22
|
+
while url:
|
|
23
|
+
response = self.session.get(url, params=params, timeout=60)
|
|
24
|
+
response.raise_for_status()
|
|
25
|
+
data = response.json()
|
|
26
|
+
for item in data.get("Items", []):
|
|
27
|
+
sku = item.get("skuName", "")
|
|
28
|
+
if sku.startswith("Mngd ") and sku.endswith(" Gl") and item.get("unitOfMeasure") == "1 Hour":
|
|
29
|
+
prices[sku.removeprefix("Mngd ").removesuffix(" Gl")] = item["retailPrice"]
|
|
30
|
+
url, params = data.get("NextPageLink"), None
|
|
31
|
+
return prices
|
|
32
|
+
|
|
33
|
+
def hourly(self, accelerator_type: str, accelerator_count: int, instances: int = 1) -> float | None:
|
|
34
|
+
try:
|
|
35
|
+
price = self.per_accelerator_hour.get(accelerator_type)
|
|
36
|
+
except requests.RequestException:
|
|
37
|
+
return None
|
|
38
|
+
return round(price * accelerator_count * instances, 2) if price is not None else None
|
|
File without changes
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
from functools import cached_property
|
|
3
|
+
|
|
4
|
+
from hf_msft_foundry.azure import AzureClient
|
|
5
|
+
from hf_msft_foundry.errors import NotFoundError
|
|
6
|
+
from hf_msft_foundry.pricing import Prices
|
|
7
|
+
|
|
8
|
+
REGISTRY = "azure-huggingface"
|
|
9
|
+
REGISTRY_HOST = "https://eastus.api.azureml.ms"
|
|
10
|
+
MODELS_API_VERSION = "2021-10-01-dataplanepreview"
|
|
11
|
+
TEMPLATES_API_VERSION = "2026-05-01-preview"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def to_foundry_name(model_id: str) -> str:
|
|
15
|
+
return model_id.strip().lower().replace("/", "--")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def template_name(asset_id: str) -> str:
|
|
19
|
+
return asset_id.split("/deploymenttemplates/", 1)[1].split("/", 1)[0]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class Model:
|
|
24
|
+
name: str
|
|
25
|
+
version: str
|
|
26
|
+
hub_id: str | None
|
|
27
|
+
task: str | None
|
|
28
|
+
input_modalities: str | None
|
|
29
|
+
context_window: str | None
|
|
30
|
+
default_template: str | None
|
|
31
|
+
templates: list[str] = field(default_factory=list)
|
|
32
|
+
|
|
33
|
+
@property
|
|
34
|
+
def uri(self) -> str:
|
|
35
|
+
return f"azureml://registries/{REGISTRY}/models/{self.name}/versions/{self.version}"
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def url(self) -> str | None:
|
|
39
|
+
return f"https://huggingface.co/{self.hub_id}" if self.hub_id else None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass
|
|
43
|
+
class Template:
|
|
44
|
+
name: str
|
|
45
|
+
version: str
|
|
46
|
+
description: str | None
|
|
47
|
+
accelerator_type: str
|
|
48
|
+
accelerator_count: int
|
|
49
|
+
max_model_len: str | None
|
|
50
|
+
environment: str
|
|
51
|
+
price_per_hour: float | None = None
|
|
52
|
+
default: bool = False
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def uri(self) -> str:
|
|
56
|
+
return f"azureml://registries/{REGISTRY}/deploymenttemplates/{self.name}/labels/latest"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class Registry:
|
|
60
|
+
def __init__(self, azure: AzureClient) -> None:
|
|
61
|
+
self.azure = azure
|
|
62
|
+
|
|
63
|
+
@cached_property
|
|
64
|
+
def discovery(self) -> dict[str, str]:
|
|
65
|
+
return self.azure.request("GET", f"{REGISTRY_HOST}/registrymanagement/v1.0/registries/{REGISTRY}/discovery")
|
|
66
|
+
|
|
67
|
+
@property
|
|
68
|
+
def path(self) -> str:
|
|
69
|
+
return (
|
|
70
|
+
f"/subscriptions/{self.discovery['subscriptionId']}/resourceGroups/{self.discovery['resourceGroup']}"
|
|
71
|
+
f"/providers/Microsoft.MachineLearningServices/registries/{REGISTRY}"
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def models_url(self) -> str:
|
|
76
|
+
host = self.discovery["primaryRegionResourceProviderUri"].rstrip("/")
|
|
77
|
+
return f"{host}/mferp/managementfrontend{self.path}/models"
|
|
78
|
+
|
|
79
|
+
def models(self, search: str | None = None) -> list[str]:
|
|
80
|
+
params = {"api-version": MODELS_API_VERSION, "listViewType": "ActiveOnly"}
|
|
81
|
+
names = sorted(item["name"] for item in self.azure.paginate(self.models_url, params))
|
|
82
|
+
if search:
|
|
83
|
+
terms = to_foundry_name(search).replace("--", " ").split()
|
|
84
|
+
names = [name for name in names if all(term in name for term in terms)]
|
|
85
|
+
return names
|
|
86
|
+
|
|
87
|
+
def model(self, model_id: str, version: str | None = None) -> Model:
|
|
88
|
+
name = to_foundry_name(model_id)
|
|
89
|
+
params = {"api-version": MODELS_API_VERSION}
|
|
90
|
+
try:
|
|
91
|
+
latest = self.azure.request("GET", f"{self.models_url}/{name}", params=params)
|
|
92
|
+
except NotFoundError as error:
|
|
93
|
+
raise NotFoundError(
|
|
94
|
+
f"Model '{model_id}' is not available in the Hugging Face collection on Microsoft Foundry, "
|
|
95
|
+
"find the available ones with `hf msft-foundry models --search <query>`"
|
|
96
|
+
) from error
|
|
97
|
+
version = version or latest["properties"]["latestVersion"]
|
|
98
|
+
try:
|
|
99
|
+
properties = self.azure.request("GET", f"{self.models_url}/{name}/versions/{version}", params=params)[
|
|
100
|
+
"properties"
|
|
101
|
+
]
|
|
102
|
+
except NotFoundError as error:
|
|
103
|
+
raise NotFoundError(
|
|
104
|
+
f"Version {version} of '{model_id}' not found, the latest is {latest['properties']['latestVersion']}"
|
|
105
|
+
) from error
|
|
106
|
+
tags = properties.get("tags", {})
|
|
107
|
+
default = (properties.get("defaultDeploymentTemplate") or {}).get("assetId")
|
|
108
|
+
return Model(
|
|
109
|
+
name=name,
|
|
110
|
+
version=version,
|
|
111
|
+
hub_id=tags.get("huggingface_model_id"),
|
|
112
|
+
task=tags.get("task"),
|
|
113
|
+
input_modalities=tags.get("inputModalities"),
|
|
114
|
+
context_window=tags.get("textContextWindow"),
|
|
115
|
+
default_template=template_name(default) if default else None,
|
|
116
|
+
templates=[template_name(t["assetId"]) for t in properties.get("allowedDeploymentTemplates") or []],
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
def template(self, name: str) -> Template:
|
|
120
|
+
url = f"{REGISTRY_HOST}/genericasset/v2.0{self.path}/deploymenttemplates"
|
|
121
|
+
params = {"api-version": TEMPLATES_API_VERSION, "listViewType": "ActiveOnly", "name": name}
|
|
122
|
+
versions = [item["properties"] for item in self.azure.request("GET", url, params=params).get("value", [])]
|
|
123
|
+
latest = next((v for v in versions if "latest" in v.get("labels", [])), None)
|
|
124
|
+
if latest is None:
|
|
125
|
+
raise NotFoundError(f"Deployment template '{name}' has no version labelled as latest")
|
|
126
|
+
accelerator = (latest.get("acceleratorMaps") or [{}])[0]
|
|
127
|
+
env = latest.get("environmentVariables") or {}
|
|
128
|
+
return Template(
|
|
129
|
+
name=name,
|
|
130
|
+
version=latest["version"],
|
|
131
|
+
description=latest.get("displayName"),
|
|
132
|
+
accelerator_type=accelerator.get("acceleratorType", ""),
|
|
133
|
+
accelerator_count=accelerator.get("numberOfAcceleratorsPerModelInstance", 0),
|
|
134
|
+
max_model_len=next((v for k, v in env.items() if k.endswith("MAX_MODEL_LEN")), None),
|
|
135
|
+
environment=latest.get("environmentId", "").split("/environments/", 1)[-1].split("/", 1)[0],
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
def templates(self, model: Model, prices: Prices | None = None) -> list[Template]:
|
|
139
|
+
templates = [self.template(name) for name in model.templates]
|
|
140
|
+
for template in templates:
|
|
141
|
+
template.default = template.name == model.default_template
|
|
142
|
+
if prices:
|
|
143
|
+
template.price_per_hour = prices.hourly(template.accelerator_type, template.accelerator_count)
|
|
144
|
+
return templates
|
|
145
|
+
|
|
146
|
+
def resolve_template(self, model: Model, value: str | None) -> Template:
|
|
147
|
+
if value is None:
|
|
148
|
+
if model.default_template is None:
|
|
149
|
+
raise ValueError(f"{model.name} has no default deployment template, pass --deployment-template")
|
|
150
|
+
return self.template(model.default_template)
|
|
151
|
+
name = template_name(value) if "/deploymenttemplates/" in value else value.lower()
|
|
152
|
+
matches = [t for t in model.templates if t == name or t == f"{model.name.replace('.', '-')}--{name}"]
|
|
153
|
+
if not matches:
|
|
154
|
+
matches = [t for t in model.templates if t.endswith(f"--{name}")]
|
|
155
|
+
if len(matches) != 1:
|
|
156
|
+
options = ", ".join(model.templates)
|
|
157
|
+
raise ValueError(f"Deployment template '{value}' is not available for {model.name}, use one of: {options}")
|
|
158
|
+
return self.template(matches[0])
|