sima-cli 0.0.35__py3-none-any.whl → 0.0.37__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sima_cli/__version__.py +1 -1
- sima_cli/app_zoo/app.py +435 -0
- sima_cli/auth/basic_auth.py +0 -1
- sima_cli/cli.py +74 -6
- sima_cli/download/downloader.py +32 -0
- sima_cli/install/metadata_installer.py +317 -9
- sima_cli/model_zoo/model.py +76 -28
- {sima_cli-0.0.35.dist-info → sima_cli-0.0.37.dist-info}/METADATA +1 -1
- {sima_cli-0.0.35.dist-info → sima_cli-0.0.37.dist-info}/RECORD +13 -13
- {sima_cli-0.0.35.dist-info → sima_cli-0.0.37.dist-info}/WHEEL +0 -0
- {sima_cli-0.0.35.dist-info → sima_cli-0.0.37.dist-info}/entry_points.txt +0 -0
- {sima_cli-0.0.35.dist-info → sima_cli-0.0.37.dist-info}/licenses/LICENSE +0 -0
- {sima_cli-0.0.35.dist-info → sima_cli-0.0.37.dist-info}/top_level.txt +0 -0
sima_cli/__version__.py
CHANGED
@@ -1,2 +1,2 @@
|
|
1
1
|
# sima_cli/__version__.py
|
2
|
-
__version__ = "0.0.
|
2
|
+
__version__ = "0.0.37"
|
sima_cli/app_zoo/app.py
CHANGED
@@ -0,0 +1,435 @@
|
|
1
|
+
# model_zoo/models.py
|
2
|
+
|
3
|
+
import requests
|
4
|
+
import click
|
5
|
+
import os
|
6
|
+
import yaml
|
7
|
+
import zipfile
|
8
|
+
import shutil
|
9
|
+
from urllib.parse import urlparse
|
10
|
+
from rich import print
|
11
|
+
from rich.text import Text
|
12
|
+
from rich.table import Table
|
13
|
+
from rich.panel import Panel
|
14
|
+
from rich.console import Console
|
15
|
+
from sima_cli.utils.config import get_auth_token
|
16
|
+
from sima_cli.utils.config_loader import artifactory_url
|
17
|
+
from sima_cli.download import download_file_from_url
|
18
|
+
|
19
|
+
ARTIFACTORY_BASE_URL = artifactory_url() + '/artifactory'
|
20
|
+
|
21
|
+
def _is_valid_url(url: str) -> bool:
|
22
|
+
try:
|
23
|
+
result = urlparse(url)
|
24
|
+
return all([result.scheme, result.netloc])
|
25
|
+
except:
|
26
|
+
return False
|
27
|
+
|
28
|
+
def _describe_app_internal(ver: str, app_name: str):
|
29
|
+
repo = "vdp"
|
30
|
+
base_path = f"vdp-app-config-default/{ver}/{app_name}"
|
31
|
+
aql_query = f"""
|
32
|
+
items.find({{
|
33
|
+
"repo": "{repo}",
|
34
|
+
"path": "{base_path}",
|
35
|
+
"$or": [
|
36
|
+
{{ "name": {{ "$match": "*.yaml" }} }},
|
37
|
+
{{ "name": {{ "$match": "*.yml" }} }}
|
38
|
+
],
|
39
|
+
"type": "file"
|
40
|
+
}}).include("repo","path","name")
|
41
|
+
""".strip()
|
42
|
+
|
43
|
+
headers = {
|
44
|
+
"Content-Type": "text/plain",
|
45
|
+
"Authorization": f"Bearer {get_auth_token(internal=True)}"
|
46
|
+
}
|
47
|
+
aql_url = f"{ARTIFACTORY_BASE_URL}/api/search/aql"
|
48
|
+
response = requests.post(aql_url, data=aql_query, headers=headers)
|
49
|
+
if response.status_code != 200:
|
50
|
+
click.echo(f"❌ Failed to list app files. Status: {response.status_code}")
|
51
|
+
click.echo(response.text)
|
52
|
+
return
|
53
|
+
|
54
|
+
results = response.json().get("results", [])
|
55
|
+
yaml_file = next((f for f in results if f["name"].endswith((".yaml", ".yml"))), None)
|
56
|
+
|
57
|
+
if not yaml_file:
|
58
|
+
click.echo(f"⚠️ No .yaml or .yml file found under: {base_path}")
|
59
|
+
return
|
60
|
+
|
61
|
+
# Download the YAML
|
62
|
+
yaml_url = f"{ARTIFACTORY_BASE_URL}/{repo}/{yaml_file['path']}/{yaml_file['name']}"
|
63
|
+
response = requests.get(yaml_url, headers={"Authorization": f"Bearer {get_auth_token(internal=True)}"})
|
64
|
+
if response.status_code != 200:
|
65
|
+
click.echo(f"❌ Failed to fetch YAML: {response.status_code}")
|
66
|
+
return
|
67
|
+
|
68
|
+
try:
|
69
|
+
data = yaml.safe_load(response.text)
|
70
|
+
except yaml.YAMLError as e:
|
71
|
+
click.echo(f"❌ Failed to parse YAML: {e}")
|
72
|
+
return
|
73
|
+
|
74
|
+
# ---- PIPELINE (new app YAML) ----
|
75
|
+
if "pipeline" in data:
|
76
|
+
pipeline = data.get("pipeline", {})
|
77
|
+
|
78
|
+
header = f"[bold green]{pipeline.get('name', app_name)}[/bold green] - {pipeline.get('category', 'Unknown')}"
|
79
|
+
desc = pipeline.get("short_description", "")
|
80
|
+
|
81
|
+
console = Console()
|
82
|
+
# Panel body holds description so it wraps
|
83
|
+
body = Text(desc, style="yellow", no_wrap=False)
|
84
|
+
panel_width = min(console.width, 100)
|
85
|
+
|
86
|
+
print(Panel(body, title=header, expand=False, width=80))
|
87
|
+
|
88
|
+
p_table = Table(title="Pipeline", header_style="bold magenta")
|
89
|
+
p_table.add_column("Field")
|
90
|
+
p_table.add_column("Value")
|
91
|
+
p_table.add_row("Input Format", pipeline.get("in_format", "-"))
|
92
|
+
p_table.add_row("Output Format", pipeline.get("out_format", "-"))
|
93
|
+
|
94
|
+
perf = pipeline.get("performance", {})
|
95
|
+
p_table.add_row("Davinci FPS", str(perf.get("davinci_fps", "-")))
|
96
|
+
p_table.add_row("Modalix FPS", str(perf.get("modalix_fps", "-")))
|
97
|
+
|
98
|
+
print(p_table)
|
99
|
+
# ---- MODEL(S) ----
|
100
|
+
if "model" in data:
|
101
|
+
models = data["model"]
|
102
|
+
if isinstance(models, dict):
|
103
|
+
models = [models]
|
104
|
+
elif not isinstance(models, list):
|
105
|
+
models = []
|
106
|
+
|
107
|
+
for idx, model in enumerate(models, start=1):
|
108
|
+
title = f"Model #{idx}" if len(models) > 1 else "Model"
|
109
|
+
m_table = Table(title=title, header_style="bold cyan")
|
110
|
+
m_table.add_column("Field")
|
111
|
+
m_table.add_column("Value")
|
112
|
+
|
113
|
+
m_table.add_row("Name", model.get("name", "-"))
|
114
|
+
|
115
|
+
if inp := model.get("input_description"):
|
116
|
+
m_table.add_row("Resolution", str(inp.get("resolution", "-")))
|
117
|
+
m_table.add_row("Format", inp.get("format", "-"))
|
118
|
+
|
119
|
+
if resize := model.get("resize_configuration"):
|
120
|
+
m_table.add_row("Resize Format", resize.get("input_image_format", "-"))
|
121
|
+
m_table.add_row("Input Shape", str(resize.get("input_shape", "-")))
|
122
|
+
m_table.add_row("Scaling Type", resize.get("scaling_type", "-"))
|
123
|
+
m_table.add_row("Padding Type", resize.get("padding_type", "-"))
|
124
|
+
m_table.add_row("Aspect Ratio", str(resize.get("aspect_ratio", "-")))
|
125
|
+
|
126
|
+
if norm := model.get("normalization_configuration"):
|
127
|
+
m_table.add_row("Channel Mean", str(norm.get("channel_mean", "-")))
|
128
|
+
m_table.add_row("Channel Stddev", str(norm.get("channel_stddev", "-")))
|
129
|
+
if "accuracy" in norm:
|
130
|
+
m_table.add_row("Accuracy", str(norm.get("accuracy", "-")))
|
131
|
+
|
132
|
+
# Legacy model-centric fields
|
133
|
+
if "dataset" in model:
|
134
|
+
ds = model["dataset"]
|
135
|
+
m_table.add_row("Dataset", ds.get("name", "-"))
|
136
|
+
for k, v in (ds.get("params") or {}).items():
|
137
|
+
m_table.add_row(k, str(v))
|
138
|
+
m_table.add_row("Accuracy", ds.get("accuracy", "-"))
|
139
|
+
m_table.add_row("Calibration", ds.get("calibration", "-"))
|
140
|
+
|
141
|
+
if "quantization_settings" in model:
|
142
|
+
q = model["quantization_settings"]
|
143
|
+
m_table.add_row("Calibration Samples", str(q.get("calibration_num_samples", "-")))
|
144
|
+
m_table.add_row("Calibration Method", q.get("calibration_method", "-"))
|
145
|
+
m_table.add_row("Requantization Mode", q.get("requantization_mode", "-"))
|
146
|
+
m_table.add_row("Bias Correction", str(q.get("bias_correction", "-")))
|
147
|
+
|
148
|
+
print(m_table)
|
149
|
+
|
150
|
+
# ---- PIPELINE TRANSFORMS (legacy YAML) ----
|
151
|
+
if "pipeline" in data and "transforms" in data["pipeline"]:
|
152
|
+
transforms = data["pipeline"]["transforms"]
|
153
|
+
if isinstance(transforms, list):
|
154
|
+
t_table = Table(title="Pipeline Transforms", header_style="bold green")
|
155
|
+
t_table.add_column("Name")
|
156
|
+
t_table.add_column("Params")
|
157
|
+
|
158
|
+
for step in transforms:
|
159
|
+
name = step.get("name")
|
160
|
+
params = step.get("params", {})
|
161
|
+
param_str = ", ".join(f"{k}={v}" for k, v in params.items()) if params else "-"
|
162
|
+
t_table.add_row(name, param_str)
|
163
|
+
|
164
|
+
print(t_table)
|
165
|
+
|
166
|
+
# If nothing useful parsed
|
167
|
+
if not any(k in data for k in ("pipeline", "model")):
|
168
|
+
click.echo("⚠️ YAML parsed, but no recognizable `pipeline` or `model` sections found.")
|
169
|
+
|
170
|
+
def _download_app_internal(ver: str, app_name: str):
|
171
|
+
"""
|
172
|
+
Download a specific app (.zip file) from Artifactory App Zoo, unzip it,
|
173
|
+
flatten the extracted content into the app folder, and clean up.
|
174
|
+
"""
|
175
|
+
repo = "vdp"
|
176
|
+
base_path = f"vdp-app-config-default/{ver}/{app_name}"
|
177
|
+
aql_query = f"""
|
178
|
+
items.find({{
|
179
|
+
"repo": "{repo}",
|
180
|
+
"path": "{base_path}",
|
181
|
+
"name": {{"$match": "*.zip"}},
|
182
|
+
"type": "file"
|
183
|
+
}}).include("repo","path","name")
|
184
|
+
""".strip()
|
185
|
+
|
186
|
+
aql_url = f"{ARTIFACTORY_BASE_URL}/api/search/aql"
|
187
|
+
headers = {
|
188
|
+
"Content-Type": "text/plain",
|
189
|
+
"Authorization": f"Bearer {get_auth_token(internal=True)}"
|
190
|
+
}
|
191
|
+
|
192
|
+
response = requests.post(aql_url, data=aql_query, headers=headers)
|
193
|
+
if response.status_code != 200:
|
194
|
+
click.echo(f"❌ Failed to list app files. Status: {response.status_code}, path: {aql_url}")
|
195
|
+
click.echo(response.text)
|
196
|
+
return None
|
197
|
+
|
198
|
+
results = response.json().get("results", [])
|
199
|
+
if not results:
|
200
|
+
click.echo(f"⚠️ No .zip file found for app: {app_name}")
|
201
|
+
return None
|
202
|
+
|
203
|
+
# Expect only one .zip file per app
|
204
|
+
file_info = results[0]
|
205
|
+
file_path = file_info["path"]
|
206
|
+
file_name = file_info["name"]
|
207
|
+
download_url = f"{ARTIFACTORY_BASE_URL}/{repo}/{file_path}/{file_name}"
|
208
|
+
|
209
|
+
dest_dir = os.path.join(os.getcwd(), app_name)
|
210
|
+
os.makedirs(dest_dir, exist_ok=True)
|
211
|
+
|
212
|
+
click.echo(f"⬇️ Downloading app '{app_name}' to '{dest_dir}'...")
|
213
|
+
|
214
|
+
try:
|
215
|
+
local_zip = download_file_from_url(download_url, dest_folder=dest_dir, internal=True)
|
216
|
+
click.echo(f"✅ {file_name} -> {local_zip}")
|
217
|
+
except Exception as e:
|
218
|
+
click.echo(f"❌ Failed to download {file_name}: {e}")
|
219
|
+
return None
|
220
|
+
|
221
|
+
# Unzip into the destination folder
|
222
|
+
try:
|
223
|
+
with zipfile.ZipFile(local_zip, "r") as zip_ref:
|
224
|
+
zip_ref.extractall(dest_dir)
|
225
|
+
click.echo(f"📦 Extracted {file_name} into {dest_dir}")
|
226
|
+
except Exception as e:
|
227
|
+
click.echo(f"❌ Failed to unzip {file_name}: {e}")
|
228
|
+
return None
|
229
|
+
|
230
|
+
# Move contents up if unzipped into a nested folder
|
231
|
+
extracted_root = os.path.join(dest_dir, file_name.replace(".zip", ""))
|
232
|
+
if os.path.isdir(extracted_root):
|
233
|
+
for item in os.listdir(extracted_root):
|
234
|
+
src = os.path.join(extracted_root, item)
|
235
|
+
dst = os.path.join(dest_dir, item)
|
236
|
+
if os.path.isdir(src):
|
237
|
+
if os.path.exists(dst):
|
238
|
+
shutil.rmtree(dst)
|
239
|
+
shutil.move(src, dst)
|
240
|
+
else:
|
241
|
+
shutil.move(src, dst)
|
242
|
+
shutil.rmtree(extracted_root)
|
243
|
+
|
244
|
+
# Remove the original zip file
|
245
|
+
try:
|
246
|
+
os.remove(local_zip)
|
247
|
+
except OSError:
|
248
|
+
pass
|
249
|
+
|
250
|
+
click.echo(f"✅ App '{app_name}' ready at {dest_dir}")
|
251
|
+
return dest_dir
|
252
|
+
|
253
|
+
def _list_available_app_versions_internal(match_keyword: str = None):
|
254
|
+
"""
|
255
|
+
List all available App Zoo versions from Artifactory (vdp-app-config-default).
|
256
|
+
If match_keyword is provided, only versions containing the keyword (case-insensitive) are returned.
|
257
|
+
"""
|
258
|
+
repo = "vdp"
|
259
|
+
base_path = "vdp-app-config-default"
|
260
|
+
aql_query = f"""
|
261
|
+
items.find({{
|
262
|
+
"repo": "{repo}",
|
263
|
+
"path": {{"$match": "{base_path}/*"}},
|
264
|
+
"type": "folder"
|
265
|
+
}}).include("repo","path","name")
|
266
|
+
""".strip()
|
267
|
+
|
268
|
+
aql_url = f"{ARTIFACTORY_BASE_URL}/api/search/aql"
|
269
|
+
headers = {
|
270
|
+
"Content-Type": "text/plain",
|
271
|
+
"Authorization": f"Bearer {get_auth_token(internal=True)}"
|
272
|
+
}
|
273
|
+
|
274
|
+
response = requests.post(aql_url, data=aql_query, headers=headers)
|
275
|
+
|
276
|
+
if response.status_code == 401:
|
277
|
+
print('❌ You are not authorized to access Artifactory. Use `sima-cli -i login` with your Artifactory identity token to authenticate, then try again.')
|
278
|
+
return []
|
279
|
+
|
280
|
+
if response.status_code != 200:
|
281
|
+
print(f"❌ Failed to retrieve app versions (status {response.status_code})")
|
282
|
+
print(response.text)
|
283
|
+
return []
|
284
|
+
|
285
|
+
results = response.json().get("results", [])
|
286
|
+
|
287
|
+
# Versions = folder names under vdp-app-config-default/
|
288
|
+
versions = sorted({
|
289
|
+
item["path"].replace(base_path + "/", "").split("/")[0]
|
290
|
+
for item in results
|
291
|
+
if item["path"].startswith(base_path + "/")
|
292
|
+
})
|
293
|
+
|
294
|
+
if match_keyword:
|
295
|
+
mk = match_keyword.lower()
|
296
|
+
versions = [v for v in versions if mk in v.lower()]
|
297
|
+
|
298
|
+
return versions
|
299
|
+
|
300
|
+
|
301
|
+
def _list_available_apps_internal(version: str):
|
302
|
+
"""
|
303
|
+
List available app folders from Artifactory for the given SDK version.
|
304
|
+
Returns distinct top-level folders under vdp-app-config-default/{version}/.
|
305
|
+
After selecting an app, shows its description and action menu.
|
306
|
+
"""
|
307
|
+
repo = "vdp"
|
308
|
+
base_prefix = f"vdp-app-config-default/{version}"
|
309
|
+
aql_query = f"""
|
310
|
+
items.find({{
|
311
|
+
"repo": "{repo}",
|
312
|
+
"path": {{"$match": "{base_prefix}/*"}}
|
313
|
+
}}).include("repo","path","name")
|
314
|
+
""".strip()
|
315
|
+
|
316
|
+
aql_url = f"{ARTIFACTORY_BASE_URL}/api/search/aql"
|
317
|
+
headers = {
|
318
|
+
"Content-Type": "text/plain",
|
319
|
+
"Authorization": f"Bearer {get_auth_token(internal=True)}"
|
320
|
+
}
|
321
|
+
|
322
|
+
response = requests.post(aql_url, data=aql_query, headers=headers)
|
323
|
+
if response.status_code != 200:
|
324
|
+
click.echo(f"❌ Failed to retrieve app list (status {response.status_code})")
|
325
|
+
click.echo(response.text)
|
326
|
+
return None
|
327
|
+
|
328
|
+
results = response.json().get("results", [])
|
329
|
+
|
330
|
+
# Collect only top-level folder names under {version}/
|
331
|
+
app_names = sorted({
|
332
|
+
item["path"].replace(base_prefix + "/", "").split("/")[0]
|
333
|
+
for item in results
|
334
|
+
if item["path"].startswith(base_prefix + "/")
|
335
|
+
})
|
336
|
+
|
337
|
+
if not app_names:
|
338
|
+
click.echo("⚠️ No apps found.")
|
339
|
+
return None
|
340
|
+
|
341
|
+
# Add Exit option at the bottom
|
342
|
+
app_names_with_exit = app_names + ["Exit"]
|
343
|
+
|
344
|
+
from InquirerPy import inquirer
|
345
|
+
|
346
|
+
while True:
|
347
|
+
# Step 1: Select app
|
348
|
+
selected_app = inquirer.fuzzy(
|
349
|
+
message=f"Select an app from version {version}:",
|
350
|
+
choices=app_names_with_exit,
|
351
|
+
max_height="70%",
|
352
|
+
instruction="(Use ↑↓ to navigate, / to search, Enter to select)"
|
353
|
+
).execute()
|
354
|
+
|
355
|
+
if not selected_app or selected_app == "Exit":
|
356
|
+
click.echo("👋 Exiting.")
|
357
|
+
break
|
358
|
+
|
359
|
+
# Step 2: Describe the app
|
360
|
+
_describe_app_internal(version, selected_app)
|
361
|
+
|
362
|
+
# Step 3: Action menu
|
363
|
+
action = inquirer.select(
|
364
|
+
message=f"What would you like to do with {selected_app}?",
|
365
|
+
choices=["Download", "Back", "Exit"],
|
366
|
+
default="Back"
|
367
|
+
).execute()
|
368
|
+
|
369
|
+
if action == "Download":
|
370
|
+
_download_app_internal(version, selected_app)
|
371
|
+
click.echo(f"✅ Downloaded {selected_app}")
|
372
|
+
# loop back to menu again after download
|
373
|
+
elif action == "Back":
|
374
|
+
# back to app list (continue loop)
|
375
|
+
continue
|
376
|
+
elif action == "Exit":
|
377
|
+
click.echo("👋 Exiting.")
|
378
|
+
break
|
379
|
+
|
380
|
+
|
381
|
+
def list_apps(internal, ver):
|
382
|
+
if internal:
|
383
|
+
click.echo("App Zoo Source : SiMa Artifactory...")
|
384
|
+
versions = _list_available_app_versions_internal(ver)
|
385
|
+
|
386
|
+
if len(versions) == 1:
|
387
|
+
# Exactly one match → go straight to listing apps
|
388
|
+
return _list_available_apps_internal(versions[0])
|
389
|
+
elif len(versions) == 0:
|
390
|
+
print(f'❌ No version match found in Artifactory for [{ver}]')
|
391
|
+
return []
|
392
|
+
else:
|
393
|
+
# Multiple matches → prompt user to choose
|
394
|
+
click.echo("Multiple app zoo versions found matching your input:")
|
395
|
+
|
396
|
+
from InquirerPy import inquirer
|
397
|
+
selected_version = inquirer.fuzzy(
|
398
|
+
message="Select a version:",
|
399
|
+
choices=versions,
|
400
|
+
max_height="70%", # scrollable
|
401
|
+
instruction="(Use ↑↓ to navigate, / to search, Enter to select)"
|
402
|
+
).execute()
|
403
|
+
|
404
|
+
if not selected_version:
|
405
|
+
click.echo("No selection made. Exiting.", err=True)
|
406
|
+
raise SystemExit(1)
|
407
|
+
|
408
|
+
return _list_available_apps_internal(selected_version)
|
409
|
+
|
410
|
+
else:
|
411
|
+
print('External app zoo not supported yet')
|
412
|
+
return []
|
413
|
+
|
414
|
+
def download_app(internal, ver, model_name):
|
415
|
+
if internal:
|
416
|
+
click.echo("App Zoo Source : SiMa Artifactory...")
|
417
|
+
return _download_app_internal(ver, model_name)
|
418
|
+
else:
|
419
|
+
print('External app zoo not supported yet')
|
420
|
+
|
421
|
+
def describe_app(internal, ver, model_name):
|
422
|
+
if internal:
|
423
|
+
click.echo("App Zoo Source : SiMa Artifactory...")
|
424
|
+
return _describe_app_internal(ver, model_name)
|
425
|
+
else:
|
426
|
+
print('External app zoo not supported yet')
|
427
|
+
|
428
|
+
# Module CLI tests
|
429
|
+
if __name__ == "__main__":
|
430
|
+
import sys
|
431
|
+
if len(sys.argv) < 2:
|
432
|
+
print("Usage: python models.py <version>")
|
433
|
+
else:
|
434
|
+
version_arg = sys.argv[1]
|
435
|
+
_list_available_apps_internal(version_arg)
|
sima_cli/auth/basic_auth.py
CHANGED
sima_cli/cli.py
CHANGED
@@ -3,6 +3,7 @@ import click
|
|
3
3
|
from sima_cli.utils.env import get_environment_type
|
4
4
|
from sima_cli.update.updater import perform_update
|
5
5
|
from sima_cli.model_zoo.model import list_models, download_model, describe_model
|
6
|
+
from sima_cli.app_zoo.app import list_apps, download_app, describe_app
|
6
7
|
from sima_cli.utils.config_loader import internal_resource_exists
|
7
8
|
from sima_cli.mla.meminfo import monitor_simaai_mem_chart
|
8
9
|
from sima_cli.__version__ import __version__
|
@@ -167,12 +168,25 @@ def update(ctx, version_or_url, ip, yes, passwd, flavor):
|
|
167
168
|
# Model Zoo Subcommands
|
168
169
|
# ----------------------
|
169
170
|
@main.group()
|
170
|
-
@click.option(
|
171
|
+
@click.option(
|
172
|
+
"-v", "--ver", "--version",
|
173
|
+
"ver",
|
174
|
+
required=False,
|
175
|
+
help="SDK version (e.g. 1.7.0, 2.0.0). If not provided, you can select from available versions.",
|
176
|
+
)
|
177
|
+
@click.option(
|
178
|
+
"--boardtype",
|
179
|
+
type=click.Choice(["mlsoc", "modalix"], case_sensitive=False),
|
180
|
+
default='modalix',
|
181
|
+
required=False,
|
182
|
+
help="Target board type (mlsoc or modalix).",
|
183
|
+
)
|
171
184
|
@click.pass_context
|
172
|
-
def modelzoo(ctx, ver):
|
185
|
+
def modelzoo(ctx, ver, boardtype):
|
173
186
|
"""Access models from the Model Zoo."""
|
174
187
|
ctx.ensure_object(dict)
|
175
188
|
ctx.obj['ver'] = ver
|
189
|
+
ctx.obj["boardtype"] = boardtype
|
176
190
|
internal = ctx.obj.get("internal", False)
|
177
191
|
if not internal:
|
178
192
|
click.echo(f"Public developer portal environment is not supported yet..")
|
@@ -186,8 +200,9 @@ def list_models_cmd(ctx):
|
|
186
200
|
"""List available models."""
|
187
201
|
internal = ctx.obj.get("internal", False)
|
188
202
|
version = ctx.obj.get("ver")
|
203
|
+
boardtype = ctx.obj.get('boardtype')
|
189
204
|
click.echo(f"Listing models for version: {version}")
|
190
|
-
list_models(internal, version)
|
205
|
+
list_models(internal, version, boardtype)
|
191
206
|
|
192
207
|
@modelzoo.command("get")
|
193
208
|
@click.argument('model_name')
|
@@ -209,6 +224,57 @@ def get_model(ctx, model_name):
|
|
209
224
|
click.echo(f"Getting model '{model_name}' for version: {ver}")
|
210
225
|
describe_model(internal, ver, model_name)
|
211
226
|
|
227
|
+
# ----------------------
|
228
|
+
# App Zoo Subcommands
|
229
|
+
# ----------------------
|
230
|
+
@main.group()
|
231
|
+
@click.option(
|
232
|
+
"-v", "--ver", "--version",
|
233
|
+
"ver",
|
234
|
+
required=False,
|
235
|
+
help="SDK version (e.g. 1.7.0, 2.0.0). If not provided, you can select from available versions.",
|
236
|
+
)
|
237
|
+
@click.pass_context
|
238
|
+
def appzoo(ctx, ver):
|
239
|
+
"""Access models from the Model Zoo."""
|
240
|
+
ctx.ensure_object(dict)
|
241
|
+
ctx.obj['ver'] = ver
|
242
|
+
internal = ctx.obj.get("internal", False)
|
243
|
+
if not internal:
|
244
|
+
click.echo(f"Public developer portal environment is not supported yet..")
|
245
|
+
exit(0)
|
246
|
+
|
247
|
+
pass
|
248
|
+
|
249
|
+
@appzoo.command("list")
|
250
|
+
@click.pass_context
|
251
|
+
def list_apps_cmd(ctx):
|
252
|
+
"""List available models."""
|
253
|
+
internal = ctx.obj.get("internal", False)
|
254
|
+
version = ctx.obj.get("ver")
|
255
|
+
click.echo(f"Listing apps for version: {version}")
|
256
|
+
list_apps(internal, version)
|
257
|
+
|
258
|
+
@appzoo.command("get")
|
259
|
+
@click.argument('app_name')
|
260
|
+
@click.pass_context
|
261
|
+
def get_app(ctx, app_name):
|
262
|
+
"""Download a specific model."""
|
263
|
+
ver = ctx.obj.get("ver")
|
264
|
+
internal = ctx.obj.get("internal", False)
|
265
|
+
click.echo(f"Getting app '{app_name}' for version: {ver}")
|
266
|
+
download_app(internal, ver, app_name)
|
267
|
+
|
268
|
+
@appzoo.command("describe")
|
269
|
+
@click.argument('app_name')
|
270
|
+
@click.pass_context
|
271
|
+
def get_model(ctx, app_name):
|
272
|
+
"""Download a specific model."""
|
273
|
+
ver = ctx.obj.get("ver")
|
274
|
+
internal = ctx.obj.get("internal", False)
|
275
|
+
click.echo(f"Getting model '{app_name}' for version: {ver}")
|
276
|
+
describe_app(internal, ver, app_name)
|
277
|
+
|
212
278
|
# ----------------------
|
213
279
|
# Authentication Command
|
214
280
|
# ----------------------
|
@@ -313,9 +379,7 @@ def install_cmd(ctx, component, version, mirror, tag):
|
|
313
379
|
if component:
|
314
380
|
click.echo(f"⚠️ Component '{component}' is ignored when using --metadata. Proceeding with metadata-based installation.")
|
315
381
|
click.echo(f"🔧 Installing generic component from metadata URL: {mirror}")
|
316
|
-
|
317
|
-
click.echo("✅ Installation complete.")
|
318
|
-
return
|
382
|
+
return install_from_metadata(metadata_url=mirror, internal=internal)
|
319
383
|
|
320
384
|
# No component and no metadata: error
|
321
385
|
if not component:
|
@@ -324,6 +388,10 @@ def install_cmd(ctx, component, version, mirror, tag):
|
|
324
388
|
|
325
389
|
component = component.lower()
|
326
390
|
|
391
|
+
# if user specified gh: as component, treat it the same as -m
|
392
|
+
if component.startswith("gh:"):
|
393
|
+
return install_from_metadata(metadata_url=component, internal=False)
|
394
|
+
|
327
395
|
# Validate version requirement
|
328
396
|
if component in SDK_DEPENDENT_COMPONENTS and not version:
|
329
397
|
click.echo(f"❌ The component '{component}' requires a specific SDK version. Please provide one using -v.")
|
sima_cli/download/downloader.py
CHANGED
@@ -135,6 +135,38 @@ def download_file_from_url(url: str, dest_folder: str = ".", internal: bool = Fa
|
|
135
135
|
|
136
136
|
return dest_path
|
137
137
|
|
138
|
+
def check_url_available(url: str, internal: bool = False) -> bool:
|
139
|
+
"""
|
140
|
+
Perform a HEAD request to check if a resource is available.
|
141
|
+
|
142
|
+
Args:
|
143
|
+
url (str): The full URL to check.
|
144
|
+
internal (bool): Whether this is an internal resource on Artifactory.
|
145
|
+
|
146
|
+
Returns:
|
147
|
+
bool: True if the resource is available (status 200–399), False otherwise.
|
148
|
+
"""
|
149
|
+
headers = {}
|
150
|
+
try:
|
151
|
+
if internal:
|
152
|
+
auth_token = get_auth_token(internal=True)
|
153
|
+
if auth_token:
|
154
|
+
headers["Authorization"] = f"Bearer {auth_token}"
|
155
|
+
head_fn = requests.head
|
156
|
+
elif 'https://docs.sima.ai' in url:
|
157
|
+
session = login_external()
|
158
|
+
head_fn = session.head
|
159
|
+
else:
|
160
|
+
session = requests.Session()
|
161
|
+
head_fn = session.head
|
162
|
+
|
163
|
+
resp = head_fn(url, headers=headers, timeout=10, allow_redirects=True)
|
164
|
+
# Consider any 2xx or 3xx as "available"
|
165
|
+
return 200 <= resp.status_code < 400
|
166
|
+
|
167
|
+
except Exception as e:
|
168
|
+
print(f"⚠️ HEAD check failed for {url}: {e}")
|
169
|
+
return False
|
138
170
|
|
139
171
|
def download_folder_from_url(url: str, dest_folder: str = ".", internal: bool = False) -> List[str]:
|
140
172
|
"""
|
@@ -8,6 +8,7 @@ import shutil
|
|
8
8
|
import tarfile
|
9
9
|
import zipfile
|
10
10
|
import stat
|
11
|
+
import shlex
|
11
12
|
from urllib.parse import urlparse
|
12
13
|
|
13
14
|
from typing import Dict
|
@@ -15,6 +16,7 @@ from tqdm import tqdm
|
|
15
16
|
from urllib.parse import urljoin
|
16
17
|
from pathlib import Path
|
17
18
|
import subprocess
|
19
|
+
import requests
|
18
20
|
|
19
21
|
from rich.console import Console
|
20
22
|
from rich.panel import Panel
|
@@ -29,7 +31,224 @@ from sima_cli.install.metadata_info import print_metadata_summary, parse_size_st
|
|
29
31
|
|
30
32
|
console = Console()
|
31
33
|
|
32
|
-
def
|
34
|
+
def _copy_dir(src: Path, dest: Path, label: str):
|
35
|
+
"""
|
36
|
+
Copy files from src → dest, merging with existing files (no deletion).
|
37
|
+
Does NOT overwrite files if they already exist.
|
38
|
+
Ensures that all parent directories for dest are created.
|
39
|
+
"""
|
40
|
+
if not src.exists():
|
41
|
+
raise FileNotFoundError(f"SDK {label} not found: {src}")
|
42
|
+
|
43
|
+
dest.mkdir(parents=True, exist_ok=True)
|
44
|
+
|
45
|
+
for item in src.iterdir():
|
46
|
+
target = dest / item.name
|
47
|
+
if item.is_dir():
|
48
|
+
_copy_dir(item, target, label)
|
49
|
+
else:
|
50
|
+
if not target.exists():
|
51
|
+
shutil.copy2(item, target)
|
52
|
+
|
53
|
+
click.echo(f"✅ Copied {label} into {dest}")
|
54
|
+
|
55
|
+
def _prepare_pipeline_project(repo_dir: Path):
|
56
|
+
"""
|
57
|
+
Prepare a pipeline project by copying required SDK sources into the repo.
|
58
|
+
|
59
|
+
Steps:
|
60
|
+
1. Copy core sources into the project folder
|
61
|
+
2. Parse .project/pluginsInfo
|
62
|
+
3. Copy required plugin sources from the SDK plugin zoo
|
63
|
+
"""
|
64
|
+
plugins_info_file = repo_dir / ".project" / "pluginsInfo.json"
|
65
|
+
if not plugins_info_file.exists():
|
66
|
+
return
|
67
|
+
|
68
|
+
click.echo("📦 Preparing pipeline project...")
|
69
|
+
|
70
|
+
try:
|
71
|
+
data = json.loads(plugins_info_file.read_text())
|
72
|
+
plugins = data.get("pluginsInfo", [])
|
73
|
+
except Exception as e:
|
74
|
+
raise RuntimeError(f"Failed to read {plugins_info_file}: {e}")
|
75
|
+
|
76
|
+
# Step a: copy core
|
77
|
+
# Define what to copy
|
78
|
+
copy_map = [
|
79
|
+
(
|
80
|
+
Path("/usr/local/simaai/plugin_zoo/gst-simaai-plugins-base/core"),
|
81
|
+
repo_dir / "core",
|
82
|
+
"core"
|
83
|
+
),
|
84
|
+
(
|
85
|
+
Path("/usr/local/simaai/utils/gst_app"),
|
86
|
+
repo_dir / "dependencies" / "gst_app",
|
87
|
+
"dependencies/gst_app"
|
88
|
+
),
|
89
|
+
(
|
90
|
+
Path("/usr/local/simaai/plugin_zoo/gst-simaai-plugins-base/gst/templates"),
|
91
|
+
repo_dir / "plugins" / "templates",
|
92
|
+
"plugins/templates"
|
93
|
+
),
|
94
|
+
]
|
95
|
+
|
96
|
+
# Execute
|
97
|
+
for src, dest, label in copy_map:
|
98
|
+
_copy_dir(src, dest, label)
|
99
|
+
|
100
|
+
# Step b/c: scan plugin paths and copy SDK plugins
|
101
|
+
sdk_plugins_base = Path("/usr/local/simaai/plugin_zoo/gst-simaai-plugins-base/gst")
|
102
|
+
sdk_alt_base = sdk_plugins_base / "PyGast-plugins"
|
103
|
+
|
104
|
+
dest_plugins_dir = repo_dir / "plugins"
|
105
|
+
dest_plugins_dir.mkdir(exist_ok=True)
|
106
|
+
|
107
|
+
for plugin in plugins:
|
108
|
+
try:
|
109
|
+
path = plugin.get("path", "")
|
110
|
+
if not path:
|
111
|
+
continue
|
112
|
+
parts = path.split("/")
|
113
|
+
if len(parts) < 2:
|
114
|
+
continue
|
115
|
+
|
116
|
+
plugin_name = parts[1]
|
117
|
+
|
118
|
+
# Look first in gst/, then fallback to gst/PyGast-plugins/
|
119
|
+
sdk_plugin_path = sdk_plugins_base / plugin_name
|
120
|
+
if not sdk_plugin_path.exists():
|
121
|
+
sdk_plugin_path = sdk_alt_base / plugin_name
|
122
|
+
|
123
|
+
if not sdk_plugin_path.exists():
|
124
|
+
click.echo(
|
125
|
+
f"⚠️ Missing plugin source: {plugin_name} in the SDK, skipping. "
|
126
|
+
"It is likely a custom plugin already in the repo so it's safe to ignore this warning."
|
127
|
+
)
|
128
|
+
continue
|
129
|
+
|
130
|
+
dest_plugin_path = dest_plugins_dir / plugin_name
|
131
|
+
dest_plugin_path.mkdir(parents=True, exist_ok=True)
|
132
|
+
|
133
|
+
# Walk the SDK plugin dir and copy only missing files
|
134
|
+
for src_file in sdk_plugin_path.rglob("*"):
|
135
|
+
if src_file.is_file():
|
136
|
+
rel_path = src_file.relative_to(sdk_plugin_path)
|
137
|
+
dest_file = dest_plugin_path / rel_path
|
138
|
+
if dest_file.exists():
|
139
|
+
click.echo(f"↩️ Skipped existing file in the repo: {dest_file}")
|
140
|
+
continue
|
141
|
+
dest_file.parent.mkdir(parents=True, exist_ok=True)
|
142
|
+
shutil.copy2(src_file, dest_file)
|
143
|
+
|
144
|
+
click.echo(f"✅ Copied plugin {plugin_name} into {dest_plugin_path} (safe copy)")
|
145
|
+
|
146
|
+
except Exception as e:
|
147
|
+
click.echo(f"❌ Error copying plugin {plugin}: {e}")
|
148
|
+
|
149
|
+
click.echo("🎉 Pipeline project prepared.")
|
150
|
+
|
151
|
+
def _download_requirements_wheels(repo_dir: Path):
|
152
|
+
"""
|
153
|
+
Look for resources/dependencies/requirements.txt under the repo,
|
154
|
+
parse each line, and download wheels into the same folder.
|
155
|
+
Supports optional pip download flags in parentheses.
|
156
|
+
|
157
|
+
Example line formats:
|
158
|
+
jax==0.6.2
|
159
|
+
jaxlib==0.6.2 (--platform manylinux2014_aarch64 --python-version 310 --abi cp310)
|
160
|
+
"""
|
161
|
+
deps_dir = repo_dir / "resources" / "dependencies"
|
162
|
+
req_file = deps_dir / "requirements.txt"
|
163
|
+
|
164
|
+
if not req_file.exists():
|
165
|
+
click.echo("⚠️ No requirements.txt found under resources/dependencies, skipping wheel download.")
|
166
|
+
return
|
167
|
+
|
168
|
+
deps_dir.mkdir(parents=True, exist_ok=True)
|
169
|
+
|
170
|
+
with req_file.open("r") as f:
|
171
|
+
lines = [line.strip() for line in f if line.strip() and not line.startswith("#")]
|
172
|
+
|
173
|
+
if not lines:
|
174
|
+
click.echo("⚠️ requirements.txt is empty, nothing to download.")
|
175
|
+
return
|
176
|
+
|
177
|
+
for line in lines:
|
178
|
+
# Split package and extra params if present
|
179
|
+
if "(" in line and ")" in line:
|
180
|
+
pkg_part, extra = line.split("(", 1)
|
181
|
+
package = pkg_part.strip()
|
182
|
+
extra_args = shlex.split(extra.strip(") "))
|
183
|
+
else:
|
184
|
+
package = line.strip()
|
185
|
+
extra_args = []
|
186
|
+
|
187
|
+
click.echo(f"⬇️ Downloading {package} {extra_args if extra_args else ''}")
|
188
|
+
|
189
|
+
try:
|
190
|
+
cmd = [
|
191
|
+
"pip3", "download", "--no-deps",
|
192
|
+
"--only-binary=:all:",
|
193
|
+
"-d", str(deps_dir),
|
194
|
+
package,
|
195
|
+
] + extra_args
|
196
|
+
|
197
|
+
rc = os.system(" ".join(shlex.quote(c) for c in cmd))
|
198
|
+
if rc != 0:
|
199
|
+
click.echo(f"❌ pip download failed for {package}")
|
200
|
+
else:
|
201
|
+
click.echo(f"✅ Downloaded {package} into {deps_dir}")
|
202
|
+
except Exception as e:
|
203
|
+
click.echo(f"❌ Error downloading {package}: {e}")
|
204
|
+
|
205
|
+
def _download_github_repo(owner: str, repo: str, ref: str, dest_folder: str, token: str = None) -> str:
|
206
|
+
"""
|
207
|
+
Download and extract a GitHub repo tarball via the REST API (no git required).
|
208
|
+
|
209
|
+
Args:
|
210
|
+
owner (str): GitHub org/user
|
211
|
+
repo (str): Repo name
|
212
|
+
ref (str): Branch, tag, or commit (default = default branch)
|
213
|
+
dest_folder (str): Where to extract
|
214
|
+
token (str): Optional GitHub token for private repos
|
215
|
+
|
216
|
+
Returns:
|
217
|
+
str: Path to the extracted repo
|
218
|
+
"""
|
219
|
+
url = f"https://api.github.com/repos/{owner}/{repo}/tarball/{ref}" if ref else f"https://api.github.com/repos/{owner}/{repo}/tarball"
|
220
|
+
headers = {}
|
221
|
+
if token:
|
222
|
+
headers["Authorization"] = f"Bearer {token}"
|
223
|
+
|
224
|
+
click.echo(f"🐙 Downloading GitHub repo: {owner}/{repo}" + (f"@{ref}" if ref else ""))
|
225
|
+
|
226
|
+
with requests.get(url, headers=headers, stream=True) as r:
|
227
|
+
if r.status_code in (401, 403):
|
228
|
+
raise PermissionError("Authentication required for GitHub repo")
|
229
|
+
r.raise_for_status()
|
230
|
+
|
231
|
+
with tempfile.NamedTemporaryFile(delete=False, suffix=".tar.gz") as tmp_file:
|
232
|
+
for chunk in r.iter_content(chunk_size=8192):
|
233
|
+
tmp_file.write(chunk)
|
234
|
+
tmp_path = Path(tmp_file.name)
|
235
|
+
|
236
|
+
repo_dir = Path(dest_folder) / repo
|
237
|
+
repo_dir.mkdir(parents=True, exist_ok=True)
|
238
|
+
|
239
|
+
_extract_tar_strip_top_level(tmp_path, repo_dir)
|
240
|
+
tmp_path.unlink(missing_ok=True)
|
241
|
+
click.echo(f"✅ Downloaded GitHub repo to folder: {repo_dir}")
|
242
|
+
_download_requirements_wheels(repo_dir=repo_dir)
|
243
|
+
|
244
|
+
try:
|
245
|
+
_prepare_pipeline_project(repo_dir)
|
246
|
+
except Exception as e:
|
247
|
+
click.echo(f"⚠️ Pipeline preparation skipped: {e}")
|
248
|
+
|
249
|
+
return str(repo_dir)
|
250
|
+
|
251
|
+
def _download_assets(metadata: dict, base_url: str, dest_folder: str, internal: bool = False, skip_models: bool = False, tag: str = None) -> list:
|
33
252
|
"""
|
34
253
|
Downloads resources defined in metadata to a local destination folder.
|
35
254
|
|
@@ -39,6 +258,7 @@ def _download_assets(metadata: dict, base_url: str, dest_folder: str, internal:
|
|
39
258
|
dest_folder (str): Local path to download resources into
|
40
259
|
internal (bool): Whether to use internal download routing (if applicable)
|
41
260
|
skip_models (bool): If True, skips downloading any file path starting with 'models/'
|
261
|
+
tag (str): metadata.json tag from GitHub will be passed into the resources in the file
|
42
262
|
|
43
263
|
Returns:
|
44
264
|
list: Paths to the downloaded local files
|
@@ -84,11 +304,34 @@ def _download_assets(metadata: dict, base_url: str, dest_folder: str, internal:
|
|
84
304
|
model_path = snapshot_download(
|
85
305
|
repo_id=repo_id,
|
86
306
|
local_dir=target_dir,
|
87
|
-
|
307
|
+
local_dir_use_symlinks=False
|
88
308
|
)
|
89
309
|
local_paths.append(model_path)
|
90
310
|
continue
|
91
311
|
|
312
|
+
if resource.startswith("gh:"):
|
313
|
+
resource_spec = resource[3:]
|
314
|
+
if "@" in resource_spec:
|
315
|
+
repo_id, ref = resource_spec.split("@", 1)
|
316
|
+
else:
|
317
|
+
repo_id, ref = resource_spec, tag
|
318
|
+
|
319
|
+
if "/" not in repo_id:
|
320
|
+
raise click.ClickException(f"❌ Invalid GitHub repo spec: {resource}")
|
321
|
+
|
322
|
+
owner, name = repo_id.split("/", 1)
|
323
|
+
|
324
|
+
try:
|
325
|
+
token = os.getenv("GITHUB_TOKEN", None)
|
326
|
+
repo_path = _download_github_repo(owner, name, ref, dest_folder, token)
|
327
|
+
except Exception as e:
|
328
|
+
raise click.ClickException(
|
329
|
+
f"❌ Failed to download GitHub repo {owner}/{name}@{ref or 'default'}: {e}"
|
330
|
+
)
|
331
|
+
|
332
|
+
local_paths.append(repo_path)
|
333
|
+
continue
|
334
|
+
|
92
335
|
# Handle normal relative or absolute URLs
|
93
336
|
resource_url = urljoin(base_url, resource)
|
94
337
|
local_path = download_file_from_url(
|
@@ -296,6 +539,25 @@ def _extract_zip_streaming(zip_path: Path, extract_dir: Path, overwrite: bool =
|
|
296
539
|
|
297
540
|
print(f"✅ Extracted {len(members)} entries to {extract_dir}/")
|
298
541
|
|
542
|
+
def _extract_tar_strip_top_level(tar_path: Path, extract_dir: Path):
|
543
|
+
"""Extract a GitHub tarball, stripping the top-level folder."""
|
544
|
+
with tarfile.open(tar_path, "r:*") as tar:
|
545
|
+
members = tar.getmembers()
|
546
|
+
|
547
|
+
# Detect top-level prefix (first part before '/')
|
548
|
+
top_level = None
|
549
|
+
if members:
|
550
|
+
first_name = members[0].name
|
551
|
+
top_level = first_name.split("/", 1)[0]
|
552
|
+
|
553
|
+
for member in members:
|
554
|
+
# Strip top-level folder
|
555
|
+
if top_level and member.name.startswith(top_level + "/"):
|
556
|
+
member.name = member.name[len(top_level) + 1 :]
|
557
|
+
if not member.name:
|
558
|
+
continue
|
559
|
+
tar.extract(member, path=extract_dir)
|
560
|
+
|
299
561
|
def _combine_multipart_files(folder: str):
|
300
562
|
"""
|
301
563
|
Scan a folder for multipart files like name-split-aa, -ab, etc.,
|
@@ -353,20 +615,27 @@ def _combine_multipart_files(folder: str):
|
|
353
615
|
|
354
616
|
print(f"✅ Extracted to: {extract_dir}/")
|
355
617
|
|
356
|
-
def _extract_archives_in_folder(folder: str):
|
618
|
+
def _extract_archives_in_folder(folder: str, local_paths):
|
357
619
|
"""
|
358
|
-
Extract
|
620
|
+
Extract .tar, .gz, .tar.gz, and .zip files in the given folder,
|
621
|
+
but only if they are listed in local_paths.
|
359
622
|
Uses streaming to avoid NFS performance issues.
|
360
623
|
"""
|
361
624
|
folder = Path(folder)
|
625
|
+
local_paths = {str(Path(p).resolve()) for p in local_paths}
|
626
|
+
|
362
627
|
for file in folder.iterdir():
|
363
628
|
if not file.is_file():
|
364
629
|
continue
|
365
630
|
|
366
|
-
|
367
|
-
if
|
631
|
+
file_resolved = str(file.resolve())
|
632
|
+
if file_resolved not in local_paths:
|
633
|
+
continue
|
634
|
+
|
635
|
+
# TAR, GZ, TAR.GZ → all handled by _extract_tar_streaming
|
636
|
+
if file.suffix in [".tar", ".gz"] or file.name.endswith(".tar.gz"):
|
368
637
|
extract_dir = folder / file.stem.replace(".tar", "")
|
369
|
-
print(f"📦 Extracting TAR
|
638
|
+
print(f"📦 Extracting TAR/GZ: {file.name} to {extract_dir}/")
|
370
639
|
_extract_tar_streaming(file, extract_dir)
|
371
640
|
|
372
641
|
# ZIP
|
@@ -471,17 +740,56 @@ def _run_installation_script(metadata: Dict, extract_path: str = "."):
|
|
471
740
|
|
472
741
|
print("✅ Installation completed successfully.")
|
473
742
|
|
743
|
+
def _resolve_github_metadata_url(gh_ref: str) -> str:
|
744
|
+
"""
|
745
|
+
Resolve a GitHub shorthand like gh:org/repo@tag into a raw URL for metadata.json.
|
746
|
+
If tag is omitted, defaults to 'main'.
|
747
|
+
"""
|
748
|
+
try:
|
749
|
+
_, repo_ref = gh_ref.split(":", 1) # remove 'gh:'
|
750
|
+
if "@" in repo_ref:
|
751
|
+
org_repo, tag = repo_ref.split("@", 1)
|
752
|
+
else:
|
753
|
+
org_repo, tag = repo_ref, "main"
|
754
|
+
|
755
|
+
owner, repo = org_repo.split("/", 1)
|
756
|
+
token = os.getenv("GITHUB_TOKEN")
|
757
|
+
|
758
|
+
# Use GitHub API to fetch the metadata.json file
|
759
|
+
api_url = f"https://api.github.com/repos/{owner}/{repo}/contents/metadata.json?ref={tag}"
|
760
|
+
headers = {"Accept": "application/vnd.github.v3.raw"}
|
761
|
+
if token:
|
762
|
+
headers["Authorization"] = f"token {token}"
|
763
|
+
|
764
|
+
r = requests.get(api_url, headers=headers)
|
765
|
+
r.raise_for_status()
|
766
|
+
|
767
|
+
# Write metadata.json locally so downstream logic can use a file/URL
|
768
|
+
local_path = os.path.join("/tmp", f"{repo}-{tag}-metadata.json")
|
769
|
+
with open(local_path, "wb") as f:
|
770
|
+
f.write(r.content)
|
771
|
+
|
772
|
+
return local_path, tag
|
773
|
+
except Exception as e:
|
774
|
+
raise RuntimeError(f"Failed to resolve GitHub metadata URL {gh_ref}: {e}")
|
775
|
+
|
474
776
|
def install_from_metadata(metadata_url: str, internal: bool, install_dir: str = '.'):
|
475
777
|
try:
|
778
|
+
tag = None
|
779
|
+
|
780
|
+
if metadata_url.startswith("gh:"):
|
781
|
+
metadata_url, tag = _resolve_github_metadata_url(metadata_url)
|
782
|
+
internal = False
|
783
|
+
|
476
784
|
metadata, _ = _download_and_validate_metadata(metadata_url, internal)
|
477
785
|
print_metadata_summary(metadata=metadata)
|
478
786
|
|
479
787
|
if _check_whether_disk_is_big_enough(metadata):
|
480
788
|
if _is_platform_compatible(metadata):
|
481
|
-
local_paths = _download_assets(metadata, metadata_url, install_dir, internal)
|
789
|
+
local_paths = _download_assets(metadata, metadata_url, install_dir, internal, tag=tag)
|
482
790
|
if len(local_paths) > 0:
|
483
791
|
_combine_multipart_files(install_dir)
|
484
|
-
_extract_archives_in_folder(install_dir)
|
792
|
+
_extract_archives_in_folder(install_dir, local_paths)
|
485
793
|
_run_installation_script(metadata=metadata, extract_path=install_dir)
|
486
794
|
|
487
795
|
except Exception as e:
|
sima_cli/model_zoo/model.py
CHANGED
@@ -4,6 +4,7 @@ import requests
|
|
4
4
|
import click
|
5
5
|
import os
|
6
6
|
import yaml
|
7
|
+
from InquirerPy import inquirer
|
7
8
|
from urllib.parse import urlparse
|
8
9
|
from rich import print
|
9
10
|
from rich.table import Table
|
@@ -186,35 +187,38 @@ def _download_model_internal(ver: str, model_name: str):
|
|
186
187
|
else:
|
187
188
|
click.echo("⚠️ model_path.txt exists but does not contain a valid URL.")
|
188
189
|
|
189
|
-
def _list_available_models_internal(version: str):
|
190
|
+
def _list_available_models_internal(version: str, boardtype: str):
|
191
|
+
"""
|
192
|
+
Query Artifactory for available models for the given SDK version.
|
193
|
+
Display them in an interactive menu with an 'Exit' option.
|
194
|
+
Apply boardtype filtering:
|
195
|
+
- gen1_target* → only shown for mlsoc
|
196
|
+
- gen2_target* → only shown for modalix
|
197
|
+
- others → always shown
|
198
|
+
"""
|
190
199
|
repo_path = f"SiMaCLI-SDK-Releases/{version}-Release/modelzoo_edgematic"
|
191
200
|
aql_query = f"""
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
}}).include("repo", "path", "name")
|
199
|
-
""".strip()
|
201
|
+
items.find({{
|
202
|
+
"repo": "sima-qa-releases",
|
203
|
+
"path": {{"$match": "{repo_path}/*"}},
|
204
|
+
"type": "folder"
|
205
|
+
}}).include("repo","path","name")
|
206
|
+
""".strip()
|
200
207
|
|
201
208
|
aql_url = f"{ARTIFACTORY_BASE_URL}/api/search/aql"
|
202
|
-
print(aql_url)
|
203
209
|
headers = {
|
204
210
|
"Content-Type": "text/plain",
|
205
211
|
"Authorization": f"Bearer {get_auth_token(internal=True)}"
|
206
212
|
}
|
207
213
|
|
208
214
|
response = requests.post(aql_url, data=aql_query, headers=headers)
|
209
|
-
|
210
215
|
if response.status_code != 200:
|
211
|
-
click.echo(f"Failed to retrieve model list
|
216
|
+
click.echo(f"❌ Failed to retrieve model list (status {response.status_code})")
|
212
217
|
click.echo(response.text)
|
213
|
-
return
|
218
|
+
return None
|
214
219
|
|
215
220
|
results = response.json().get("results", [])
|
216
|
-
|
217
|
-
base_prefix = f"SiMaCLI-SDK-Releases/{version}-Release/modelzoo_edgematic/"
|
221
|
+
base_prefix = f"{repo_path}/"
|
218
222
|
model_paths = sorted({
|
219
223
|
item["path"].replace(base_prefix, "").rstrip("/") + "/" + item["name"]
|
220
224
|
for item in results
|
@@ -222,20 +226,63 @@ def _list_available_models_internal(version: str):
|
|
222
226
|
|
223
227
|
if not model_paths:
|
224
228
|
click.echo("No models found.")
|
225
|
-
return
|
226
|
-
|
227
|
-
#
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
229
|
+
return None
|
230
|
+
|
231
|
+
# Apply boardtype filtering
|
232
|
+
filtered_models = []
|
233
|
+
for model in model_paths:
|
234
|
+
if model.startswith("gen1_target") and boardtype != "mlsoc":
|
235
|
+
continue
|
236
|
+
if model.startswith("gen2_target") and boardtype != "modalix":
|
237
|
+
continue
|
238
|
+
filtered_models.append(model)
|
239
|
+
|
240
|
+
if not filtered_models:
|
241
|
+
click.echo(f"No models found for board type '{boardtype}'.")
|
242
|
+
return None
|
243
|
+
|
244
|
+
while True:
|
245
|
+
# Add Exit option
|
246
|
+
choices = filtered_models + ["Exit"]
|
247
|
+
|
248
|
+
# Interactive selection with InquirerPy
|
249
|
+
selected_model = inquirer.fuzzy(
|
250
|
+
message=f"Select a model from version {version}:",
|
251
|
+
choices=choices,
|
252
|
+
max_height="70%",
|
253
|
+
instruction="(Use ↑↓ to navigate, / to search, Enter to select)"
|
254
|
+
).execute()
|
255
|
+
|
256
|
+
if selected_model == "Exit":
|
257
|
+
click.echo("👋 Exiting without selecting a model.")
|
258
|
+
return None
|
259
|
+
|
260
|
+
click.echo(f"✅ Selected model: {selected_model}")
|
261
|
+
|
262
|
+
# Auto-describe
|
263
|
+
_describe_model_internal(version, selected_model)
|
264
|
+
|
265
|
+
# Action menu loop
|
266
|
+
while True:
|
267
|
+
action = inquirer.select(
|
268
|
+
message=f"What do you want to do with {selected_model}?",
|
269
|
+
choices=["Download model", "Back", "Exit"],
|
270
|
+
default="Download model",
|
271
|
+
qmark="👉",
|
272
|
+
).execute()
|
273
|
+
|
274
|
+
if action == "Download model":
|
275
|
+
_download_model_internal(version, selected_model)
|
276
|
+
elif action == "Back":
|
277
|
+
break # back to model list
|
278
|
+
else: # Exit
|
279
|
+
click.echo("👋 Exiting.")
|
280
|
+
return None
|
281
|
+
|
282
|
+
def list_models(internal, ver, boardtype):
|
236
283
|
if internal:
|
237
284
|
click.echo("Model Zoo Source : SiMa Artifactory...")
|
238
|
-
return _list_available_models_internal(ver)
|
285
|
+
return _list_available_models_internal(ver, boardtype)
|
239
286
|
else:
|
240
287
|
print('External model zoo not supported yet')
|
241
288
|
|
@@ -260,4 +307,5 @@ if __name__ == "__main__":
|
|
260
307
|
print("Usage: python models.py <version>")
|
261
308
|
else:
|
262
309
|
version_arg = sys.argv[1]
|
263
|
-
|
310
|
+
boardtype = sys.argv[2]
|
311
|
+
_list_available_models_internal(version_arg, boardtype)
|
@@ -1,26 +1,26 @@
|
|
1
1
|
sima_cli/__init__.py,sha256=Nb2jSg9-CX1XvSc1c21U9qQ3atINxphuNkNfmR-9P3o,332
|
2
2
|
sima_cli/__main__.py,sha256=ehzD6AZ7zGytC2gLSvaJatxeD0jJdaEvNJvwYeGsWOg,69
|
3
|
-
sima_cli/__version__.py,sha256=
|
4
|
-
sima_cli/cli.py,sha256=
|
3
|
+
sima_cli/__version__.py,sha256=4m-QBPuK5Old2YxxO8B7YbrZhHPXQ0fWWTNKgReu9TE,49
|
4
|
+
sima_cli/cli.py,sha256=uPfpXDkmX_bv5s1zjUD_pZabxnRxihg6ad_2obs9IZ8,19276
|
5
5
|
sima_cli/app_zoo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
6
|
-
sima_cli/app_zoo/app.py,sha256=
|
6
|
+
sima_cli/app_zoo/app.py,sha256=6u3iIqVZkuMK49kK0f3dVJCCf5-qC-xzLOS78-TkeN8,15738
|
7
7
|
sima_cli/auth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
-
sima_cli/auth/basic_auth.py,sha256=
|
8
|
+
sima_cli/auth/basic_auth.py,sha256=UMEXCCnNQpjpp6RZxREs6iiKtYyaeqZBnTSR0wA8s6Q,8767
|
9
9
|
sima_cli/auth/login.py,sha256=nE-dSHK_husXw1XJaEcOe3I1_bnwHkLgO_BqKuQODDM,3781
|
10
10
|
sima_cli/data/resources_internal.yaml,sha256=zlQD4cSnZK86bLtTWuvEudZTARKiuIKmB--Jv4ajL8o,200
|
11
11
|
sima_cli/data/resources_public.yaml,sha256=U7hmUomGeQ2ULdo1BU2OQHr0PyKBamIdK9qrutDlX8o,201
|
12
12
|
sima_cli/download/__init__.py,sha256=6y4O2FOCYFR2jdnQoVi3hRtEoZ0Gw6rydlTy1SGJ5FE,218
|
13
|
-
sima_cli/download/downloader.py,sha256=
|
13
|
+
sima_cli/download/downloader.py,sha256=UQdrBWLQsPQygaoVaufOjbzWmRoNnk6pgLdnbnVi04U,6436
|
14
14
|
sima_cli/install/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
15
15
|
sima_cli/install/hostdriver.py,sha256=kAWDLebs60mbWIyTbUxmNrChcKW1uD5r7FtWNSUVUE4,5852
|
16
16
|
sima_cli/install/metadata_info.py,sha256=wmMqwzGfXbuilkqaxRVrFOzOtTOiONkmPCyA2oDAQpA,2168
|
17
|
-
sima_cli/install/metadata_installer.py,sha256=
|
17
|
+
sima_cli/install/metadata_installer.py,sha256=DtfuyBy0eGm6itL6wC9d4tj16nIDRxvszEQ7z2ogW74,30306
|
18
18
|
sima_cli/install/metadata_validator.py,sha256=7954rp9vFRNnqmIMvCVTjq40kUIEbGXzfc8HmQmChe0,5221
|
19
19
|
sima_cli/install/optiview.py,sha256=r4DYdQDTUbZVCR87hl5T21gsjZrhqpU8hWnYxKmUJ_k,4790
|
20
20
|
sima_cli/install/palette.py,sha256=uRznoHa4Mv9ZXHp6AoqknfC3RxpYNKi9Ins756Cyifk,3930
|
21
21
|
sima_cli/mla/meminfo.py,sha256=ndc8kQJmWGEIdvNh6iIhATGdrkqM2pbddr_eHxaPNfg,1466
|
22
22
|
sima_cli/model_zoo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
23
|
-
sima_cli/model_zoo/model.py,sha256=
|
23
|
+
sima_cli/model_zoo/model.py,sha256=WjkVd5ZNadWkBRaOpiBBvgFmrZ2CcEtV56vj3h63eH8,11613
|
24
24
|
sima_cli/network/network.py,sha256=kXYI2oxgeIg7LoGt2VKF9JlK8DIfQT87A9-x-D6uHCA,7714
|
25
25
|
sima_cli/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
26
26
|
sima_cli/sdk/syscheck.py,sha256=h9zCULW67y4i2hqiGc-hc1ucBDShA5FAe9NxwBGq-fM,4575
|
@@ -46,7 +46,7 @@ sima_cli/utils/env.py,sha256=LtV8S1kCkOpi-7Gj4rhidQRN13x_NDKy4W_LxujheeI,8400
|
|
46
46
|
sima_cli/utils/net.py,sha256=WVntA4CqipkNrrkA4tBVRadJft_pMcGYh4Re5xk3rqo,971
|
47
47
|
sima_cli/utils/network.py,sha256=UvqxbqbWUczGFyO-t1SybG7Q-x9kjUVRNIn_D6APzy8,1252
|
48
48
|
sima_cli/utils/pkg_update_check.py,sha256=IAV_NAOsBDL_lYNYMRYfdZWuVq-rJ_zzHjJJZ7UQaoc,3274
|
49
|
-
sima_cli-0.0.
|
49
|
+
sima_cli-0.0.37.dist-info/licenses/LICENSE,sha256=a260OFuV4SsMZ6sQCkoYbtws_4o2deFtbnT9kg7Rfd4,1082
|
50
50
|
tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
51
51
|
tests/test_app_zoo.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
52
52
|
tests/test_auth.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -55,8 +55,8 @@ tests/test_download.py,sha256=t87DwxlHs26_ws9rpcHGwr_OrcRPd3hz6Zmm0vRee2U,4465
|
|
55
55
|
tests/test_firmware.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
56
56
|
tests/test_model_zoo.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
57
57
|
tests/test_utils.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
58
|
-
sima_cli-0.0.
|
59
|
-
sima_cli-0.0.
|
60
|
-
sima_cli-0.0.
|
61
|
-
sima_cli-0.0.
|
62
|
-
sima_cli-0.0.
|
58
|
+
sima_cli-0.0.37.dist-info/METADATA,sha256=q4R_yQkhPyP_qD9c7ZIwI5RN-2wnMVyw1mChmwrhG4o,3705
|
59
|
+
sima_cli-0.0.37.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
60
|
+
sima_cli-0.0.37.dist-info/entry_points.txt,sha256=xRYrDq1nCs6R8wEdB3c1kKuimxEjWJkHuCzArQPT0Xk,47
|
61
|
+
sima_cli-0.0.37.dist-info/top_level.txt,sha256=FtrbAUdHNohtEPteOblArxQNwoX9_t8qJQd59fagDlc,15
|
62
|
+
sima_cli-0.0.37.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|