socaity-cli 0.1.6__tar.gz → 0.1.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/PKG-INFO +1 -1
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/pyproject.toml +1 -1
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/auth.py +5 -1
- socaity_cli-0.1.7/socaity_cli/deploy_messages.py +171 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/deployment.py +227 -59
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/errors.py +7 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/http.py +10 -2
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/registry_push.py +133 -12
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli.egg-info/PKG-INFO +1 -1
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli.egg-info/SOURCES.txt +1 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/test/test_registry_push.py +28 -1
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/README.md +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/setup.cfg +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/__init__.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/__main__.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/apipod_commands.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/backend_client.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/cli.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/clients/__init__.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/clients/analytics.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/clients/catalog.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/clients/deployment.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/clients/files.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/clients/jobs.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/clients/projects.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/clients/sdk_install.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/credentials.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/prompts.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/requires.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli/sdk_commands.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli.egg-info/dependency_links.txt +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli.egg-info/entry_points.txt +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli.egg-info/requires.txt +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/socaity_cli.egg-info/top_level.txt +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/test/test_cli.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/test/test_credentials.py +0 -0
- {socaity_cli-0.1.6 → socaity_cli-0.1.7}/test/test_private_slot_error.py +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "socaity-cli"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.7"
|
|
8
8
|
description = "Socaity platform CLI: login, catalog browsing, SDK installs and APIPod deployment commands."
|
|
9
9
|
requires-python = ">=3.8"
|
|
10
10
|
authors = [
|
|
@@ -7,6 +7,7 @@ import shutil
|
|
|
7
7
|
import subprocess
|
|
8
8
|
import time
|
|
9
9
|
import webbrowser
|
|
10
|
+
from importlib.metadata import PackageNotFoundError, version as pkg_version
|
|
10
11
|
from typing import Any, Dict, Optional
|
|
11
12
|
from urllib.parse import urljoin
|
|
12
13
|
|
|
@@ -20,7 +21,10 @@ from socaity_cli.credentials import (
|
|
|
20
21
|
save_credentials,
|
|
21
22
|
)
|
|
22
23
|
|
|
23
|
-
|
|
24
|
+
try:
|
|
25
|
+
USER_AGENT = f"socaity-cli/{pkg_version('socaity-cli')}"
|
|
26
|
+
except PackageNotFoundError:
|
|
27
|
+
USER_AGENT = "socaity-cli"
|
|
24
28
|
|
|
25
29
|
|
|
26
30
|
def _backend_base_url() -> str:
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""User-facing copy for deploy progress and failures.
|
|
2
|
+
|
|
3
|
+
Backend status/phase/error stay machine-readable. This module turns them
|
|
4
|
+
into short sentences a non-technical user can follow in the CLI.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
from typing import Any, Dict, Optional, Tuple
|
|
11
|
+
|
|
12
|
+
from socaity_cli.credentials import DEFAULT_FRONTEND_URL
|
|
13
|
+
|
|
14
|
+
_WORKER_COUNTS_RE = re.compile(
|
|
15
|
+
r"RunPod workers\s+ready=(?P<ready>\d+)\s+initializing=(?P<initializing>\d+)\s+"
|
|
16
|
+
r"unhealthy=(?P<unhealthy>\d+)\s+idle=(?P<idle>\d+)",
|
|
17
|
+
re.IGNORECASE,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
_STEP_BY_STATUS = {
|
|
21
|
+
"draft": "Preparing your deployment...",
|
|
22
|
+
"awaiting_image": "Waiting to upload your image...",
|
|
23
|
+
"pushing": "Uploading your image...",
|
|
24
|
+
"validating": "Checking your image...",
|
|
25
|
+
"promoting": "Saving your image...",
|
|
26
|
+
"provisioning": "Starting your service...",
|
|
27
|
+
"live": "Your service is live.",
|
|
28
|
+
"failed_push": "Could not upload your image.",
|
|
29
|
+
"failed_validation": "Your image did not pass checks.",
|
|
30
|
+
"failed_provision": "Could not start your service.",
|
|
31
|
+
"cancelled": "This deployment was cancelled.",
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
_STEP_BY_PHASE = {
|
|
35
|
+
"queued_for_validation": "Checking your image...",
|
|
36
|
+
"vulnerability_scan": "Scanning your image for security issues...",
|
|
37
|
+
"promoting_image": "Saving your image...",
|
|
38
|
+
"provision_queued": "Starting your service...",
|
|
39
|
+
"waiting_for_service_health": "Waiting for your service to start...",
|
|
40
|
+
"waiting_for_openapi": "Your service is running. Finishing setup...",
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def site_url() -> str:
|
|
45
|
+
return os.environ.get("SOCAITY_FRONTEND_URL", DEFAULT_FRONTEND_URL).rstrip("/")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def service_page_url(service_id: Optional[str]) -> str:
|
|
49
|
+
base = site_url()
|
|
50
|
+
if service_id:
|
|
51
|
+
return f"{base}/account/my-services/{service_id}"
|
|
52
|
+
return f"{base}/account/hosting"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def parse_worker_counts(text: Optional[str]) -> Optional[Dict[str, int]]:
|
|
56
|
+
if not text:
|
|
57
|
+
return None
|
|
58
|
+
match = _WORKER_COUNTS_RE.search(text)
|
|
59
|
+
if not match:
|
|
60
|
+
return None
|
|
61
|
+
return {key: int(value) for key, value in match.groupdict().items()}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def is_progress_noise(text: Optional[str]) -> bool:
|
|
65
|
+
"""True when a backend 'error' field is actually worker progress."""
|
|
66
|
+
if not text:
|
|
67
|
+
return False
|
|
68
|
+
if "traceback" in text.lower():
|
|
69
|
+
return False
|
|
70
|
+
return parse_worker_counts(text) is not None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _worker_detail(counts: Dict[str, int], phase: Optional[str]) -> str:
|
|
74
|
+
if counts.get("unhealthy", 0) > 0:
|
|
75
|
+
return "The machine reported a problem. Waiting to confirm..."
|
|
76
|
+
if counts.get("initializing", 0) > 0:
|
|
77
|
+
return "A machine is starting up. This can take a few minutes."
|
|
78
|
+
live = counts.get("ready", 0) + counts.get("idle", 0)
|
|
79
|
+
if live > 0:
|
|
80
|
+
if phase == "waiting_for_openapi":
|
|
81
|
+
return "Your service is running. Finishing setup..."
|
|
82
|
+
return "Your service is running. Almost done..."
|
|
83
|
+
return "Waiting for a machine. This can take a few minutes."
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def progress_lines(status: Dict[str, Any]) -> Tuple[str, Optional[str]]:
|
|
87
|
+
"""Return ``(step, optional detail)`` for one poll snapshot."""
|
|
88
|
+
state = str(status.get("status") or "").strip().lower()
|
|
89
|
+
phase = status.get("phase")
|
|
90
|
+
phase_key = str(phase).strip() if phase else None
|
|
91
|
+
error = status.get("error")
|
|
92
|
+
analysis = status.get("analysis") if isinstance(status.get("analysis"), dict) else {}
|
|
93
|
+
hint = None
|
|
94
|
+
if isinstance(analysis, dict):
|
|
95
|
+
hint = analysis.get("runpod_progress")
|
|
96
|
+
hint = hint or (error if is_progress_noise(error) else None)
|
|
97
|
+
|
|
98
|
+
explicit = status.get("progress") or status.get("message")
|
|
99
|
+
if isinstance(explicit, str) and explicit.strip() and not is_progress_noise(explicit):
|
|
100
|
+
return explicit.strip(), None
|
|
101
|
+
|
|
102
|
+
step = _STEP_BY_PHASE.get(phase_key) or _STEP_BY_STATUS.get(state) or "Working on your deployment..."
|
|
103
|
+
detail = None
|
|
104
|
+
counts = parse_worker_counts(hint if isinstance(hint, str) else None)
|
|
105
|
+
if counts and state == "provisioning":
|
|
106
|
+
detail = _worker_detail(counts, phase_key)
|
|
107
|
+
if detail == step:
|
|
108
|
+
detail = None
|
|
109
|
+
return step, detail
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def humanize_failure(error: Optional[str], status: Optional[str] = None) -> str:
|
|
113
|
+
"""One-sentence failure for the user. Technical text is not the headline."""
|
|
114
|
+
state = str(status or "").strip().lower()
|
|
115
|
+
text = (error or "").strip()
|
|
116
|
+
lower = text.lower()
|
|
117
|
+
|
|
118
|
+
if state == "cancelled":
|
|
119
|
+
return "This deployment was cancelled."
|
|
120
|
+
if is_progress_noise(text):
|
|
121
|
+
text = ""
|
|
122
|
+
lower = ""
|
|
123
|
+
|
|
124
|
+
if "user is stopped" in lower:
|
|
125
|
+
return (
|
|
126
|
+
"Your account is paused. Check billing, then try deploying again.\n"
|
|
127
|
+
f" {site_url()}/account/billing"
|
|
128
|
+
)
|
|
129
|
+
if "no active" in lower and "vram" in lower:
|
|
130
|
+
return (
|
|
131
|
+
"No GPU with that much memory is available right now. "
|
|
132
|
+
"Lower gpu_vram_gb in your config, or try again later."
|
|
133
|
+
)
|
|
134
|
+
if "could not assign a worker" in lower:
|
|
135
|
+
return "We could not start a machine for your service. Please try again in a few minutes."
|
|
136
|
+
if "no live or starting workers" in lower or "no live workers" in lower:
|
|
137
|
+
return "The service stopped before it finished starting. Please try deploying again."
|
|
138
|
+
if "traceback" in lower or "container exited" in lower:
|
|
139
|
+
return "Your service crashed while starting. Fix the issue in your code, then deploy again."
|
|
140
|
+
if "could not promote" in lower or "harbor" in lower:
|
|
141
|
+
return "We could not save your image. Please try deploying again."
|
|
142
|
+
if "critical" in lower or "cve" in lower or "vulnerab" in lower:
|
|
143
|
+
return "Your image did not pass the security scan. Fix the reported issues, then deploy again."
|
|
144
|
+
if "openapi" in lower:
|
|
145
|
+
return (
|
|
146
|
+
"Your service started, but we could not read its API definition. "
|
|
147
|
+
"Make sure the service exposes /openapi.json, then deploy again."
|
|
148
|
+
)
|
|
149
|
+
if "quota" in lower or ("registry" in lower and "space" in lower):
|
|
150
|
+
return "Your image storage is full. Delete unused services, then try again."
|
|
151
|
+
|
|
152
|
+
if text:
|
|
153
|
+
return text
|
|
154
|
+
if state == "failed_push":
|
|
155
|
+
return "We could not upload your image. Check your connection and try again."
|
|
156
|
+
if state == "failed_validation":
|
|
157
|
+
return "Your image did not pass checks. Please try deploying again."
|
|
158
|
+
if state == "failed_provision":
|
|
159
|
+
return "We could not start your service. Please try deploying again."
|
|
160
|
+
return "Deployment did not finish. Please try again."
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def support_detail(error: Optional[str]) -> Optional[str]:
|
|
164
|
+
"""Technical leftover to print under the human sentence, if useful."""
|
|
165
|
+
text = (error or "").strip()
|
|
166
|
+
if not text or is_progress_noise(text):
|
|
167
|
+
return None
|
|
168
|
+
human = humanize_failure(text)
|
|
169
|
+
if text == human or text in human:
|
|
170
|
+
return None
|
|
171
|
+
return text
|
|
@@ -10,7 +10,17 @@ from typing import Any, Dict, Optional
|
|
|
10
10
|
|
|
11
11
|
from socaity_cli.backend_client import SocaityBackendClient
|
|
12
12
|
from socaity_cli.credentials import require_credentials
|
|
13
|
-
from socaity_cli.
|
|
13
|
+
from socaity_cli.deploy_messages import (
|
|
14
|
+
humanize_failure,
|
|
15
|
+
progress_lines,
|
|
16
|
+
service_page_url,
|
|
17
|
+
support_detail,
|
|
18
|
+
)
|
|
19
|
+
from socaity_cli.errors import (
|
|
20
|
+
BackendApiError,
|
|
21
|
+
PrivateSlotLimitError,
|
|
22
|
+
RegistryQuotaExceededError,
|
|
23
|
+
)
|
|
14
24
|
from socaity_cli.prompts import input_yes_no
|
|
15
25
|
from socaity_cli.registry_push import (
|
|
16
26
|
CredentialsExpired,
|
|
@@ -48,15 +58,24 @@ def print_analysis_report(report: Dict[str, Any]) -> None:
|
|
|
48
58
|
line += f" | catalog: {catalog.get('name')}{vram}"
|
|
49
59
|
print(line)
|
|
50
60
|
for warning in model.get("warnings", []):
|
|
51
|
-
print(f"
|
|
61
|
+
print(f" Note: {warning}")
|
|
52
62
|
gpu = report.get("recommended_gpu")
|
|
53
|
-
|
|
54
|
-
|
|
63
|
+
vram = report.get("recommended_gpu_vram_gb")
|
|
64
|
+
if gpu and vram:
|
|
65
|
+
gpu_has_vram = str(int(vram)) in str(gpu)
|
|
66
|
+
print(f" Recommended GPU: {gpu}" + ("" if gpu_has_vram else f" ({int(vram)} GB)"))
|
|
67
|
+
elif gpu:
|
|
68
|
+
print(f" Recommended GPU: {gpu}")
|
|
69
|
+
elif vram:
|
|
70
|
+
print(f" Recommended GPU memory: {int(vram)} GB")
|
|
71
|
+
disk = report.get("recommended_disk_gb")
|
|
72
|
+
if disk:
|
|
73
|
+
print(f" Recommended disk: {disk} GB")
|
|
55
74
|
for service in report.get("similar_services", []):
|
|
56
75
|
name = service.get("display_name") or service.get("name")
|
|
57
|
-
print(f" Similar public service: {name}
|
|
76
|
+
print(f" Similar public service: {name}")
|
|
58
77
|
for warning in report.get("warnings", []):
|
|
59
|
-
print(f"
|
|
78
|
+
print(f" Note: {warning}")
|
|
60
79
|
|
|
61
80
|
|
|
62
81
|
def analyze_deployment(config: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|
@@ -117,6 +136,35 @@ def prompt_gated_hf_token(client: SocaityBackendClient, report: Dict[str, Any])
|
|
|
117
136
|
print("Token not stored; gated repos will fail to download on deploy.")
|
|
118
137
|
|
|
119
138
|
|
|
139
|
+
def apply_gpu_upgrade_if_needed(
|
|
140
|
+
config: Dict[str, Any],
|
|
141
|
+
report: Optional[Dict[str, Any]],
|
|
142
|
+
*,
|
|
143
|
+
assume_yes: bool = False,
|
|
144
|
+
) -> bool:
|
|
145
|
+
"""Ask to bump gpu_vram_gb when analyze offers a larger SKU. False aborts."""
|
|
146
|
+
upgrade = report.get("gpu_upgrade") if isinstance(report, dict) else None
|
|
147
|
+
if not upgrade:
|
|
148
|
+
return True
|
|
149
|
+
requested = upgrade.get("requested_vram_gb")
|
|
150
|
+
available = upgrade.get("available_vram_gb")
|
|
151
|
+
gpu = upgrade.get("gpu") or f"{available} GB"
|
|
152
|
+
question = (
|
|
153
|
+
f"No {requested} GB GPU is available. The next size is {available} GB "
|
|
154
|
+
f"({gpu}). Use the larger GPU?"
|
|
155
|
+
)
|
|
156
|
+
if assume_yes:
|
|
157
|
+
print(f" {question} yes")
|
|
158
|
+
accepted = True
|
|
159
|
+
else:
|
|
160
|
+
accepted = input_yes_no(question)
|
|
161
|
+
if not accepted:
|
|
162
|
+
print("Aborted. Deploy will keep your current GPU size and will not pick a smaller one.")
|
|
163
|
+
return False
|
|
164
|
+
config["gpu_vram_gb"] = int(available)
|
|
165
|
+
return True
|
|
166
|
+
|
|
167
|
+
|
|
120
168
|
def create_deployment_draft(config: Dict[str, Any], report: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
|
|
121
169
|
"""Create a deployment draft after optional confirmation."""
|
|
122
170
|
require_credentials()
|
|
@@ -128,6 +176,9 @@ def create_deployment_draft(config: Dict[str, Any], report: Optional[Dict[str, A
|
|
|
128
176
|
return None
|
|
129
177
|
print_analysis_report(report)
|
|
130
178
|
|
|
179
|
+
if not apply_gpu_upgrade_if_needed(config, report):
|
|
180
|
+
return None
|
|
181
|
+
|
|
131
182
|
resolutions = prompt_model_resolutions(report)
|
|
132
183
|
prompt_gated_hf_token(client, report)
|
|
133
184
|
|
|
@@ -137,17 +188,28 @@ def create_deployment_draft(config: Dict[str, Any], report: Optional[Dict[str, A
|
|
|
137
188
|
|
|
138
189
|
try:
|
|
139
190
|
draft = client.create_deployment_draft(config, model_resolutions=resolutions)
|
|
140
|
-
except
|
|
141
|
-
|
|
142
|
-
|
|
191
|
+
except BackendApiError as exc:
|
|
192
|
+
if getattr(exc, "code", None) == "GPU_UPGRADE_AVAILABLE":
|
|
193
|
+
print(f"\n{exc}")
|
|
194
|
+
print("Confirm the larger GPU in analyze, or change gpu_vram_gb, then retry.")
|
|
195
|
+
return None
|
|
196
|
+
if getattr(exc, "code", None) == "AUTO_RECHARGE_REQUIRED":
|
|
197
|
+
print(f"\n{exc}")
|
|
198
|
+
print(
|
|
199
|
+
"Enable auto-recharge and save a card on Billing. "
|
|
200
|
+
"If a previous draft failed, delete that unused private service in My Services, then retry."
|
|
201
|
+
)
|
|
202
|
+
return None
|
|
203
|
+
if isinstance(exc, PrivateSlotLimitError):
|
|
204
|
+
print(f"\n{exc}")
|
|
205
|
+
print("Tip: publish a private service from the service page, or upgrade your plan.")
|
|
206
|
+
raise
|
|
143
207
|
raise
|
|
144
208
|
if draft is None:
|
|
145
209
|
print("Draft creation failed.")
|
|
146
210
|
return None
|
|
147
|
-
print(
|
|
148
|
-
|
|
149
|
-
"Next: the image will be pushed to the Socaity registry and provisioned automatically."
|
|
150
|
-
)
|
|
211
|
+
print("Draft created. Next we will upload your image and start the service.")
|
|
212
|
+
print(f" Resume later with: socaity deploy --resume {draft.get('deployment_id')}")
|
|
151
213
|
return draft
|
|
152
214
|
|
|
153
215
|
|
|
@@ -160,9 +222,17 @@ def push_image_to_registry(
|
|
|
160
222
|
"""Push a local image to the deployment staging repo. Returns the manifest digest."""
|
|
161
223
|
client = client or SocaityBackendClient()
|
|
162
224
|
estimated_size = image_size_bytes(local_tag)
|
|
163
|
-
|
|
225
|
+
try:
|
|
226
|
+
creds = client.get_push_credentials(deployment_id, estimated_size_bytes=estimated_size)
|
|
227
|
+
except RegistryQuotaExceededError as exc:
|
|
228
|
+
print(f"\n{exc}")
|
|
229
|
+
print(
|
|
230
|
+
"This attempt was dropped so it does not occupy a private slot. "
|
|
231
|
+
"Delete unused services to free storage space, then deploy again."
|
|
232
|
+
)
|
|
233
|
+
return None
|
|
164
234
|
if creds is None:
|
|
165
|
-
print("Could not
|
|
235
|
+
print("Could not get permission to upload your image. Please try again.")
|
|
166
236
|
return None
|
|
167
237
|
|
|
168
238
|
refreshed = False
|
|
@@ -180,14 +250,22 @@ def push_image_to_registry(
|
|
|
180
250
|
return digest
|
|
181
251
|
except CredentialsExpired:
|
|
182
252
|
if not refreshed:
|
|
183
|
-
print("
|
|
184
|
-
|
|
253
|
+
print("Upload permission expired. Getting a new one...")
|
|
254
|
+
try:
|
|
255
|
+
creds = client.get_push_credentials(deployment_id, estimated_size_bytes=estimated_size)
|
|
256
|
+
except RegistryQuotaExceededError as exc:
|
|
257
|
+
print(f"\n{exc}")
|
|
258
|
+
print(
|
|
259
|
+
"This attempt was dropped so it does not occupy a private slot. "
|
|
260
|
+
"Delete unused services to free storage space, then deploy again."
|
|
261
|
+
)
|
|
262
|
+
return None
|
|
185
263
|
if creds is None:
|
|
186
|
-
print("Could not refresh
|
|
264
|
+
print("Could not refresh upload permission. Please try again.")
|
|
187
265
|
return None
|
|
188
266
|
refreshed = True
|
|
189
267
|
continue
|
|
190
|
-
print("
|
|
268
|
+
print("Upload permission expired again. Please retry the deploy.")
|
|
191
269
|
return None
|
|
192
270
|
except DockerNotAvailable as exc:
|
|
193
271
|
print(str(exc))
|
|
@@ -199,10 +277,9 @@ def push_image_to_registry(
|
|
|
199
277
|
):
|
|
200
278
|
wait = _TRANSIENT_PUSH_BACKOFF_S[transient_attempts]
|
|
201
279
|
transient_attempts += 1
|
|
202
|
-
print(
|
|
280
|
+
print("Upload was interrupted. Waiting for the registry to finish the last layer...")
|
|
203
281
|
print(
|
|
204
|
-
f"
|
|
205
|
-
f"then retrying the same repository "
|
|
282
|
+
f"Retrying in {wait}s "
|
|
206
283
|
f"({transient_attempts}/{len(_TRANSIENT_PUSH_BACKOFF_S)})..."
|
|
207
284
|
)
|
|
208
285
|
time.sleep(wait)
|
|
@@ -216,30 +293,54 @@ def poll_deployment_until_terminal(
|
|
|
216
293
|
*,
|
|
217
294
|
timeout_s: float = _DEFAULT_TIMEOUT_S,
|
|
218
295
|
client: Optional[SocaityBackendClient] = None,
|
|
296
|
+
printer=print,
|
|
297
|
+
sleep=time.sleep,
|
|
298
|
+
now=time.monotonic,
|
|
219
299
|
) -> Optional[Dict[str, Any]]:
|
|
220
300
|
"""Poll pipeline status until live or a terminal failure."""
|
|
221
301
|
client = client or SocaityBackendClient()
|
|
222
|
-
deadline =
|
|
302
|
+
deadline = now() + timeout_s
|
|
223
303
|
last_status: Optional[Dict[str, Any]] = None
|
|
304
|
+
last_step: Optional[str] = None
|
|
305
|
+
last_detail: Optional[str] = None
|
|
306
|
+
last_print_at = now()
|
|
307
|
+
saw_progress = False
|
|
224
308
|
|
|
225
|
-
while
|
|
309
|
+
while now() < deadline:
|
|
226
310
|
status = client.get_deployment_pipeline_status(deployment_id)
|
|
227
311
|
if status is None:
|
|
228
|
-
|
|
229
|
-
|
|
312
|
+
printer("Could not check progress. Retrying...")
|
|
313
|
+
sleep(_POLL_INTERVAL_S)
|
|
230
314
|
continue
|
|
231
315
|
|
|
232
316
|
last_status = status
|
|
233
|
-
phase = status.get("phase")
|
|
234
317
|
state = str(status.get("status", "")).lower()
|
|
235
|
-
label = f"{state}" + (f" ({phase})" if phase else "")
|
|
236
|
-
print(f" deployment status: {label}")
|
|
237
|
-
|
|
238
318
|
if state in _TERMINAL_STATUSES:
|
|
239
319
|
return status
|
|
240
|
-
time.sleep(_POLL_INTERVAL_S)
|
|
241
320
|
|
|
242
|
-
|
|
321
|
+
step, detail = progress_lines(status)
|
|
322
|
+
if step != last_step:
|
|
323
|
+
printer(f" {step}")
|
|
324
|
+
last_step = step
|
|
325
|
+
last_detail = None
|
|
326
|
+
last_print_at = now()
|
|
327
|
+
saw_progress = True
|
|
328
|
+
if detail and detail != last_detail and detail != step:
|
|
329
|
+
printer(f" {detail}")
|
|
330
|
+
last_detail = detail
|
|
331
|
+
last_print_at = now()
|
|
332
|
+
saw_progress = True
|
|
333
|
+
elif saw_progress and now() - last_print_at >= 60:
|
|
334
|
+
printer(" Still working...")
|
|
335
|
+
last_print_at = now()
|
|
336
|
+
|
|
337
|
+
sleep(_POLL_INTERVAL_S)
|
|
338
|
+
|
|
339
|
+
minutes = max(1, int(timeout_s) // 60)
|
|
340
|
+
printer(
|
|
341
|
+
f"Timed out after {minutes} minute{'s' if minutes != 1 else ''}. "
|
|
342
|
+
"Check the service page or try deploying again."
|
|
343
|
+
)
|
|
243
344
|
return last_status
|
|
244
345
|
|
|
245
346
|
|
|
@@ -255,6 +356,9 @@ def _create_draft_with_prompts(
|
|
|
255
356
|
return None
|
|
256
357
|
print_analysis_report(report)
|
|
257
358
|
|
|
359
|
+
if not apply_gpu_upgrade_if_needed(config, report, assume_yes=assume_yes):
|
|
360
|
+
return None
|
|
361
|
+
|
|
258
362
|
if assume_yes:
|
|
259
363
|
resolutions: Dict[str, Optional[str]] = {}
|
|
260
364
|
for model in report.get("models", []):
|
|
@@ -274,20 +378,75 @@ def _create_draft_with_prompts(
|
|
|
274
378
|
|
|
275
379
|
try:
|
|
276
380
|
draft = client.create_deployment_draft(config, model_resolutions=resolutions)
|
|
277
|
-
except
|
|
278
|
-
|
|
279
|
-
|
|
381
|
+
except BackendApiError as exc:
|
|
382
|
+
if getattr(exc, "code", None) == "GPU_UPGRADE_AVAILABLE":
|
|
383
|
+
print(f"\n{exc}")
|
|
384
|
+
print("Confirm the larger GPU in analyze, or change gpu_vram_gb, then retry.")
|
|
385
|
+
return None
|
|
386
|
+
if getattr(exc, "code", None) == "AUTO_RECHARGE_REQUIRED":
|
|
387
|
+
print(f"\n{exc}")
|
|
388
|
+
print(
|
|
389
|
+
"Enable auto-recharge and save a card on Billing. "
|
|
390
|
+
"If a previous draft failed, delete that unused private service in My Services, then retry."
|
|
391
|
+
)
|
|
392
|
+
return None
|
|
393
|
+
if isinstance(exc, PrivateSlotLimitError):
|
|
394
|
+
print(f"\n{exc}")
|
|
395
|
+
print("Tip: publish a private service from the service page, or upgrade your plan.")
|
|
396
|
+
raise
|
|
280
397
|
raise
|
|
281
398
|
if draft is None:
|
|
282
399
|
print("Draft creation failed.")
|
|
283
400
|
return None
|
|
284
|
-
print(
|
|
285
|
-
|
|
286
|
-
f"deployment_id={draft.get('deployment_id')}"
|
|
287
|
-
)
|
|
401
|
+
print("Draft created. Next we will upload your image and start the service.")
|
|
402
|
+
print(f" Resume later with: socaity deploy --resume {draft.get('deployment_id')}")
|
|
288
403
|
return draft
|
|
289
404
|
|
|
290
405
|
|
|
406
|
+
def _print_failure_and_maybe_delete(
|
|
407
|
+
client: SocaityBackendClient,
|
|
408
|
+
deployment_id: str,
|
|
409
|
+
status: Optional[Dict[str, Any]],
|
|
410
|
+
*,
|
|
411
|
+
assume_yes: bool,
|
|
412
|
+
) -> None:
|
|
413
|
+
"""Show why the deploy failed, then delete when asked (or with --yes)."""
|
|
414
|
+
snapshot = dict(status) if isinstance(status, dict) else {}
|
|
415
|
+
if not snapshot.get("error") and not snapshot.get("logs"):
|
|
416
|
+
fresh = client.get_deployment_pipeline_status(deployment_id)
|
|
417
|
+
if isinstance(fresh, dict):
|
|
418
|
+
snapshot = fresh
|
|
419
|
+
state = str(snapshot.get("status") or "").lower()
|
|
420
|
+
print(f"\n{humanize_failure(snapshot.get('error'), state)}")
|
|
421
|
+
logs = snapshot.get("logs")
|
|
422
|
+
if logs:
|
|
423
|
+
print("\nLogs:")
|
|
424
|
+
for line in str(logs).splitlines():
|
|
425
|
+
print(f" {line}")
|
|
426
|
+
else:
|
|
427
|
+
detail = support_detail(snapshot.get("error"))
|
|
428
|
+
if detail:
|
|
429
|
+
print(f" Details: {detail}")
|
|
430
|
+
if assume_yes:
|
|
431
|
+
deleted = client.cancel_deployment_pipeline(deployment_id)
|
|
432
|
+
if deleted is None:
|
|
433
|
+
print("Could not delete the failed service. Delete it from My Services.")
|
|
434
|
+
print(f" {service_page_url(snapshot.get('service_id'))}")
|
|
435
|
+
return
|
|
436
|
+
print("Deleted the failed service.")
|
|
437
|
+
return
|
|
438
|
+
print("This failed service is on My Services and still uses a private slot.")
|
|
439
|
+
if not input_yes_no("Delete this failed service now?", default=False):
|
|
440
|
+
print(f" {service_page_url(snapshot.get('service_id'))}")
|
|
441
|
+
return
|
|
442
|
+
deleted = client.cancel_deployment_pipeline(deployment_id)
|
|
443
|
+
if deleted is None:
|
|
444
|
+
print("Could not delete the failed service. Delete it from My Services.")
|
|
445
|
+
print(f" {service_page_url(snapshot.get('service_id'))}")
|
|
446
|
+
return
|
|
447
|
+
print("Deleted the failed service.")
|
|
448
|
+
|
|
449
|
+
|
|
291
450
|
def run_full_deploy(
|
|
292
451
|
config: Dict[str, Any],
|
|
293
452
|
*,
|
|
@@ -303,19 +462,20 @@ def run_full_deploy(
|
|
|
303
462
|
if deployment_id:
|
|
304
463
|
status = client.get_deployment_pipeline_status(deployment_id)
|
|
305
464
|
if status is None:
|
|
306
|
-
print(
|
|
465
|
+
print("That deployment was not found, or you do not have access to it.")
|
|
307
466
|
return None
|
|
308
467
|
state = str(status.get("status", "")).lower()
|
|
309
468
|
if state == "live":
|
|
310
|
-
print(
|
|
469
|
+
print("This service is already live.")
|
|
470
|
+
if status.get("service_id"):
|
|
471
|
+
print(f" {service_page_url(status.get('service_id'))}")
|
|
311
472
|
return status
|
|
312
473
|
if state in _TERMINAL_STATUSES - {"live"}:
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
"Create a new draft or fix the issue before resuming."
|
|
474
|
+
_print_failure_and_maybe_delete(
|
|
475
|
+
client, deployment_id, status, assume_yes=assume_yes,
|
|
316
476
|
)
|
|
317
477
|
return status
|
|
318
|
-
print(
|
|
478
|
+
print("Continuing your deployment...")
|
|
319
479
|
else:
|
|
320
480
|
draft = _create_draft_with_prompts(config, client=client, assume_yes=assume_yes)
|
|
321
481
|
if draft is None:
|
|
@@ -328,37 +488,45 @@ def run_full_deploy(
|
|
|
328
488
|
current = client.get_deployment_pipeline_status(deployment_id) or {}
|
|
329
489
|
state = str(current.get("status", "")).lower()
|
|
330
490
|
if state in {"draft", "awaiting_image", "pushing", "failed_push"}:
|
|
331
|
-
print(
|
|
491
|
+
print("Uploading your image...")
|
|
332
492
|
digest = push_image_to_registry(deployment_id, local_tag, client=client)
|
|
333
493
|
if digest is None:
|
|
494
|
+
_print_failure_and_maybe_delete(
|
|
495
|
+
client, deployment_id, current, assume_yes=assume_yes,
|
|
496
|
+
)
|
|
334
497
|
return None
|
|
335
498
|
size_bytes = image_size_bytes(local_tag)
|
|
336
499
|
pushed = client.confirm_image_pushed(deployment_id, digest=digest, size_bytes=size_bytes)
|
|
337
500
|
if pushed is None:
|
|
338
|
-
print("
|
|
501
|
+
print("The server did not accept the uploaded image. Please try again.")
|
|
502
|
+
_print_failure_and_maybe_delete(
|
|
503
|
+
client, deployment_id, current, assume_yes=assume_yes,
|
|
504
|
+
)
|
|
339
505
|
return None
|
|
340
|
-
print(
|
|
506
|
+
print("Image uploaded. Checking it and starting your service...")
|
|
341
507
|
elif state in {"validating", "promoting", "provisioning"}:
|
|
342
|
-
print(
|
|
508
|
+
print("Image already uploaded. Checking it and starting your service...")
|
|
343
509
|
elif state == "live":
|
|
344
|
-
print(
|
|
510
|
+
print("This service is already live.")
|
|
511
|
+
if current.get("service_id"):
|
|
512
|
+
print(f" {service_page_url(current.get('service_id'))}")
|
|
345
513
|
return current
|
|
346
514
|
|
|
347
|
-
print("
|
|
515
|
+
print("This can take several minutes.")
|
|
348
516
|
final = poll_deployment_until_terminal(deployment_id, client=client)
|
|
349
517
|
if final is None:
|
|
350
518
|
return None
|
|
351
519
|
|
|
352
520
|
final_state = str(final.get("status", "")).lower()
|
|
353
521
|
if final_state == "live":
|
|
354
|
-
print(
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
522
|
+
print("\nYour service is live.")
|
|
523
|
+
print(f" {service_page_url(final.get('service_id'))}")
|
|
524
|
+
return final
|
|
525
|
+
if final_state in _TERMINAL_STATUSES:
|
|
526
|
+
_print_failure_and_maybe_delete(
|
|
527
|
+
client, deployment_id, final, assume_yes=assume_yes,
|
|
528
|
+
)
|
|
529
|
+
return final
|
|
362
530
|
return final
|
|
363
531
|
|
|
364
532
|
|
|
@@ -33,3 +33,10 @@ class PrivateSlotLimitError(BackendApiError):
|
|
|
33
33
|
"Publish an existing private service or upgrade your plan to free a slot."
|
|
34
34
|
)
|
|
35
35
|
super().__init__(403, message, code="PRIVATE_SLOT_LIMIT", detail=detail)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class RegistryQuotaExceededError(BackendApiError):
|
|
39
|
+
"""Harbor image quota would be exceeded by this push."""
|
|
40
|
+
|
|
41
|
+
def __init__(self, message: str, *, detail: Any = None):
|
|
42
|
+
super().__init__(413, message, code="REGISTRY_QUOTA_EXCEEDED", detail=detail)
|
|
@@ -6,7 +6,7 @@ from typing import Any, Dict, Optional
|
|
|
6
6
|
import httpx
|
|
7
7
|
|
|
8
8
|
from socaity_cli.credentials import DEFAULT_BACKEND_URL, get_api_key
|
|
9
|
-
from socaity_cli.errors import BackendApiError, PrivateSlotLimitError
|
|
9
|
+
from socaity_cli.errors import BackendApiError, PrivateSlotLimitError, RegistryQuotaExceededError
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class BackendHttp:
|
|
@@ -30,6 +30,14 @@ class BackendHttp:
|
|
|
30
30
|
detail = body.get("detail") if isinstance(body, dict) else None
|
|
31
31
|
if isinstance(detail, dict) and detail.get("code") == "PRIVATE_SLOT_LIMIT":
|
|
32
32
|
raise PrivateSlotLimitError(detail)
|
|
33
|
+
if status_code == 413:
|
|
34
|
+
if isinstance(detail, str) and detail.strip():
|
|
35
|
+
message = detail.strip()
|
|
36
|
+
elif isinstance(detail, dict):
|
|
37
|
+
message = str(detail.get("message") or detail)
|
|
38
|
+
else:
|
|
39
|
+
message = (body_text or "Registry quota exceeded.")[:300]
|
|
40
|
+
raise RegistryQuotaExceededError(message, detail=detail)
|
|
33
41
|
if isinstance(detail, dict) and detail.get("code"):
|
|
34
42
|
raise BackendApiError(
|
|
35
43
|
status_code,
|
|
@@ -64,7 +72,7 @@ class BackendHttp:
|
|
|
64
72
|
self._raise_for_structured_error(response.status_code, response.text)
|
|
65
73
|
print(f"Request to {path} failed with status {response.status_code}: {response.text[:300]}")
|
|
66
74
|
return None
|
|
67
|
-
except (PrivateSlotLimitError, BackendApiError):
|
|
75
|
+
except (PrivateSlotLimitError, RegistryQuotaExceededError, BackendApiError):
|
|
68
76
|
raise
|
|
69
77
|
except Exception as e:
|
|
70
78
|
print(f"Request to {path} failed: {e}")
|
|
@@ -1,15 +1,19 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Push a local image to the Socaity registry with ephemeral credentials.
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
Prefers ``skopeo`` (long command timeout, serial layers, built-in retries).
|
|
4
|
+
Docker 29's ``docker push`` uses a short ResponseHeaderTimeout and aborts
|
|
5
|
+
Harbor commits that later succeed. Credentials never land in ~/.docker.
|
|
5
6
|
"""
|
|
6
7
|
from __future__ import annotations
|
|
7
8
|
|
|
9
|
+
import base64
|
|
8
10
|
import json
|
|
9
11
|
import os
|
|
10
12
|
import re
|
|
13
|
+
import shutil
|
|
11
14
|
import subprocess
|
|
12
15
|
import tempfile
|
|
16
|
+
from pathlib import Path
|
|
13
17
|
from typing import Optional, Tuple
|
|
14
18
|
|
|
15
19
|
_DIGEST_RE = re.compile(r"digest:\s*(sha256:[0-9a-f]{64})")
|
|
@@ -21,6 +25,8 @@ _TRANSIENT_PUSH_MARKERS = (
|
|
|
21
25
|
"tls handshake",
|
|
22
26
|
"unexpected eof",
|
|
23
27
|
)
|
|
28
|
+
_SKOPEO_TIMEOUT = os.environ.get("SOCAITY_PUSH_TIMEOUT", "2h")
|
|
29
|
+
_SKOPEO_RETRIES = int(os.environ.get("SOCAITY_PUSH_RETRIES", "20"))
|
|
24
30
|
|
|
25
31
|
|
|
26
32
|
class DockerNotAvailable(RuntimeError):
|
|
@@ -41,6 +47,44 @@ def is_transient_push_error(message: str) -> bool:
|
|
|
41
47
|
return any(marker in lower for marker in _TRANSIENT_PUSH_MARKERS)
|
|
42
48
|
|
|
43
49
|
|
|
50
|
+
def daemon_ref(local_tag: str) -> str:
|
|
51
|
+
"""``docker-daemon:`` reference; add ``:latest`` when the tag is omitted."""
|
|
52
|
+
name = local_tag.strip()
|
|
53
|
+
if not name:
|
|
54
|
+
raise PushFailed("Local image tag is empty.")
|
|
55
|
+
if ":" not in name.rsplit("/", 1)[-1]:
|
|
56
|
+
return f"{name}:latest"
|
|
57
|
+
return name
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _skopeo_copy_args(
|
|
61
|
+
*,
|
|
62
|
+
src: str,
|
|
63
|
+
dest: str,
|
|
64
|
+
authfile: str,
|
|
65
|
+
digestfile: str,
|
|
66
|
+
) -> list:
|
|
67
|
+
return [
|
|
68
|
+
"skopeo",
|
|
69
|
+
f"--command-timeout={_SKOPEO_TIMEOUT}",
|
|
70
|
+
"copy",
|
|
71
|
+
"--dest-tls-verify=true",
|
|
72
|
+
f"--dest-authfile={authfile}",
|
|
73
|
+
"--image-parallel-copies=1",
|
|
74
|
+
f"--retry-times={_SKOPEO_RETRIES}",
|
|
75
|
+
"--retry-delay=15s",
|
|
76
|
+
"--dest-precompute-digests",
|
|
77
|
+
f"--digestfile={digestfile}",
|
|
78
|
+
src,
|
|
79
|
+
dest,
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _write_authfile(path: str, registry: str, username: str, password: str) -> None:
|
|
84
|
+
token = base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
85
|
+
Path(path).write_text(json.dumps({"auths": {registry: {"auth": token}}}))
|
|
86
|
+
|
|
87
|
+
|
|
44
88
|
def _push_command(ref: str) -> list:
|
|
45
89
|
"""``docker push`` args. Docker 25+ dropped ``--max-concurrent-uploads``."""
|
|
46
90
|
help_out = _docker(["push", "--help"])
|
|
@@ -78,7 +122,7 @@ def push_image(
|
|
|
78
122
|
password: str,
|
|
79
123
|
local_tag: str,
|
|
80
124
|
) -> Tuple[str, int]:
|
|
81
|
-
"""
|
|
125
|
+
"""Copy ``local_tag`` to ``registry/repository:tag``.
|
|
82
126
|
|
|
83
127
|
Returns:
|
|
84
128
|
(manifest digest reported by the registry, local image size in bytes).
|
|
@@ -86,20 +130,94 @@ def push_image(
|
|
|
86
130
|
Raises:
|
|
87
131
|
DockerNotAvailable, CredentialsExpired, PushFailed.
|
|
88
132
|
"""
|
|
89
|
-
ref = f"{registry}/{repository}:{tag}"
|
|
90
133
|
size = image_size_bytes(local_tag)
|
|
91
134
|
if size == 0:
|
|
92
135
|
raise PushFailed(f"Local image '{local_tag}' not found. Build it first (apipod build).")
|
|
93
136
|
|
|
94
|
-
|
|
137
|
+
dest = f"{registry}/{repository}:{tag}"
|
|
138
|
+
if shutil.which("skopeo"):
|
|
139
|
+
print(
|
|
140
|
+
f" Pushing with skopeo ({_SKOPEO_TIMEOUT} timeout, one layer at a time, "
|
|
141
|
+
f"{_SKOPEO_RETRIES} retries)..."
|
|
142
|
+
)
|
|
143
|
+
return _push_with_skopeo(
|
|
144
|
+
registry=registry,
|
|
145
|
+
dest=dest,
|
|
146
|
+
username=username,
|
|
147
|
+
password=password,
|
|
148
|
+
local_tag=local_tag,
|
|
149
|
+
size=size,
|
|
150
|
+
)
|
|
151
|
+
print(" skopeo not found; falling back to docker push (short Harbor header timeout).")
|
|
152
|
+
return _push_with_docker(
|
|
153
|
+
registry=registry,
|
|
154
|
+
dest=dest,
|
|
155
|
+
username=username,
|
|
156
|
+
password=password,
|
|
157
|
+
local_tag=local_tag,
|
|
158
|
+
size=size,
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _push_with_skopeo(
|
|
163
|
+
*,
|
|
164
|
+
registry: str,
|
|
165
|
+
dest: str,
|
|
166
|
+
username: str,
|
|
167
|
+
password: str,
|
|
168
|
+
local_tag: str,
|
|
169
|
+
size: int,
|
|
170
|
+
) -> Tuple[str, int]:
|
|
171
|
+
src = f"docker-daemon:{daemon_ref(local_tag)}"
|
|
172
|
+
dest_ref = f"docker://{dest}"
|
|
173
|
+
with tempfile.TemporaryDirectory(prefix="socaity-skopeo-") as tmp:
|
|
174
|
+
authfile = os.path.join(tmp, "auth.json")
|
|
175
|
+
digestfile = os.path.join(tmp, "digest")
|
|
176
|
+
_write_authfile(authfile, registry, username, password)
|
|
177
|
+
cmd = _skopeo_copy_args(src=src, dest=dest_ref, authfile=authfile, digestfile=digestfile)
|
|
178
|
+
try:
|
|
179
|
+
push = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
|
180
|
+
except FileNotFoundError as exc:
|
|
181
|
+
raise PushFailed("skopeo disappeared from PATH during the push.") from exc
|
|
182
|
+
last_lines = []
|
|
183
|
+
for line in push.stdout:
|
|
184
|
+
line = line.rstrip()
|
|
185
|
+
last_lines = (last_lines + [line])[-8:]
|
|
186
|
+
if line:
|
|
187
|
+
print(f" {line}")
|
|
188
|
+
push.wait()
|
|
189
|
+
tail = " | ".join(last_lines)
|
|
190
|
+
if push.returncode != 0:
|
|
191
|
+
if "unauthorized" in tail.lower() or "401" in tail.lower() or "authentication required" in tail.lower():
|
|
192
|
+
raise CredentialsExpired("Push rejected mid-transfer; the credentials expired.")
|
|
193
|
+
raise PushFailed(f"skopeo copy failed: {tail[:400]}")
|
|
194
|
+
digest = Path(digestfile).read_text().strip() if os.path.exists(digestfile) else ""
|
|
195
|
+
if not digest.startswith("sha256:"):
|
|
196
|
+
raise PushFailed("skopeo copy succeeded but no digest file was written.")
|
|
197
|
+
return digest, size
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _push_with_docker(
|
|
201
|
+
*,
|
|
202
|
+
registry: str,
|
|
203
|
+
dest: str,
|
|
204
|
+
username: str,
|
|
205
|
+
password: str,
|
|
206
|
+
local_tag: str,
|
|
207
|
+
size: int,
|
|
208
|
+
) -> Tuple[str, int]:
|
|
209
|
+
result = _docker(["tag", local_tag, dest])
|
|
95
210
|
if result.returncode != 0:
|
|
96
211
|
raise PushFailed(f"docker tag failed: {result.stderr.decode()[:300]}")
|
|
97
212
|
|
|
98
213
|
with tempfile.TemporaryDirectory(prefix="socaity-push-") as docker_config:
|
|
99
214
|
env = {**os.environ, "DOCKER_CONFIG": docker_config}
|
|
100
215
|
try:
|
|
101
|
-
login = _docker(
|
|
102
|
-
|
|
216
|
+
login = _docker(
|
|
217
|
+
["login", registry, "-u", username, "--password-stdin"],
|
|
218
|
+
env=env,
|
|
219
|
+
input_bytes=password.encode(),
|
|
220
|
+
)
|
|
103
221
|
if login.returncode != 0:
|
|
104
222
|
stderr = login.stderr.decode()
|
|
105
223
|
if "unauthorized" in stderr.lower() or "401" in stderr:
|
|
@@ -107,8 +225,11 @@ def push_image(
|
|
|
107
225
|
raise PushFailed(f"docker login failed: {stderr[:300]}")
|
|
108
226
|
|
|
109
227
|
push = subprocess.Popen(
|
|
110
|
-
_push_command(
|
|
111
|
-
stdout=subprocess.PIPE,
|
|
228
|
+
_push_command(dest),
|
|
229
|
+
stdout=subprocess.PIPE,
|
|
230
|
+
stderr=subprocess.STDOUT,
|
|
231
|
+
env=env,
|
|
232
|
+
text=True,
|
|
112
233
|
)
|
|
113
234
|
digest, last_lines = None, []
|
|
114
235
|
for line in push.stdout:
|
|
@@ -126,12 +247,12 @@ def push_image(
|
|
|
126
247
|
raise CredentialsExpired("Push rejected mid-transfer; the credentials expired.")
|
|
127
248
|
raise PushFailed(f"docker push failed: {tail[:400]}")
|
|
128
249
|
if not digest:
|
|
129
|
-
digest = _digest_from_local_metadata(
|
|
250
|
+
digest = _digest_from_local_metadata(dest, env)
|
|
130
251
|
if not digest:
|
|
131
252
|
raise PushFailed("Push succeeded but no digest was reported by the registry.")
|
|
132
253
|
return digest, size
|
|
133
254
|
finally:
|
|
134
|
-
_docker(["rmi",
|
|
255
|
+
_docker(["rmi", dest], env=env)
|
|
135
256
|
|
|
136
257
|
|
|
137
258
|
def _digest_from_local_metadata(ref: str, env: dict) -> Optional[str]:
|
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
from socaity_cli.registry_push import
|
|
1
|
+
from socaity_cli.registry_push import (
|
|
2
|
+
_push_command,
|
|
3
|
+
_skopeo_copy_args,
|
|
4
|
+
daemon_ref,
|
|
5
|
+
is_transient_push_error,
|
|
6
|
+
)
|
|
2
7
|
|
|
3
8
|
|
|
4
9
|
def test_is_transient_push_error_matches_harbor_timeouts():
|
|
@@ -30,3 +35,25 @@ def test_push_command_uses_max_concurrent_when_supported(monkeypatch):
|
|
|
30
35
|
assert _push_command("harbor.example/repo:tag") == [
|
|
31
36
|
"docker", "push", "--max-concurrent-uploads=1", "harbor.example/repo:tag",
|
|
32
37
|
]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_daemon_ref_adds_latest():
|
|
41
|
+
assert daemon_ref("apipod-qwen38") == "apipod-qwen38:latest"
|
|
42
|
+
assert daemon_ref("apipod-qwen38:latest") == "apipod-qwen38:latest"
|
|
43
|
+
assert daemon_ref("ghcr.io/org/app:v1") == "ghcr.io/org/app:v1"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_skopeo_copy_args_are_serial_with_long_timeout():
|
|
47
|
+
args = _skopeo_copy_args(
|
|
48
|
+
src="docker-daemon:app:latest",
|
|
49
|
+
dest="docker://harbor.example/repo:tag",
|
|
50
|
+
authfile="/tmp/auth.json",
|
|
51
|
+
digestfile="/tmp/digest",
|
|
52
|
+
)
|
|
53
|
+
assert args[0] == "skopeo"
|
|
54
|
+
assert args[1] == "--command-timeout=2h"
|
|
55
|
+
assert "--image-parallel-copies=1" in args
|
|
56
|
+
assert "--retry-times=20" in args
|
|
57
|
+
assert "--dest-precompute-digests" in args
|
|
58
|
+
assert "docker-daemon:app:latest" in args
|
|
59
|
+
assert "docker://harbor.example/repo:tag" in args
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|