socaity-cli 0.1.5__tar.gz → 0.1.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/PKG-INFO +1 -1
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/pyproject.toml +1 -1
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/auth.py +5 -1
- socaity_cli-0.1.7/socaity_cli/deploy_messages.py +171 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/deployment.py +238 -59
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/errors.py +7 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/http.py +10 -2
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/registry_push.py +133 -12
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli.egg-info/PKG-INFO +1 -1
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli.egg-info/SOURCES.txt +1 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/test/test_registry_push.py +28 -1
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/README.md +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/setup.cfg +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/__init__.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/__main__.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/apipod_commands.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/backend_client.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/cli.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/clients/__init__.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/clients/analytics.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/clients/catalog.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/clients/deployment.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/clients/files.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/clients/jobs.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/clients/projects.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/clients/sdk_install.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/credentials.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/prompts.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/requires.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli/sdk_commands.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli.egg-info/dependency_links.txt +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli.egg-info/entry_points.txt +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli.egg-info/requires.txt +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/socaity_cli.egg-info/top_level.txt +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/test/test_cli.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/test/test_credentials.py +0 -0
- {socaity_cli-0.1.5 → socaity_cli-0.1.7}/test/test_private_slot_error.py +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "socaity-cli"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.7"
|
|
8
8
|
description = "Socaity platform CLI: login, catalog browsing, SDK installs and APIPod deployment commands."
|
|
9
9
|
requires-python = ">=3.8"
|
|
10
10
|
authors = [
|
|
@@ -7,6 +7,7 @@ import shutil
|
|
|
7
7
|
import subprocess
|
|
8
8
|
import time
|
|
9
9
|
import webbrowser
|
|
10
|
+
from importlib.metadata import PackageNotFoundError, version as pkg_version
|
|
10
11
|
from typing import Any, Dict, Optional
|
|
11
12
|
from urllib.parse import urljoin
|
|
12
13
|
|
|
@@ -20,7 +21,10 @@ from socaity_cli.credentials import (
|
|
|
20
21
|
save_credentials,
|
|
21
22
|
)
|
|
22
23
|
|
|
23
|
-
|
|
24
|
+
try:
|
|
25
|
+
USER_AGENT = f"socaity-cli/{pkg_version('socaity-cli')}"
|
|
26
|
+
except PackageNotFoundError:
|
|
27
|
+
USER_AGENT = "socaity-cli"
|
|
24
28
|
|
|
25
29
|
|
|
26
30
|
def _backend_base_url() -> str:
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""User-facing copy for deploy progress and failures.
|
|
2
|
+
|
|
3
|
+
Backend status/phase/error stay machine-readable. This module turns them
|
|
4
|
+
into short sentences a non-technical user can follow in the CLI.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
from typing import Any, Dict, Optional, Tuple
|
|
11
|
+
|
|
12
|
+
from socaity_cli.credentials import DEFAULT_FRONTEND_URL
|
|
13
|
+
|
|
14
|
+
_WORKER_COUNTS_RE = re.compile(
|
|
15
|
+
r"RunPod workers\s+ready=(?P<ready>\d+)\s+initializing=(?P<initializing>\d+)\s+"
|
|
16
|
+
r"unhealthy=(?P<unhealthy>\d+)\s+idle=(?P<idle>\d+)",
|
|
17
|
+
re.IGNORECASE,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
_STEP_BY_STATUS = {
|
|
21
|
+
"draft": "Preparing your deployment...",
|
|
22
|
+
"awaiting_image": "Waiting to upload your image...",
|
|
23
|
+
"pushing": "Uploading your image...",
|
|
24
|
+
"validating": "Checking your image...",
|
|
25
|
+
"promoting": "Saving your image...",
|
|
26
|
+
"provisioning": "Starting your service...",
|
|
27
|
+
"live": "Your service is live.",
|
|
28
|
+
"failed_push": "Could not upload your image.",
|
|
29
|
+
"failed_validation": "Your image did not pass checks.",
|
|
30
|
+
"failed_provision": "Could not start your service.",
|
|
31
|
+
"cancelled": "This deployment was cancelled.",
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
_STEP_BY_PHASE = {
|
|
35
|
+
"queued_for_validation": "Checking your image...",
|
|
36
|
+
"vulnerability_scan": "Scanning your image for security issues...",
|
|
37
|
+
"promoting_image": "Saving your image...",
|
|
38
|
+
"provision_queued": "Starting your service...",
|
|
39
|
+
"waiting_for_service_health": "Waiting for your service to start...",
|
|
40
|
+
"waiting_for_openapi": "Your service is running. Finishing setup...",
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def site_url() -> str:
|
|
45
|
+
return os.environ.get("SOCAITY_FRONTEND_URL", DEFAULT_FRONTEND_URL).rstrip("/")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def service_page_url(service_id: Optional[str]) -> str:
|
|
49
|
+
base = site_url()
|
|
50
|
+
if service_id:
|
|
51
|
+
return f"{base}/account/my-services/{service_id}"
|
|
52
|
+
return f"{base}/account/hosting"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def parse_worker_counts(text: Optional[str]) -> Optional[Dict[str, int]]:
|
|
56
|
+
if not text:
|
|
57
|
+
return None
|
|
58
|
+
match = _WORKER_COUNTS_RE.search(text)
|
|
59
|
+
if not match:
|
|
60
|
+
return None
|
|
61
|
+
return {key: int(value) for key, value in match.groupdict().items()}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def is_progress_noise(text: Optional[str]) -> bool:
|
|
65
|
+
"""True when a backend 'error' field is actually worker progress."""
|
|
66
|
+
if not text:
|
|
67
|
+
return False
|
|
68
|
+
if "traceback" in text.lower():
|
|
69
|
+
return False
|
|
70
|
+
return parse_worker_counts(text) is not None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _worker_detail(counts: Dict[str, int], phase: Optional[str]) -> str:
|
|
74
|
+
if counts.get("unhealthy", 0) > 0:
|
|
75
|
+
return "The machine reported a problem. Waiting to confirm..."
|
|
76
|
+
if counts.get("initializing", 0) > 0:
|
|
77
|
+
return "A machine is starting up. This can take a few minutes."
|
|
78
|
+
live = counts.get("ready", 0) + counts.get("idle", 0)
|
|
79
|
+
if live > 0:
|
|
80
|
+
if phase == "waiting_for_openapi":
|
|
81
|
+
return "Your service is running. Finishing setup..."
|
|
82
|
+
return "Your service is running. Almost done..."
|
|
83
|
+
return "Waiting for a machine. This can take a few minutes."
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def progress_lines(status: Dict[str, Any]) -> Tuple[str, Optional[str]]:
|
|
87
|
+
"""Return ``(step, optional detail)`` for one poll snapshot."""
|
|
88
|
+
state = str(status.get("status") or "").strip().lower()
|
|
89
|
+
phase = status.get("phase")
|
|
90
|
+
phase_key = str(phase).strip() if phase else None
|
|
91
|
+
error = status.get("error")
|
|
92
|
+
analysis = status.get("analysis") if isinstance(status.get("analysis"), dict) else {}
|
|
93
|
+
hint = None
|
|
94
|
+
if isinstance(analysis, dict):
|
|
95
|
+
hint = analysis.get("runpod_progress")
|
|
96
|
+
hint = hint or (error if is_progress_noise(error) else None)
|
|
97
|
+
|
|
98
|
+
explicit = status.get("progress") or status.get("message")
|
|
99
|
+
if isinstance(explicit, str) and explicit.strip() and not is_progress_noise(explicit):
|
|
100
|
+
return explicit.strip(), None
|
|
101
|
+
|
|
102
|
+
step = _STEP_BY_PHASE.get(phase_key) or _STEP_BY_STATUS.get(state) or "Working on your deployment..."
|
|
103
|
+
detail = None
|
|
104
|
+
counts = parse_worker_counts(hint if isinstance(hint, str) else None)
|
|
105
|
+
if counts and state == "provisioning":
|
|
106
|
+
detail = _worker_detail(counts, phase_key)
|
|
107
|
+
if detail == step:
|
|
108
|
+
detail = None
|
|
109
|
+
return step, detail
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def humanize_failure(error: Optional[str], status: Optional[str] = None) -> str:
|
|
113
|
+
"""One-sentence failure for the user. Technical text is not the headline."""
|
|
114
|
+
state = str(status or "").strip().lower()
|
|
115
|
+
text = (error or "").strip()
|
|
116
|
+
lower = text.lower()
|
|
117
|
+
|
|
118
|
+
if state == "cancelled":
|
|
119
|
+
return "This deployment was cancelled."
|
|
120
|
+
if is_progress_noise(text):
|
|
121
|
+
text = ""
|
|
122
|
+
lower = ""
|
|
123
|
+
|
|
124
|
+
if "user is stopped" in lower:
|
|
125
|
+
return (
|
|
126
|
+
"Your account is paused. Check billing, then try deploying again.\n"
|
|
127
|
+
f" {site_url()}/account/billing"
|
|
128
|
+
)
|
|
129
|
+
if "no active" in lower and "vram" in lower:
|
|
130
|
+
return (
|
|
131
|
+
"No GPU with that much memory is available right now. "
|
|
132
|
+
"Lower gpu_vram_gb in your config, or try again later."
|
|
133
|
+
)
|
|
134
|
+
if "could not assign a worker" in lower:
|
|
135
|
+
return "We could not start a machine for your service. Please try again in a few minutes."
|
|
136
|
+
if "no live or starting workers" in lower or "no live workers" in lower:
|
|
137
|
+
return "The service stopped before it finished starting. Please try deploying again."
|
|
138
|
+
if "traceback" in lower or "container exited" in lower:
|
|
139
|
+
return "Your service crashed while starting. Fix the issue in your code, then deploy again."
|
|
140
|
+
if "could not promote" in lower or "harbor" in lower:
|
|
141
|
+
return "We could not save your image. Please try deploying again."
|
|
142
|
+
if "critical" in lower or "cve" in lower or "vulnerab" in lower:
|
|
143
|
+
return "Your image did not pass the security scan. Fix the reported issues, then deploy again."
|
|
144
|
+
if "openapi" in lower:
|
|
145
|
+
return (
|
|
146
|
+
"Your service started, but we could not read its API definition. "
|
|
147
|
+
"Make sure the service exposes /openapi.json, then deploy again."
|
|
148
|
+
)
|
|
149
|
+
if "quota" in lower or ("registry" in lower and "space" in lower):
|
|
150
|
+
return "Your image storage is full. Delete unused services, then try again."
|
|
151
|
+
|
|
152
|
+
if text:
|
|
153
|
+
return text
|
|
154
|
+
if state == "failed_push":
|
|
155
|
+
return "We could not upload your image. Check your connection and try again."
|
|
156
|
+
if state == "failed_validation":
|
|
157
|
+
return "Your image did not pass checks. Please try deploying again."
|
|
158
|
+
if state == "failed_provision":
|
|
159
|
+
return "We could not start your service. Please try deploying again."
|
|
160
|
+
return "Deployment did not finish. Please try again."
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def support_detail(error: Optional[str]) -> Optional[str]:
|
|
164
|
+
"""Technical leftover to print under the human sentence, if useful."""
|
|
165
|
+
text = (error or "").strip()
|
|
166
|
+
if not text or is_progress_noise(text):
|
|
167
|
+
return None
|
|
168
|
+
human = humanize_failure(text)
|
|
169
|
+
if text == human or text in human:
|
|
170
|
+
return None
|
|
171
|
+
return text
|
|
@@ -10,7 +10,17 @@ from typing import Any, Dict, Optional
|
|
|
10
10
|
|
|
11
11
|
from socaity_cli.backend_client import SocaityBackendClient
|
|
12
12
|
from socaity_cli.credentials import require_credentials
|
|
13
|
-
from socaity_cli.
|
|
13
|
+
from socaity_cli.deploy_messages import (
|
|
14
|
+
humanize_failure,
|
|
15
|
+
progress_lines,
|
|
16
|
+
service_page_url,
|
|
17
|
+
support_detail,
|
|
18
|
+
)
|
|
19
|
+
from socaity_cli.errors import (
|
|
20
|
+
BackendApiError,
|
|
21
|
+
PrivateSlotLimitError,
|
|
22
|
+
RegistryQuotaExceededError,
|
|
23
|
+
)
|
|
14
24
|
from socaity_cli.prompts import input_yes_no
|
|
15
25
|
from socaity_cli.registry_push import (
|
|
16
26
|
CredentialsExpired,
|
|
@@ -29,6 +39,10 @@ _TERMINAL_STATUSES = frozenset({
|
|
|
29
39
|
"cancelled",
|
|
30
40
|
})
|
|
31
41
|
_POLL_INTERVAL_S = 5.0
|
|
42
|
+
# Harbor often returns 201 minutes after Docker's header timeout. Wait
|
|
43
|
+
# before the next docker push so the blob can be skipped instead of
|
|
44
|
+
# starting a second multi-GB PUT while the first commit is still running.
|
|
45
|
+
_TRANSIENT_PUSH_BACKOFF_S = (120, 240, 360)
|
|
32
46
|
_DEFAULT_TIMEOUT_S = 3600.0
|
|
33
47
|
|
|
34
48
|
|
|
@@ -44,15 +58,24 @@ def print_analysis_report(report: Dict[str, Any]) -> None:
|
|
|
44
58
|
line += f" | catalog: {catalog.get('name')}{vram}"
|
|
45
59
|
print(line)
|
|
46
60
|
for warning in model.get("warnings", []):
|
|
47
|
-
print(f"
|
|
61
|
+
print(f" Note: {warning}")
|
|
48
62
|
gpu = report.get("recommended_gpu")
|
|
49
|
-
|
|
50
|
-
|
|
63
|
+
vram = report.get("recommended_gpu_vram_gb")
|
|
64
|
+
if gpu and vram:
|
|
65
|
+
gpu_has_vram = str(int(vram)) in str(gpu)
|
|
66
|
+
print(f" Recommended GPU: {gpu}" + ("" if gpu_has_vram else f" ({int(vram)} GB)"))
|
|
67
|
+
elif gpu:
|
|
68
|
+
print(f" Recommended GPU: {gpu}")
|
|
69
|
+
elif vram:
|
|
70
|
+
print(f" Recommended GPU memory: {int(vram)} GB")
|
|
71
|
+
disk = report.get("recommended_disk_gb")
|
|
72
|
+
if disk:
|
|
73
|
+
print(f" Recommended disk: {disk} GB")
|
|
51
74
|
for service in report.get("similar_services", []):
|
|
52
75
|
name = service.get("display_name") or service.get("name")
|
|
53
|
-
print(f" Similar public service: {name}
|
|
76
|
+
print(f" Similar public service: {name}")
|
|
54
77
|
for warning in report.get("warnings", []):
|
|
55
|
-
print(f"
|
|
78
|
+
print(f" Note: {warning}")
|
|
56
79
|
|
|
57
80
|
|
|
58
81
|
def analyze_deployment(config: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|
@@ -113,6 +136,35 @@ def prompt_gated_hf_token(client: SocaityBackendClient, report: Dict[str, Any])
|
|
|
113
136
|
print("Token not stored; gated repos will fail to download on deploy.")
|
|
114
137
|
|
|
115
138
|
|
|
139
|
+
def apply_gpu_upgrade_if_needed(
|
|
140
|
+
config: Dict[str, Any],
|
|
141
|
+
report: Optional[Dict[str, Any]],
|
|
142
|
+
*,
|
|
143
|
+
assume_yes: bool = False,
|
|
144
|
+
) -> bool:
|
|
145
|
+
"""Ask to bump gpu_vram_gb when analyze offers a larger SKU. False aborts."""
|
|
146
|
+
upgrade = report.get("gpu_upgrade") if isinstance(report, dict) else None
|
|
147
|
+
if not upgrade:
|
|
148
|
+
return True
|
|
149
|
+
requested = upgrade.get("requested_vram_gb")
|
|
150
|
+
available = upgrade.get("available_vram_gb")
|
|
151
|
+
gpu = upgrade.get("gpu") or f"{available} GB"
|
|
152
|
+
question = (
|
|
153
|
+
f"No {requested} GB GPU is available. The next size is {available} GB "
|
|
154
|
+
f"({gpu}). Use the larger GPU?"
|
|
155
|
+
)
|
|
156
|
+
if assume_yes:
|
|
157
|
+
print(f" {question} yes")
|
|
158
|
+
accepted = True
|
|
159
|
+
else:
|
|
160
|
+
accepted = input_yes_no(question)
|
|
161
|
+
if not accepted:
|
|
162
|
+
print("Aborted. Deploy will keep your current GPU size and will not pick a smaller one.")
|
|
163
|
+
return False
|
|
164
|
+
config["gpu_vram_gb"] = int(available)
|
|
165
|
+
return True
|
|
166
|
+
|
|
167
|
+
|
|
116
168
|
def create_deployment_draft(config: Dict[str, Any], report: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
|
|
117
169
|
"""Create a deployment draft after optional confirmation."""
|
|
118
170
|
require_credentials()
|
|
@@ -124,6 +176,9 @@ def create_deployment_draft(config: Dict[str, Any], report: Optional[Dict[str, A
|
|
|
124
176
|
return None
|
|
125
177
|
print_analysis_report(report)
|
|
126
178
|
|
|
179
|
+
if not apply_gpu_upgrade_if_needed(config, report):
|
|
180
|
+
return None
|
|
181
|
+
|
|
127
182
|
resolutions = prompt_model_resolutions(report)
|
|
128
183
|
prompt_gated_hf_token(client, report)
|
|
129
184
|
|
|
@@ -133,17 +188,28 @@ def create_deployment_draft(config: Dict[str, Any], report: Optional[Dict[str, A
|
|
|
133
188
|
|
|
134
189
|
try:
|
|
135
190
|
draft = client.create_deployment_draft(config, model_resolutions=resolutions)
|
|
136
|
-
except
|
|
137
|
-
|
|
138
|
-
|
|
191
|
+
except BackendApiError as exc:
|
|
192
|
+
if getattr(exc, "code", None) == "GPU_UPGRADE_AVAILABLE":
|
|
193
|
+
print(f"\n{exc}")
|
|
194
|
+
print("Confirm the larger GPU in analyze, or change gpu_vram_gb, then retry.")
|
|
195
|
+
return None
|
|
196
|
+
if getattr(exc, "code", None) == "AUTO_RECHARGE_REQUIRED":
|
|
197
|
+
print(f"\n{exc}")
|
|
198
|
+
print(
|
|
199
|
+
"Enable auto-recharge and save a card on Billing. "
|
|
200
|
+
"If a previous draft failed, delete that unused private service in My Services, then retry."
|
|
201
|
+
)
|
|
202
|
+
return None
|
|
203
|
+
if isinstance(exc, PrivateSlotLimitError):
|
|
204
|
+
print(f"\n{exc}")
|
|
205
|
+
print("Tip: publish a private service from the service page, or upgrade your plan.")
|
|
206
|
+
raise
|
|
139
207
|
raise
|
|
140
208
|
if draft is None:
|
|
141
209
|
print("Draft creation failed.")
|
|
142
210
|
return None
|
|
143
|
-
print(
|
|
144
|
-
|
|
145
|
-
"Next: the image will be pushed to the Socaity registry and provisioned automatically."
|
|
146
|
-
)
|
|
211
|
+
print("Draft created. Next we will upload your image and start the service.")
|
|
212
|
+
print(f" Resume later with: socaity deploy --resume {draft.get('deployment_id')}")
|
|
147
213
|
return draft
|
|
148
214
|
|
|
149
215
|
|
|
@@ -156,9 +222,17 @@ def push_image_to_registry(
|
|
|
156
222
|
"""Push a local image to the deployment staging repo. Returns the manifest digest."""
|
|
157
223
|
client = client or SocaityBackendClient()
|
|
158
224
|
estimated_size = image_size_bytes(local_tag)
|
|
159
|
-
|
|
225
|
+
try:
|
|
226
|
+
creds = client.get_push_credentials(deployment_id, estimated_size_bytes=estimated_size)
|
|
227
|
+
except RegistryQuotaExceededError as exc:
|
|
228
|
+
print(f"\n{exc}")
|
|
229
|
+
print(
|
|
230
|
+
"This attempt was dropped so it does not occupy a private slot. "
|
|
231
|
+
"Delete unused services to free storage space, then deploy again."
|
|
232
|
+
)
|
|
233
|
+
return None
|
|
160
234
|
if creds is None:
|
|
161
|
-
print("Could not
|
|
235
|
+
print("Could not get permission to upload your image. Please try again.")
|
|
162
236
|
return None
|
|
163
237
|
|
|
164
238
|
refreshed = False
|
|
@@ -176,25 +250,39 @@ def push_image_to_registry(
|
|
|
176
250
|
return digest
|
|
177
251
|
except CredentialsExpired:
|
|
178
252
|
if not refreshed:
|
|
179
|
-
print("
|
|
180
|
-
|
|
253
|
+
print("Upload permission expired. Getting a new one...")
|
|
254
|
+
try:
|
|
255
|
+
creds = client.get_push_credentials(deployment_id, estimated_size_bytes=estimated_size)
|
|
256
|
+
except RegistryQuotaExceededError as exc:
|
|
257
|
+
print(f"\n{exc}")
|
|
258
|
+
print(
|
|
259
|
+
"This attempt was dropped so it does not occupy a private slot. "
|
|
260
|
+
"Delete unused services to free storage space, then deploy again."
|
|
261
|
+
)
|
|
262
|
+
return None
|
|
181
263
|
if creds is None:
|
|
182
|
-
print("Could not refresh
|
|
264
|
+
print("Could not refresh upload permission. Please try again.")
|
|
183
265
|
return None
|
|
184
266
|
refreshed = True
|
|
185
267
|
continue
|
|
186
|
-
print("
|
|
268
|
+
print("Upload permission expired again. Please retry the deploy.")
|
|
187
269
|
return None
|
|
188
270
|
except DockerNotAvailable as exc:
|
|
189
271
|
print(str(exc))
|
|
190
272
|
return None
|
|
191
273
|
except PushFailed as exc:
|
|
192
|
-
if
|
|
274
|
+
if (
|
|
275
|
+
is_transient_push_error(str(exc))
|
|
276
|
+
and transient_attempts < len(_TRANSIENT_PUSH_BACKOFF_S)
|
|
277
|
+
):
|
|
278
|
+
wait = _TRANSIENT_PUSH_BACKOFF_S[transient_attempts]
|
|
193
279
|
transient_attempts += 1
|
|
280
|
+
print("Upload was interrupted. Waiting for the registry to finish the last layer...")
|
|
194
281
|
print(
|
|
195
|
-
f"
|
|
196
|
-
f"({transient_attempts}/
|
|
282
|
+
f"Retrying in {wait}s "
|
|
283
|
+
f"({transient_attempts}/{len(_TRANSIENT_PUSH_BACKOFF_S)})..."
|
|
197
284
|
)
|
|
285
|
+
time.sleep(wait)
|
|
198
286
|
continue
|
|
199
287
|
print(str(exc))
|
|
200
288
|
return None
|
|
@@ -205,30 +293,54 @@ def poll_deployment_until_terminal(
|
|
|
205
293
|
*,
|
|
206
294
|
timeout_s: float = _DEFAULT_TIMEOUT_S,
|
|
207
295
|
client: Optional[SocaityBackendClient] = None,
|
|
296
|
+
printer=print,
|
|
297
|
+
sleep=time.sleep,
|
|
298
|
+
now=time.monotonic,
|
|
208
299
|
) -> Optional[Dict[str, Any]]:
|
|
209
300
|
"""Poll pipeline status until live or a terminal failure."""
|
|
210
301
|
client = client or SocaityBackendClient()
|
|
211
|
-
deadline =
|
|
302
|
+
deadline = now() + timeout_s
|
|
212
303
|
last_status: Optional[Dict[str, Any]] = None
|
|
304
|
+
last_step: Optional[str] = None
|
|
305
|
+
last_detail: Optional[str] = None
|
|
306
|
+
last_print_at = now()
|
|
307
|
+
saw_progress = False
|
|
213
308
|
|
|
214
|
-
while
|
|
309
|
+
while now() < deadline:
|
|
215
310
|
status = client.get_deployment_pipeline_status(deployment_id)
|
|
216
311
|
if status is None:
|
|
217
|
-
|
|
218
|
-
|
|
312
|
+
printer("Could not check progress. Retrying...")
|
|
313
|
+
sleep(_POLL_INTERVAL_S)
|
|
219
314
|
continue
|
|
220
315
|
|
|
221
316
|
last_status = status
|
|
222
|
-
phase = status.get("phase")
|
|
223
317
|
state = str(status.get("status", "")).lower()
|
|
224
|
-
label = f"{state}" + (f" ({phase})" if phase else "")
|
|
225
|
-
print(f" deployment status: {label}")
|
|
226
|
-
|
|
227
318
|
if state in _TERMINAL_STATUSES:
|
|
228
319
|
return status
|
|
229
|
-
time.sleep(_POLL_INTERVAL_S)
|
|
230
320
|
|
|
231
|
-
|
|
321
|
+
step, detail = progress_lines(status)
|
|
322
|
+
if step != last_step:
|
|
323
|
+
printer(f" {step}")
|
|
324
|
+
last_step = step
|
|
325
|
+
last_detail = None
|
|
326
|
+
last_print_at = now()
|
|
327
|
+
saw_progress = True
|
|
328
|
+
if detail and detail != last_detail and detail != step:
|
|
329
|
+
printer(f" {detail}")
|
|
330
|
+
last_detail = detail
|
|
331
|
+
last_print_at = now()
|
|
332
|
+
saw_progress = True
|
|
333
|
+
elif saw_progress and now() - last_print_at >= 60:
|
|
334
|
+
printer(" Still working...")
|
|
335
|
+
last_print_at = now()
|
|
336
|
+
|
|
337
|
+
sleep(_POLL_INTERVAL_S)
|
|
338
|
+
|
|
339
|
+
minutes = max(1, int(timeout_s) // 60)
|
|
340
|
+
printer(
|
|
341
|
+
f"Timed out after {minutes} minute{'s' if minutes != 1 else ''}. "
|
|
342
|
+
"Check the service page or try deploying again."
|
|
343
|
+
)
|
|
232
344
|
return last_status
|
|
233
345
|
|
|
234
346
|
|
|
@@ -244,6 +356,9 @@ def _create_draft_with_prompts(
|
|
|
244
356
|
return None
|
|
245
357
|
print_analysis_report(report)
|
|
246
358
|
|
|
359
|
+
if not apply_gpu_upgrade_if_needed(config, report, assume_yes=assume_yes):
|
|
360
|
+
return None
|
|
361
|
+
|
|
247
362
|
if assume_yes:
|
|
248
363
|
resolutions: Dict[str, Optional[str]] = {}
|
|
249
364
|
for model in report.get("models", []):
|
|
@@ -263,20 +378,75 @@ def _create_draft_with_prompts(
|
|
|
263
378
|
|
|
264
379
|
try:
|
|
265
380
|
draft = client.create_deployment_draft(config, model_resolutions=resolutions)
|
|
266
|
-
except
|
|
267
|
-
|
|
268
|
-
|
|
381
|
+
except BackendApiError as exc:
|
|
382
|
+
if getattr(exc, "code", None) == "GPU_UPGRADE_AVAILABLE":
|
|
383
|
+
print(f"\n{exc}")
|
|
384
|
+
print("Confirm the larger GPU in analyze, or change gpu_vram_gb, then retry.")
|
|
385
|
+
return None
|
|
386
|
+
if getattr(exc, "code", None) == "AUTO_RECHARGE_REQUIRED":
|
|
387
|
+
print(f"\n{exc}")
|
|
388
|
+
print(
|
|
389
|
+
"Enable auto-recharge and save a card on Billing. "
|
|
390
|
+
"If a previous draft failed, delete that unused private service in My Services, then retry."
|
|
391
|
+
)
|
|
392
|
+
return None
|
|
393
|
+
if isinstance(exc, PrivateSlotLimitError):
|
|
394
|
+
print(f"\n{exc}")
|
|
395
|
+
print("Tip: publish a private service from the service page, or upgrade your plan.")
|
|
396
|
+
raise
|
|
269
397
|
raise
|
|
270
398
|
if draft is None:
|
|
271
399
|
print("Draft creation failed.")
|
|
272
400
|
return None
|
|
273
|
-
print(
|
|
274
|
-
|
|
275
|
-
f"deployment_id={draft.get('deployment_id')}"
|
|
276
|
-
)
|
|
401
|
+
print("Draft created. Next we will upload your image and start the service.")
|
|
402
|
+
print(f" Resume later with: socaity deploy --resume {draft.get('deployment_id')}")
|
|
277
403
|
return draft
|
|
278
404
|
|
|
279
405
|
|
|
406
|
+
def _print_failure_and_maybe_delete(
|
|
407
|
+
client: SocaityBackendClient,
|
|
408
|
+
deployment_id: str,
|
|
409
|
+
status: Optional[Dict[str, Any]],
|
|
410
|
+
*,
|
|
411
|
+
assume_yes: bool,
|
|
412
|
+
) -> None:
|
|
413
|
+
"""Show why the deploy failed, then delete when asked (or with --yes)."""
|
|
414
|
+
snapshot = dict(status) if isinstance(status, dict) else {}
|
|
415
|
+
if not snapshot.get("error") and not snapshot.get("logs"):
|
|
416
|
+
fresh = client.get_deployment_pipeline_status(deployment_id)
|
|
417
|
+
if isinstance(fresh, dict):
|
|
418
|
+
snapshot = fresh
|
|
419
|
+
state = str(snapshot.get("status") or "").lower()
|
|
420
|
+
print(f"\n{humanize_failure(snapshot.get('error'), state)}")
|
|
421
|
+
logs = snapshot.get("logs")
|
|
422
|
+
if logs:
|
|
423
|
+
print("\nLogs:")
|
|
424
|
+
for line in str(logs).splitlines():
|
|
425
|
+
print(f" {line}")
|
|
426
|
+
else:
|
|
427
|
+
detail = support_detail(snapshot.get("error"))
|
|
428
|
+
if detail:
|
|
429
|
+
print(f" Details: {detail}")
|
|
430
|
+
if assume_yes:
|
|
431
|
+
deleted = client.cancel_deployment_pipeline(deployment_id)
|
|
432
|
+
if deleted is None:
|
|
433
|
+
print("Could not delete the failed service. Delete it from My Services.")
|
|
434
|
+
print(f" {service_page_url(snapshot.get('service_id'))}")
|
|
435
|
+
return
|
|
436
|
+
print("Deleted the failed service.")
|
|
437
|
+
return
|
|
438
|
+
print("This failed service is on My Services and still uses a private slot.")
|
|
439
|
+
if not input_yes_no("Delete this failed service now?", default=False):
|
|
440
|
+
print(f" {service_page_url(snapshot.get('service_id'))}")
|
|
441
|
+
return
|
|
442
|
+
deleted = client.cancel_deployment_pipeline(deployment_id)
|
|
443
|
+
if deleted is None:
|
|
444
|
+
print("Could not delete the failed service. Delete it from My Services.")
|
|
445
|
+
print(f" {service_page_url(snapshot.get('service_id'))}")
|
|
446
|
+
return
|
|
447
|
+
print("Deleted the failed service.")
|
|
448
|
+
|
|
449
|
+
|
|
280
450
|
def run_full_deploy(
|
|
281
451
|
config: Dict[str, Any],
|
|
282
452
|
*,
|
|
@@ -292,19 +462,20 @@ def run_full_deploy(
|
|
|
292
462
|
if deployment_id:
|
|
293
463
|
status = client.get_deployment_pipeline_status(deployment_id)
|
|
294
464
|
if status is None:
|
|
295
|
-
print(
|
|
465
|
+
print("That deployment was not found, or you do not have access to it.")
|
|
296
466
|
return None
|
|
297
467
|
state = str(status.get("status", "")).lower()
|
|
298
468
|
if state == "live":
|
|
299
|
-
print(
|
|
469
|
+
print("This service is already live.")
|
|
470
|
+
if status.get("service_id"):
|
|
471
|
+
print(f" {service_page_url(status.get('service_id'))}")
|
|
300
472
|
return status
|
|
301
473
|
if state in _TERMINAL_STATUSES - {"live"}:
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
"Create a new draft or fix the issue before resuming."
|
|
474
|
+
_print_failure_and_maybe_delete(
|
|
475
|
+
client, deployment_id, status, assume_yes=assume_yes,
|
|
305
476
|
)
|
|
306
477
|
return status
|
|
307
|
-
print(
|
|
478
|
+
print("Continuing your deployment...")
|
|
308
479
|
else:
|
|
309
480
|
draft = _create_draft_with_prompts(config, client=client, assume_yes=assume_yes)
|
|
310
481
|
if draft is None:
|
|
@@ -317,37 +488,45 @@ def run_full_deploy(
|
|
|
317
488
|
current = client.get_deployment_pipeline_status(deployment_id) or {}
|
|
318
489
|
state = str(current.get("status", "")).lower()
|
|
319
490
|
if state in {"draft", "awaiting_image", "pushing", "failed_push"}:
|
|
320
|
-
print(
|
|
491
|
+
print("Uploading your image...")
|
|
321
492
|
digest = push_image_to_registry(deployment_id, local_tag, client=client)
|
|
322
493
|
if digest is None:
|
|
494
|
+
_print_failure_and_maybe_delete(
|
|
495
|
+
client, deployment_id, current, assume_yes=assume_yes,
|
|
496
|
+
)
|
|
323
497
|
return None
|
|
324
498
|
size_bytes = image_size_bytes(local_tag)
|
|
325
499
|
pushed = client.confirm_image_pushed(deployment_id, digest=digest, size_bytes=size_bytes)
|
|
326
500
|
if pushed is None:
|
|
327
|
-
print("
|
|
501
|
+
print("The server did not accept the uploaded image. Please try again.")
|
|
502
|
+
_print_failure_and_maybe_delete(
|
|
503
|
+
client, deployment_id, current, assume_yes=assume_yes,
|
|
504
|
+
)
|
|
328
505
|
return None
|
|
329
|
-
print(
|
|
506
|
+
print("Image uploaded. Checking it and starting your service...")
|
|
330
507
|
elif state in {"validating", "promoting", "provisioning"}:
|
|
331
|
-
print(
|
|
508
|
+
print("Image already uploaded. Checking it and starting your service...")
|
|
332
509
|
elif state == "live":
|
|
333
|
-
print(
|
|
510
|
+
print("This service is already live.")
|
|
511
|
+
if current.get("service_id"):
|
|
512
|
+
print(f" {service_page_url(current.get('service_id'))}")
|
|
334
513
|
return current
|
|
335
514
|
|
|
336
|
-
print("
|
|
515
|
+
print("This can take several minutes.")
|
|
337
516
|
final = poll_deployment_until_terminal(deployment_id, client=client)
|
|
338
517
|
if final is None:
|
|
339
518
|
return None
|
|
340
519
|
|
|
341
520
|
final_state = str(final.get("status", "")).lower()
|
|
342
521
|
if final_state == "live":
|
|
343
|
-
print(
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
522
|
+
print("\nYour service is live.")
|
|
523
|
+
print(f" {service_page_url(final.get('service_id'))}")
|
|
524
|
+
return final
|
|
525
|
+
if final_state in _TERMINAL_STATUSES:
|
|
526
|
+
_print_failure_and_maybe_delete(
|
|
527
|
+
client, deployment_id, final, assume_yes=assume_yes,
|
|
528
|
+
)
|
|
529
|
+
return final
|
|
351
530
|
return final
|
|
352
531
|
|
|
353
532
|
|
|
@@ -33,3 +33,10 @@ class PrivateSlotLimitError(BackendApiError):
|
|
|
33
33
|
"Publish an existing private service or upgrade your plan to free a slot."
|
|
34
34
|
)
|
|
35
35
|
super().__init__(403, message, code="PRIVATE_SLOT_LIMIT", detail=detail)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class RegistryQuotaExceededError(BackendApiError):
|
|
39
|
+
"""Harbor image quota would be exceeded by this push."""
|
|
40
|
+
|
|
41
|
+
def __init__(self, message: str, *, detail: Any = None):
|
|
42
|
+
super().__init__(413, message, code="REGISTRY_QUOTA_EXCEEDED", detail=detail)
|
|
@@ -6,7 +6,7 @@ from typing import Any, Dict, Optional
|
|
|
6
6
|
import httpx
|
|
7
7
|
|
|
8
8
|
from socaity_cli.credentials import DEFAULT_BACKEND_URL, get_api_key
|
|
9
|
-
from socaity_cli.errors import BackendApiError, PrivateSlotLimitError
|
|
9
|
+
from socaity_cli.errors import BackendApiError, PrivateSlotLimitError, RegistryQuotaExceededError
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class BackendHttp:
|
|
@@ -30,6 +30,14 @@ class BackendHttp:
|
|
|
30
30
|
detail = body.get("detail") if isinstance(body, dict) else None
|
|
31
31
|
if isinstance(detail, dict) and detail.get("code") == "PRIVATE_SLOT_LIMIT":
|
|
32
32
|
raise PrivateSlotLimitError(detail)
|
|
33
|
+
if status_code == 413:
|
|
34
|
+
if isinstance(detail, str) and detail.strip():
|
|
35
|
+
message = detail.strip()
|
|
36
|
+
elif isinstance(detail, dict):
|
|
37
|
+
message = str(detail.get("message") or detail)
|
|
38
|
+
else:
|
|
39
|
+
message = (body_text or "Registry quota exceeded.")[:300]
|
|
40
|
+
raise RegistryQuotaExceededError(message, detail=detail)
|
|
33
41
|
if isinstance(detail, dict) and detail.get("code"):
|
|
34
42
|
raise BackendApiError(
|
|
35
43
|
status_code,
|
|
@@ -64,7 +72,7 @@ class BackendHttp:
|
|
|
64
72
|
self._raise_for_structured_error(response.status_code, response.text)
|
|
65
73
|
print(f"Request to {path} failed with status {response.status_code}: {response.text[:300]}")
|
|
66
74
|
return None
|
|
67
|
-
except (PrivateSlotLimitError, BackendApiError):
|
|
75
|
+
except (PrivateSlotLimitError, RegistryQuotaExceededError, BackendApiError):
|
|
68
76
|
raise
|
|
69
77
|
except Exception as e:
|
|
70
78
|
print(f"Request to {path} failed: {e}")
|
|
@@ -1,15 +1,19 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Push a local image to the Socaity registry with ephemeral credentials.
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
Prefers ``skopeo`` (long command timeout, serial layers, built-in retries).
|
|
4
|
+
Docker 29's ``docker push`` uses a short ResponseHeaderTimeout and aborts
|
|
5
|
+
Harbor commits that later succeed. Credentials never land in ~/.docker.
|
|
5
6
|
"""
|
|
6
7
|
from __future__ import annotations
|
|
7
8
|
|
|
9
|
+
import base64
|
|
8
10
|
import json
|
|
9
11
|
import os
|
|
10
12
|
import re
|
|
13
|
+
import shutil
|
|
11
14
|
import subprocess
|
|
12
15
|
import tempfile
|
|
16
|
+
from pathlib import Path
|
|
13
17
|
from typing import Optional, Tuple
|
|
14
18
|
|
|
15
19
|
_DIGEST_RE = re.compile(r"digest:\s*(sha256:[0-9a-f]{64})")
|
|
@@ -21,6 +25,8 @@ _TRANSIENT_PUSH_MARKERS = (
|
|
|
21
25
|
"tls handshake",
|
|
22
26
|
"unexpected eof",
|
|
23
27
|
)
|
|
28
|
+
_SKOPEO_TIMEOUT = os.environ.get("SOCAITY_PUSH_TIMEOUT", "2h")
|
|
29
|
+
_SKOPEO_RETRIES = int(os.environ.get("SOCAITY_PUSH_RETRIES", "20"))
|
|
24
30
|
|
|
25
31
|
|
|
26
32
|
class DockerNotAvailable(RuntimeError):
|
|
@@ -41,6 +47,44 @@ def is_transient_push_error(message: str) -> bool:
|
|
|
41
47
|
return any(marker in lower for marker in _TRANSIENT_PUSH_MARKERS)
|
|
42
48
|
|
|
43
49
|
|
|
50
|
+
def daemon_ref(local_tag: str) -> str:
|
|
51
|
+
"""``docker-daemon:`` reference; add ``:latest`` when the tag is omitted."""
|
|
52
|
+
name = local_tag.strip()
|
|
53
|
+
if not name:
|
|
54
|
+
raise PushFailed("Local image tag is empty.")
|
|
55
|
+
if ":" not in name.rsplit("/", 1)[-1]:
|
|
56
|
+
return f"{name}:latest"
|
|
57
|
+
return name
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _skopeo_copy_args(
|
|
61
|
+
*,
|
|
62
|
+
src: str,
|
|
63
|
+
dest: str,
|
|
64
|
+
authfile: str,
|
|
65
|
+
digestfile: str,
|
|
66
|
+
) -> list:
|
|
67
|
+
return [
|
|
68
|
+
"skopeo",
|
|
69
|
+
f"--command-timeout={_SKOPEO_TIMEOUT}",
|
|
70
|
+
"copy",
|
|
71
|
+
"--dest-tls-verify=true",
|
|
72
|
+
f"--dest-authfile={authfile}",
|
|
73
|
+
"--image-parallel-copies=1",
|
|
74
|
+
f"--retry-times={_SKOPEO_RETRIES}",
|
|
75
|
+
"--retry-delay=15s",
|
|
76
|
+
"--dest-precompute-digests",
|
|
77
|
+
f"--digestfile={digestfile}",
|
|
78
|
+
src,
|
|
79
|
+
dest,
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _write_authfile(path: str, registry: str, username: str, password: str) -> None:
|
|
84
|
+
token = base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
85
|
+
Path(path).write_text(json.dumps({"auths": {registry: {"auth": token}}}))
|
|
86
|
+
|
|
87
|
+
|
|
44
88
|
def _push_command(ref: str) -> list:
|
|
45
89
|
"""``docker push`` args. Docker 25+ dropped ``--max-concurrent-uploads``."""
|
|
46
90
|
help_out = _docker(["push", "--help"])
|
|
@@ -78,7 +122,7 @@ def push_image(
|
|
|
78
122
|
password: str,
|
|
79
123
|
local_tag: str,
|
|
80
124
|
) -> Tuple[str, int]:
|
|
81
|
-
"""
|
|
125
|
+
"""Copy ``local_tag`` to ``registry/repository:tag``.
|
|
82
126
|
|
|
83
127
|
Returns:
|
|
84
128
|
(manifest digest reported by the registry, local image size in bytes).
|
|
@@ -86,20 +130,94 @@ def push_image(
|
|
|
86
130
|
Raises:
|
|
87
131
|
DockerNotAvailable, CredentialsExpired, PushFailed.
|
|
88
132
|
"""
|
|
89
|
-
ref = f"{registry}/{repository}:{tag}"
|
|
90
133
|
size = image_size_bytes(local_tag)
|
|
91
134
|
if size == 0:
|
|
92
135
|
raise PushFailed(f"Local image '{local_tag}' not found. Build it first (apipod build).")
|
|
93
136
|
|
|
94
|
-
|
|
137
|
+
dest = f"{registry}/{repository}:{tag}"
|
|
138
|
+
if shutil.which("skopeo"):
|
|
139
|
+
print(
|
|
140
|
+
f" Pushing with skopeo ({_SKOPEO_TIMEOUT} timeout, one layer at a time, "
|
|
141
|
+
f"{_SKOPEO_RETRIES} retries)..."
|
|
142
|
+
)
|
|
143
|
+
return _push_with_skopeo(
|
|
144
|
+
registry=registry,
|
|
145
|
+
dest=dest,
|
|
146
|
+
username=username,
|
|
147
|
+
password=password,
|
|
148
|
+
local_tag=local_tag,
|
|
149
|
+
size=size,
|
|
150
|
+
)
|
|
151
|
+
print(" skopeo not found; falling back to docker push (short Harbor header timeout).")
|
|
152
|
+
return _push_with_docker(
|
|
153
|
+
registry=registry,
|
|
154
|
+
dest=dest,
|
|
155
|
+
username=username,
|
|
156
|
+
password=password,
|
|
157
|
+
local_tag=local_tag,
|
|
158
|
+
size=size,
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _push_with_skopeo(
|
|
163
|
+
*,
|
|
164
|
+
registry: str,
|
|
165
|
+
dest: str,
|
|
166
|
+
username: str,
|
|
167
|
+
password: str,
|
|
168
|
+
local_tag: str,
|
|
169
|
+
size: int,
|
|
170
|
+
) -> Tuple[str, int]:
|
|
171
|
+
src = f"docker-daemon:{daemon_ref(local_tag)}"
|
|
172
|
+
dest_ref = f"docker://{dest}"
|
|
173
|
+
with tempfile.TemporaryDirectory(prefix="socaity-skopeo-") as tmp:
|
|
174
|
+
authfile = os.path.join(tmp, "auth.json")
|
|
175
|
+
digestfile = os.path.join(tmp, "digest")
|
|
176
|
+
_write_authfile(authfile, registry, username, password)
|
|
177
|
+
cmd = _skopeo_copy_args(src=src, dest=dest_ref, authfile=authfile, digestfile=digestfile)
|
|
178
|
+
try:
|
|
179
|
+
push = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
|
180
|
+
except FileNotFoundError as exc:
|
|
181
|
+
raise PushFailed("skopeo disappeared from PATH during the push.") from exc
|
|
182
|
+
last_lines = []
|
|
183
|
+
for line in push.stdout:
|
|
184
|
+
line = line.rstrip()
|
|
185
|
+
last_lines = (last_lines + [line])[-8:]
|
|
186
|
+
if line:
|
|
187
|
+
print(f" {line}")
|
|
188
|
+
push.wait()
|
|
189
|
+
tail = " | ".join(last_lines)
|
|
190
|
+
if push.returncode != 0:
|
|
191
|
+
if "unauthorized" in tail.lower() or "401" in tail.lower() or "authentication required" in tail.lower():
|
|
192
|
+
raise CredentialsExpired("Push rejected mid-transfer; the credentials expired.")
|
|
193
|
+
raise PushFailed(f"skopeo copy failed: {tail[:400]}")
|
|
194
|
+
digest = Path(digestfile).read_text().strip() if os.path.exists(digestfile) else ""
|
|
195
|
+
if not digest.startswith("sha256:"):
|
|
196
|
+
raise PushFailed("skopeo copy succeeded but no digest file was written.")
|
|
197
|
+
return digest, size
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _push_with_docker(
|
|
201
|
+
*,
|
|
202
|
+
registry: str,
|
|
203
|
+
dest: str,
|
|
204
|
+
username: str,
|
|
205
|
+
password: str,
|
|
206
|
+
local_tag: str,
|
|
207
|
+
size: int,
|
|
208
|
+
) -> Tuple[str, int]:
|
|
209
|
+
result = _docker(["tag", local_tag, dest])
|
|
95
210
|
if result.returncode != 0:
|
|
96
211
|
raise PushFailed(f"docker tag failed: {result.stderr.decode()[:300]}")
|
|
97
212
|
|
|
98
213
|
with tempfile.TemporaryDirectory(prefix="socaity-push-") as docker_config:
|
|
99
214
|
env = {**os.environ, "DOCKER_CONFIG": docker_config}
|
|
100
215
|
try:
|
|
101
|
-
login = _docker(
|
|
102
|
-
|
|
216
|
+
login = _docker(
|
|
217
|
+
["login", registry, "-u", username, "--password-stdin"],
|
|
218
|
+
env=env,
|
|
219
|
+
input_bytes=password.encode(),
|
|
220
|
+
)
|
|
103
221
|
if login.returncode != 0:
|
|
104
222
|
stderr = login.stderr.decode()
|
|
105
223
|
if "unauthorized" in stderr.lower() or "401" in stderr:
|
|
@@ -107,8 +225,11 @@ def push_image(
|
|
|
107
225
|
raise PushFailed(f"docker login failed: {stderr[:300]}")
|
|
108
226
|
|
|
109
227
|
push = subprocess.Popen(
|
|
110
|
-
_push_command(
|
|
111
|
-
stdout=subprocess.PIPE,
|
|
228
|
+
_push_command(dest),
|
|
229
|
+
stdout=subprocess.PIPE,
|
|
230
|
+
stderr=subprocess.STDOUT,
|
|
231
|
+
env=env,
|
|
232
|
+
text=True,
|
|
112
233
|
)
|
|
113
234
|
digest, last_lines = None, []
|
|
114
235
|
for line in push.stdout:
|
|
@@ -126,12 +247,12 @@ def push_image(
|
|
|
126
247
|
raise CredentialsExpired("Push rejected mid-transfer; the credentials expired.")
|
|
127
248
|
raise PushFailed(f"docker push failed: {tail[:400]}")
|
|
128
249
|
if not digest:
|
|
129
|
-
digest = _digest_from_local_metadata(
|
|
250
|
+
digest = _digest_from_local_metadata(dest, env)
|
|
130
251
|
if not digest:
|
|
131
252
|
raise PushFailed("Push succeeded but no digest was reported by the registry.")
|
|
132
253
|
return digest, size
|
|
133
254
|
finally:
|
|
134
|
-
_docker(["rmi",
|
|
255
|
+
_docker(["rmi", dest], env=env)
|
|
135
256
|
|
|
136
257
|
|
|
137
258
|
def _digest_from_local_metadata(ref: str, env: dict) -> Optional[str]:
|
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
from socaity_cli.registry_push import
|
|
1
|
+
from socaity_cli.registry_push import (
|
|
2
|
+
_push_command,
|
|
3
|
+
_skopeo_copy_args,
|
|
4
|
+
daemon_ref,
|
|
5
|
+
is_transient_push_error,
|
|
6
|
+
)
|
|
2
7
|
|
|
3
8
|
|
|
4
9
|
def test_is_transient_push_error_matches_harbor_timeouts():
|
|
@@ -30,3 +35,25 @@ def test_push_command_uses_max_concurrent_when_supported(monkeypatch):
|
|
|
30
35
|
assert _push_command("harbor.example/repo:tag") == [
|
|
31
36
|
"docker", "push", "--max-concurrent-uploads=1", "harbor.example/repo:tag",
|
|
32
37
|
]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_daemon_ref_adds_latest():
|
|
41
|
+
assert daemon_ref("apipod-qwen38") == "apipod-qwen38:latest"
|
|
42
|
+
assert daemon_ref("apipod-qwen38:latest") == "apipod-qwen38:latest"
|
|
43
|
+
assert daemon_ref("ghcr.io/org/app:v1") == "ghcr.io/org/app:v1"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_skopeo_copy_args_are_serial_with_long_timeout():
|
|
47
|
+
args = _skopeo_copy_args(
|
|
48
|
+
src="docker-daemon:app:latest",
|
|
49
|
+
dest="docker://harbor.example/repo:tag",
|
|
50
|
+
authfile="/tmp/auth.json",
|
|
51
|
+
digestfile="/tmp/digest",
|
|
52
|
+
)
|
|
53
|
+
assert args[0] == "skopeo"
|
|
54
|
+
assert args[1] == "--command-timeout=2h"
|
|
55
|
+
assert "--image-parallel-copies=1" in args
|
|
56
|
+
assert "--retry-times=20" in args
|
|
57
|
+
assert "--dest-precompute-digests" in args
|
|
58
|
+
assert "docker-daemon:app:latest" in args
|
|
59
|
+
assert "docker://harbor.example/repo:tag" in args
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|