mimiry 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mimiry-0.1.0/LICENSE +21 -0
- mimiry-0.1.0/PKG-INFO +118 -0
- mimiry-0.1.0/README.md +93 -0
- mimiry-0.1.0/mimiry/__init__.py +54 -0
- mimiry-0.1.0/mimiry/_auth.py +70 -0
- mimiry-0.1.0/mimiry/_exceptions.py +48 -0
- mimiry-0.1.0/mimiry/_session.py +262 -0
- mimiry-0.1.0/mimiry/client.py +591 -0
- mimiry-0.1.0/mimiry.egg-info/PKG-INFO +118 -0
- mimiry-0.1.0/mimiry.egg-info/SOURCES.txt +13 -0
- mimiry-0.1.0/mimiry.egg-info/dependency_links.txt +1 -0
- mimiry-0.1.0/mimiry.egg-info/requires.txt +4 -0
- mimiry-0.1.0/mimiry.egg-info/top_level.txt +1 -0
- mimiry-0.1.0/pyproject.toml +35 -0
- mimiry-0.1.0/setup.cfg +4 -0
mimiry-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Mimiry
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
mimiry-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mimiry
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python SDK for the Mimiry GPU compute platform
|
|
5
|
+
Author-email: Mimiry <team@mimiry.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: gpu,cloud,compute,machine-learning,cuda,mimiry
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: pytest; extra == "dev"
|
|
23
|
+
Requires-Dist: ruff; extra == "dev"
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# Mimiry Python SDK
|
|
27
|
+
|
|
28
|
+
Run GPU jobs on [Mimiry](https://mimiry.com) from Python — no shell scripts, no manual API calls.
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install mimiry
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Requirements
|
|
35
|
+
|
|
36
|
+
- Python 3.10 or newer
|
|
37
|
+
- [Mimiry CLI](https://mimiry.com) installed and authenticated (`mimiry auth login`)
|
|
38
|
+
|
|
39
|
+
## Quick start
|
|
40
|
+
|
|
41
|
+
**Run a command on a GPU and get the output:**
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
from mimiry import MimiryClient
|
|
45
|
+
|
|
46
|
+
client = MimiryClient()
|
|
47
|
+
result = client.run("nvidia-smi", verbose=True)
|
|
48
|
+
result.print_logs()
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
**Run a Python script on a GPU:**
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from mimiry import MimiryClient, PYTORCH_IMAGE
|
|
55
|
+
|
|
56
|
+
client = MimiryClient()
|
|
57
|
+
result = client.run_script("train.py", image=PYTORCH_IMAGE, verbose=True)
|
|
58
|
+
result.print_logs()
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
**Or pass an inline script directly:**
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
result = client.run_script(
|
|
65
|
+
"""
|
|
66
|
+
import torch
|
|
67
|
+
print(torch.cuda.get_device_name(0))
|
|
68
|
+
""",
|
|
69
|
+
image=PYTORCH_IMAGE,
|
|
70
|
+
)
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
**Check your balance and quota:**
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
print(client.balance())
|
|
77
|
+
print(client.quota())
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
**Clean up all running sessions:**
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
client.kill_all()
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## How it works
|
|
87
|
+
|
|
88
|
+
Each `client.run()` call:
|
|
89
|
+
|
|
90
|
+
1. Creates a session via the Mimiry Sessions API
|
|
91
|
+
2. Polls until the VM reaches `running` status (~30–55 s)
|
|
92
|
+
3. Waits for Docker and the GPU driver to finish loading (~60–90 s)
|
|
93
|
+
4. Fetches the command output from the platform log endpoint
|
|
94
|
+
5. Terminates the session automatically
|
|
95
|
+
|
|
96
|
+
Sessions are always terminated after the job finishes. If a script crashes before cleanup, run `client.kill_all()`.
|
|
97
|
+
|
|
98
|
+
## Images
|
|
99
|
+
|
|
100
|
+
| Constant | Image | Size | Use when |
|
|
101
|
+
|---|---|---|---|
|
|
102
|
+
| `CUDA_BASE` | `nvcr.io/nvidia/cuda:12.1.0-base-ubuntu22.04` | ~0.2 GB | Quick checks, no PyTorch |
|
|
103
|
+
| `PYTORCH_IMAGE` | `pytorch/pytorch:2.3.1-cuda12.1-cudnn8-runtime` | ~3.8 GB | Most ML jobs |
|
|
104
|
+
| `PYTORCH_NGC` | `nvcr.io/nvidia/pytorch:24.01-py3` | ~14 GB | Full NGC stack |
|
|
105
|
+
|
|
106
|
+
## Authentication
|
|
107
|
+
|
|
108
|
+
The SDK reads credentials from the Mimiry CLI automatically. Run once before using the SDK:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
mimiry auth login
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Tokens expire in ~5–8 minutes but are refreshed automatically by the SDK.
|
|
115
|
+
|
|
116
|
+
## License
|
|
117
|
+
|
|
118
|
+
MIT
|
mimiry-0.1.0/README.md
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# Mimiry Python SDK
|
|
2
|
+
|
|
3
|
+
Run GPU jobs on [Mimiry](https://mimiry.com) from Python — no shell scripts, no manual API calls.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install mimiry
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
## Requirements
|
|
10
|
+
|
|
11
|
+
- Python 3.10 or newer
|
|
12
|
+
- [Mimiry CLI](https://mimiry.com) installed and authenticated (`mimiry auth login`)
|
|
13
|
+
|
|
14
|
+
## Quick start
|
|
15
|
+
|
|
16
|
+
**Run a command on a GPU and get the output:**
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
from mimiry import MimiryClient
|
|
20
|
+
|
|
21
|
+
client = MimiryClient()
|
|
22
|
+
result = client.run("nvidia-smi", verbose=True)
|
|
23
|
+
result.print_logs()
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
**Run a Python script on a GPU:**
|
|
27
|
+
|
|
28
|
+
```python
|
|
29
|
+
from mimiry import MimiryClient, PYTORCH_IMAGE
|
|
30
|
+
|
|
31
|
+
client = MimiryClient()
|
|
32
|
+
result = client.run_script("train.py", image=PYTORCH_IMAGE, verbose=True)
|
|
33
|
+
result.print_logs()
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
**Or pass an inline script directly:**
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
result = client.run_script(
|
|
40
|
+
"""
|
|
41
|
+
import torch
|
|
42
|
+
print(torch.cuda.get_device_name(0))
|
|
43
|
+
""",
|
|
44
|
+
image=PYTORCH_IMAGE,
|
|
45
|
+
)
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
**Check your balance and quota:**
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
print(client.balance())
|
|
52
|
+
print(client.quota())
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
**Clean up all running sessions:**
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
client.kill_all()
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## How it works
|
|
62
|
+
|
|
63
|
+
Each `client.run()` call:
|
|
64
|
+
|
|
65
|
+
1. Creates a session via the Mimiry Sessions API
|
|
66
|
+
2. Polls until the VM reaches `running` status (~30–55 s)
|
|
67
|
+
3. Waits for Docker and the GPU driver to finish loading (~60–90 s)
|
|
68
|
+
4. Fetches the command output from the platform log endpoint
|
|
69
|
+
5. Terminates the session automatically
|
|
70
|
+
|
|
71
|
+
Sessions are always terminated after the job finishes. If a script crashes before cleanup, run `client.kill_all()`.
|
|
72
|
+
|
|
73
|
+
## Images
|
|
74
|
+
|
|
75
|
+
| Constant | Image | Size | Use when |
|
|
76
|
+
|---|---|---|---|
|
|
77
|
+
| `CUDA_BASE` | `nvcr.io/nvidia/cuda:12.1.0-base-ubuntu22.04` | ~0.2 GB | Quick checks, no PyTorch |
|
|
78
|
+
| `PYTORCH_IMAGE` | `pytorch/pytorch:2.3.1-cuda12.1-cudnn8-runtime` | ~3.8 GB | Most ML jobs |
|
|
79
|
+
| `PYTORCH_NGC` | `nvcr.io/nvidia/pytorch:24.01-py3` | ~14 GB | Full NGC stack |
|
|
80
|
+
|
|
81
|
+
## Authentication
|
|
82
|
+
|
|
83
|
+
The SDK reads credentials from the Mimiry CLI automatically. Run once before using the SDK:
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
mimiry auth login
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Tokens expire in ~5–8 minutes but are refreshed automatically by the SDK.
|
|
90
|
+
|
|
91
|
+
## License
|
|
92
|
+
|
|
93
|
+
MIT
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Mimiry Python SDK
|
|
3
|
+
=================
|
|
4
|
+
|
|
5
|
+
Simple, high-level client for the Mimiry compute platform.
|
|
6
|
+
|
|
7
|
+
Quick start::
|
|
8
|
+
|
|
9
|
+
from mimiry import MimiryClient
|
|
10
|
+
|
|
11
|
+
client = MimiryClient()
|
|
12
|
+
|
|
13
|
+
# Run a command and get the output
|
|
14
|
+
result = client.run("nvidia-smi")
|
|
15
|
+
result.print_logs()
|
|
16
|
+
|
|
17
|
+
# Run a Python script
|
|
18
|
+
result = client.run_script("train.py", verbose=True)
|
|
19
|
+
|
|
20
|
+
# Create a session for SSH access
|
|
21
|
+
session = client.create_session(auto_terminate=False)
|
|
22
|
+
session.wait_until_running()
|
|
23
|
+
print(session.ssh_command())
|
|
24
|
+
|
|
25
|
+
Authentication:
|
|
26
|
+
Run `mimiry auth login` once before using this SDK.
|
|
27
|
+
Tokens are refreshed automatically.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from .client import (
|
|
31
|
+
CUDA_BASE,
|
|
32
|
+
PYTORCH_IMAGE,
|
|
33
|
+
PYTORCH_NGC,
|
|
34
|
+
MimiryClient,
|
|
35
|
+
RunResult,
|
|
36
|
+
)
|
|
37
|
+
from ._exceptions import AuthError, LogsError, MimiryError, QuotaError, SessionError
|
|
38
|
+
from ._session import Session
|
|
39
|
+
|
|
40
|
+
__all__ = [
|
|
41
|
+
"MimiryClient",
|
|
42
|
+
"RunResult",
|
|
43
|
+
"Session",
|
|
44
|
+
"CUDA_BASE",
|
|
45
|
+
"PYTORCH_IMAGE",
|
|
46
|
+
"PYTORCH_NGC",
|
|
47
|
+
"MimiryError",
|
|
48
|
+
"AuthError",
|
|
49
|
+
"QuotaError",
|
|
50
|
+
"SessionError",
|
|
51
|
+
"LogsError",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Token management for the Mimiry SDK.
|
|
3
|
+
|
|
4
|
+
Platform note: JWTs issued by `mimiry auth token --refresh` expire in
|
|
5
|
+
approximately 5-8 minutes. Any API call after expiry returns 401, and
|
|
6
|
+
cleanup DELETE calls silently fail, leaving sessions running and billing
|
|
7
|
+
the account. This manager refreshes proactively every 4 minutes.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import subprocess
|
|
12
|
+
import time
|
|
13
|
+
|
|
14
|
+
from ._exceptions import AuthError
|
|
15
|
+
|
|
16
|
+
# Refresh well before the ~5-8 min expiry window
|
|
17
|
+
_REFRESH_INTERVAL = 240 # 4 minutes
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TokenManager:
|
|
21
|
+
"""
|
|
22
|
+
Retrieves and caches a Mimiry JWT, auto-refreshing before expiry.
|
|
23
|
+
|
|
24
|
+
Calls `mimiry auth token --refresh --json` under the hood.
|
|
25
|
+
Run `mimiry auth login` once before using the SDK.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
def __init__(self):
|
|
29
|
+
self._token: str | None = None
|
|
30
|
+
self._fetched_at: float = 0.0
|
|
31
|
+
|
|
32
|
+
def get(self, force: bool = False) -> str:
|
|
33
|
+
"""Return a valid token, refreshing from the CLI if needed."""
|
|
34
|
+
age = time.time() - self._fetched_at
|
|
35
|
+
if force or not self._token or age >= _REFRESH_INTERVAL:
|
|
36
|
+
self._token = self._fetch()
|
|
37
|
+
self._fetched_at = time.time()
|
|
38
|
+
return self._token
|
|
39
|
+
|
|
40
|
+
def _fetch(self) -> str:
|
|
41
|
+
result = subprocess.run(
|
|
42
|
+
["mimiry", "auth", "token", "--refresh", "--json"],
|
|
43
|
+
capture_output=True,
|
|
44
|
+
text=True,
|
|
45
|
+
timeout=30,
|
|
46
|
+
)
|
|
47
|
+
if result.returncode != 0:
|
|
48
|
+
raise AuthError(
|
|
49
|
+
"Failed to get auth token. Run: mimiry auth login\n"
|
|
50
|
+
f"stderr: {result.stderr.strip()}"
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
# Try JSON response first (CLI >= 1.1)
|
|
54
|
+
try:
|
|
55
|
+
data = json.loads(result.stdout)
|
|
56
|
+
token = data.get("access_token") or data.get("token")
|
|
57
|
+
if token:
|
|
58
|
+
return token
|
|
59
|
+
except json.JSONDecodeError:
|
|
60
|
+
pass
|
|
61
|
+
|
|
62
|
+
# Fallback: older CLI versions print a bare JWT line
|
|
63
|
+
for line in result.stdout.splitlines():
|
|
64
|
+
line = line.strip()
|
|
65
|
+
if line.startswith("eyJ"):
|
|
66
|
+
return line
|
|
67
|
+
|
|
68
|
+
raise AuthError(
|
|
69
|
+
f"Could not parse token from CLI output: {result.stdout[:300]!r}"
|
|
70
|
+
)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Mimiry SDK exception hierarchy.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class MimiryError(Exception):
|
|
7
|
+
"""Base exception for all Mimiry SDK errors."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class AuthError(MimiryError):
|
|
11
|
+
"""
|
|
12
|
+
Authentication failed or token could not be retrieved.
|
|
13
|
+
Fix: run `mimiry auth login` then retry.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class QuotaError(MimiryError):
|
|
18
|
+
"""
|
|
19
|
+
Account quota exceeded or insufficient credits.
|
|
20
|
+
Check balance with client.balance() and active sessions with client.sessions().
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class SessionError(MimiryError):
|
|
25
|
+
"""
|
|
26
|
+
Session creation, provisioning, or execution failed.
|
|
27
|
+
The session ID is available on the exception when applicable.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
def __init__(self, message, session_id=None):
|
|
31
|
+
super().__init__(message)
|
|
32
|
+
self.session_id = session_id
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class LogsError(MimiryError):
|
|
36
|
+
"""
|
|
37
|
+
Failed to retrieve session logs after all retries.
|
|
38
|
+
This typically means the VM setup took longer than expected.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class _HttpError(Exception):
|
|
43
|
+
"""Internal: raised for non-2xx HTTP responses."""
|
|
44
|
+
|
|
45
|
+
def __init__(self, status, body):
|
|
46
|
+
self.status = status
|
|
47
|
+
self.body = body
|
|
48
|
+
super().__init__(f"HTTP {status}: {body}")
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Session resource — wraps a single Mimiry compute session.
|
|
3
|
+
|
|
4
|
+
Platform quirks handled here (transparent to callers):
|
|
5
|
+
- status='running' means VM SSH is up, NOT Docker/GPU ready (Bug 2)
|
|
6
|
+
- /logs returns 503 'vm_setup_in_progress' for ~60-90s after 'running'
|
|
7
|
+
- Token is auto-refreshed before every network call
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import time
|
|
11
|
+
|
|
12
|
+
from ._exceptions import LogsError, SessionError, _HttpError
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Session:
|
|
16
|
+
"""
|
|
17
|
+
A Mimiry compute session.
|
|
18
|
+
|
|
19
|
+
Returned by MimiryClient.create_session() and MimiryClient.run().
|
|
20
|
+
Do not construct directly.
|
|
21
|
+
|
|
22
|
+
Attributes:
|
|
23
|
+
id : Session UUID.
|
|
24
|
+
status : Last-known status string (call refresh() to update).
|
|
25
|
+
host : SSH hostname (None until provisioning completes).
|
|
26
|
+
port : SSH port (default 22).
|
|
27
|
+
username : SSH username (default 'ubuntu').
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
def __init__(self, session_id: str, client):
|
|
31
|
+
self.id = session_id
|
|
32
|
+
self._client = client
|
|
33
|
+
self.status = "pending"
|
|
34
|
+
self.host: str | None = None
|
|
35
|
+
self.port: int = 22
|
|
36
|
+
self.username: str = "ubuntu"
|
|
37
|
+
|
|
38
|
+
# ── State ──────────────────────────────────────────────────────────────────
|
|
39
|
+
|
|
40
|
+
def refresh(self) -> "Session":
|
|
41
|
+
"""Fetch the latest session state from the API."""
|
|
42
|
+
data = self._client._get(f"/sessions/{self.id}")
|
|
43
|
+
self.status = data.get("status", "unknown")
|
|
44
|
+
ssh = data.get("ssh") or {}
|
|
45
|
+
if ssh.get("host"):
|
|
46
|
+
self.host = ssh["host"]
|
|
47
|
+
self.port = ssh.get("port", 22)
|
|
48
|
+
self.username = ssh.get("username", "ubuntu")
|
|
49
|
+
return self
|
|
50
|
+
|
|
51
|
+
def wait_until_running(self, timeout: int = 300, poll_interval: int = 5) -> "Session":
|
|
52
|
+
"""
|
|
53
|
+
Block until status == 'running' (VM SSH daemon is up).
|
|
54
|
+
|
|
55
|
+
Typical time: 23-55 seconds. After this returns, wait another
|
|
56
|
+
60-90 seconds before calling logs() — Docker and the GPU driver
|
|
57
|
+
are still loading.
|
|
58
|
+
|
|
59
|
+
Raises:
|
|
60
|
+
SessionError: if the session fails or the timeout is exceeded.
|
|
61
|
+
"""
|
|
62
|
+
start = time.time()
|
|
63
|
+
last_status = ""
|
|
64
|
+
|
|
65
|
+
while True:
|
|
66
|
+
elapsed = time.time() - start
|
|
67
|
+
if elapsed >= timeout:
|
|
68
|
+
raise SessionError(
|
|
69
|
+
f"Timed out after {timeout}s waiting for 'running' "
|
|
70
|
+
f"(last status: {self.status})",
|
|
71
|
+
session_id=self.id,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
self.refresh()
|
|
75
|
+
|
|
76
|
+
if self.status != last_status:
|
|
77
|
+
last_status = self.status
|
|
78
|
+
|
|
79
|
+
if self.status == "running":
|
|
80
|
+
return self
|
|
81
|
+
|
|
82
|
+
if self.status in ("completed", "done", "succeeded"):
|
|
83
|
+
# Fast session that completed before we polled
|
|
84
|
+
return self
|
|
85
|
+
|
|
86
|
+
if self.status in ("failed", "terminated", "cancelled"):
|
|
87
|
+
raise SessionError(
|
|
88
|
+
f"Session {self.id} reached '{self.status}' during provisioning",
|
|
89
|
+
session_id=self.id,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
time.sleep(poll_interval)
|
|
93
|
+
|
|
94
|
+
# ── Logs ───────────────────────────────────────────────────────────────────
|
|
95
|
+
|
|
96
|
+
def logs(self, tail: int = 500, max_retries: int = 12, initial_delay: int = 15) -> str:
|
|
97
|
+
"""
|
|
98
|
+
Fetch container logs, retrying automatically on 503.
|
|
99
|
+
|
|
100
|
+
The /logs endpoint returns 503 'vm_setup_in_progress' for ~60-90s
|
|
101
|
+
after status=running while Docker and the GPU driver finish loading.
|
|
102
|
+
This is handled transparently with exponential backoff.
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
Log text as a string. Empty string if the container produced
|
|
106
|
+
no output (normal for sessions without a command field set).
|
|
107
|
+
|
|
108
|
+
Raises:
|
|
109
|
+
LogsError: if all retry attempts are exhausted.
|
|
110
|
+
"""
|
|
111
|
+
delay = initial_delay
|
|
112
|
+
|
|
113
|
+
for attempt in range(1, max_retries + 1):
|
|
114
|
+
self._client._tokens.get() # proactive token refresh
|
|
115
|
+
|
|
116
|
+
try:
|
|
117
|
+
data = self._client._get(
|
|
118
|
+
f"/sessions/{self.id}/logs?tail={tail}×tamps=false"
|
|
119
|
+
)
|
|
120
|
+
return data.get("logs") or ""
|
|
121
|
+
|
|
122
|
+
except _HttpError as exc:
|
|
123
|
+
if exc.status == 503:
|
|
124
|
+
retry_after = (exc.body or {}).get("retry_after_seconds", delay)
|
|
125
|
+
time.sleep(retry_after)
|
|
126
|
+
delay = min(delay * 2, 60)
|
|
127
|
+
continue
|
|
128
|
+
if exc.status == 409:
|
|
129
|
+
# Session not in a loggable state — may have already exited cleanly
|
|
130
|
+
return ""
|
|
131
|
+
raise LogsError(
|
|
132
|
+
f"Unexpected HTTP {exc.status} fetching logs "
|
|
133
|
+
f"for session {self.id}: {exc.body}"
|
|
134
|
+
) from exc
|
|
135
|
+
|
|
136
|
+
raise LogsError(
|
|
137
|
+
f"Could not fetch logs for session {self.id} "
|
|
138
|
+
f"after {max_retries} attempts (VM setup still in progress)"
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
def poll_logs(
|
|
142
|
+
self,
|
|
143
|
+
tail: int = 1000,
|
|
144
|
+
poll_interval: int = 30,
|
|
145
|
+
timeout: int = 2400,
|
|
146
|
+
stop_pattern: str | None = None,
|
|
147
|
+
on_output=None,
|
|
148
|
+
) -> str:
|
|
149
|
+
"""
|
|
150
|
+
Poll logs repeatedly until the session ends or a stop_pattern matches.
|
|
151
|
+
|
|
152
|
+
Designed for long-running jobs (e.g. ML training) where you want to
|
|
153
|
+
see progress as it happens. Token is refreshed automatically.
|
|
154
|
+
|
|
155
|
+
Args:
|
|
156
|
+
tail: Number of log lines to fetch per poll.
|
|
157
|
+
poll_interval: Seconds between polls once logs are flowing.
|
|
158
|
+
timeout: Hard stop in seconds.
|
|
159
|
+
stop_pattern: Stop when this string appears in the logs.
|
|
160
|
+
on_output: Optional callable(new_text) — called with each
|
|
161
|
+
new chunk of log content.
|
|
162
|
+
|
|
163
|
+
Returns:
|
|
164
|
+
The final complete log string (all lines since session start).
|
|
165
|
+
"""
|
|
166
|
+
start = time.time()
|
|
167
|
+
last_log = ""
|
|
168
|
+
|
|
169
|
+
while True:
|
|
170
|
+
elapsed = time.time() - start
|
|
171
|
+
if elapsed >= timeout:
|
|
172
|
+
break
|
|
173
|
+
|
|
174
|
+
self._client._tokens.get()
|
|
175
|
+
|
|
176
|
+
try:
|
|
177
|
+
text = self.logs(tail=tail)
|
|
178
|
+
except LogsError:
|
|
179
|
+
break
|
|
180
|
+
|
|
181
|
+
if text and text != last_log:
|
|
182
|
+
new_content = text[len(last_log):]
|
|
183
|
+
last_log = text
|
|
184
|
+
if on_output and new_content.strip():
|
|
185
|
+
on_output(new_content)
|
|
186
|
+
|
|
187
|
+
if stop_pattern and last_log and stop_pattern in last_log:
|
|
188
|
+
break
|
|
189
|
+
|
|
190
|
+
self.refresh()
|
|
191
|
+
if self.status in ("terminated", "completed", "done", "failed"):
|
|
192
|
+
# One final log fetch to capture anything written right before exit
|
|
193
|
+
try:
|
|
194
|
+
final = self.logs(tail=tail)
|
|
195
|
+
if final and final != last_log:
|
|
196
|
+
new_content = final[len(last_log):]
|
|
197
|
+
last_log = final
|
|
198
|
+
if on_output and new_content.strip():
|
|
199
|
+
on_output(new_content)
|
|
200
|
+
except LogsError:
|
|
201
|
+
pass
|
|
202
|
+
break
|
|
203
|
+
|
|
204
|
+
time.sleep(poll_interval)
|
|
205
|
+
|
|
206
|
+
return last_log
|
|
207
|
+
|
|
208
|
+
# ── SSH ────────────────────────────────────────────────────────────────────
|
|
209
|
+
|
|
210
|
+
def ssh_command(self, key_path: str = "~/.ssh/mimiry_api") -> str:
|
|
211
|
+
"""
|
|
212
|
+
Return the SSH command string to connect to this session.
|
|
213
|
+
|
|
214
|
+
Note: 'running' status means SSH is up, but the GPU driver takes
|
|
215
|
+
another ~60-90s to load. Connecting too early will show
|
|
216
|
+
'NVIDIA-SMI has failed' — this is normal, just wait and retry.
|
|
217
|
+
|
|
218
|
+
Raises:
|
|
219
|
+
SessionError: if the session has no SSH host assigned.
|
|
220
|
+
"""
|
|
221
|
+
if not self.host:
|
|
222
|
+
self.refresh()
|
|
223
|
+
if not self.host:
|
|
224
|
+
raise SessionError(
|
|
225
|
+
f"Session {self.id} has no SSH host "
|
|
226
|
+
f"(status: {self.status}). "
|
|
227
|
+
"Did you set ssh_enabled=True when creating the session?",
|
|
228
|
+
session_id=self.id,
|
|
229
|
+
)
|
|
230
|
+
return (
|
|
231
|
+
f"ssh -i {key_path} "
|
|
232
|
+
f"-o StrictHostKeyChecking=no "
|
|
233
|
+
f"-p {self.port} "
|
|
234
|
+
f"{self.username}@{self.host}"
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
# ── Lifecycle ──────────────────────────────────────────────────────────────
|
|
238
|
+
|
|
239
|
+
def terminate(self) -> None:
|
|
240
|
+
"""
|
|
241
|
+
Terminate this session (DELETE /sessions/{id}).
|
|
242
|
+
|
|
243
|
+
Always call this when done — auto_terminate may not fire reliably
|
|
244
|
+
when background processes keep the host alive. Silently ignores
|
|
245
|
+
errors (e.g. session already terminated).
|
|
246
|
+
"""
|
|
247
|
+
try:
|
|
248
|
+
self._client._delete(f"/sessions/{self.id}")
|
|
249
|
+
except Exception:
|
|
250
|
+
pass
|
|
251
|
+
self.status = "terminated"
|
|
252
|
+
|
|
253
|
+
def __repr__(self) -> str:
|
|
254
|
+
return (
|
|
255
|
+
f"Session(id={self.id!r}, status={self.status!r}, host={self.host!r})"
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
def __enter__(self) -> "Session":
|
|
259
|
+
return self
|
|
260
|
+
|
|
261
|
+
def __exit__(self, *_) -> None:
|
|
262
|
+
self.terminate()
|