runara 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runara-0.1.0/PKG-INFO +115 -0
- runara-0.1.0/README.md +95 -0
- runara-0.1.0/pyproject.toml +50 -0
- runara-0.1.0/runara/__init__.py +39 -0
- runara-0.1.0/runara/_futures.py +64 -0
- runara-0.1.0/runara/_http.py +155 -0
- runara-0.1.0/runara/client.py +286 -0
- runara-0.1.0/runara/types.py +111 -0
- runara-0.1.0/runara.egg-info/PKG-INFO +115 -0
- runara-0.1.0/runara.egg-info/SOURCES.txt +18 -0
- runara-0.1.0/runara.egg-info/dependency_links.txt +1 -0
- runara-0.1.0/runara.egg-info/requires.txt +6 -0
- runara-0.1.0/runara.egg-info/top_level.txt +2 -0
- runara-0.1.0/runara_cookbook/__init__.py +11 -0
- runara-0.1.0/runara_cookbook/data.py +88 -0
- runara-0.1.0/runara_cookbook/renderers/__init__.py +60 -0
- runara-0.1.0/runara_cookbook/renderers/base.py +203 -0
- runara-0.1.0/setup.cfg +4 -0
- runara-0.1.0/tests/test_client.py +343 -0
- runara-0.1.0/tests/test_cookbook.py +189 -0
runara-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: runara
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Fine-tune open models on Runara. You write the training loop.
|
|
5
|
+
Author: Runara
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://runara.ai
|
|
8
|
+
Project-URL: Source, https://github.com/Runaraai/runara-python
|
|
9
|
+
Keywords: llm,fine-tuning,lora,training
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
14
|
+
Requires-Python: >=3.10
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
Provides-Extra: cookbook
|
|
17
|
+
Requires-Dist: transformers>=4.45; extra == "cookbook"
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
20
|
+
|
|
21
|
+
# runara
|
|
22
|
+
|
|
23
|
+
Fine-tune open models without hosting them. You write the training loop; we
|
|
24
|
+
hold the base weights and run the forward and backward passes.
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
pip install runara runara-cookbook
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
import runara
|
|
32
|
+
from runara_cookbook import renderers
|
|
33
|
+
|
|
34
|
+
sc = runara.ServiceClient(api_key="rk_live_...")
|
|
35
|
+
tc = sc.create_lora_training_client(base_model="qwen3.5-4b", rank=32)
|
|
36
|
+
|
|
37
|
+
renderer = renderers.get("qwen3_5", tokenizer=tc.get_tokenizer())
|
|
38
|
+
rows = [json.loads(line) for line in open("my_tickets.jsonl")] # your disk
|
|
39
|
+
|
|
40
|
+
for epoch in range(3):
|
|
41
|
+
for chunk in batches(rows, 8):
|
|
42
|
+
data = [renderer.to_datum(row["messages"]) for row in chunk]
|
|
43
|
+
fb = tc.forward_backward(data) # submitted, not awaited
|
|
44
|
+
tc.optim_step(runara.AdamParams(1e-4)) # submitted straight after
|
|
45
|
+
print(fb.result().loss) # the loop blocks here
|
|
46
|
+
|
|
47
|
+
tc.save_state(name="support-v1")
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Your data does not leave your machine
|
|
51
|
+
|
|
52
|
+
`open(...)` runs on your computer. The cookbook tokenizes locally and builds a
|
|
53
|
+
per-token loss mask, so what crosses the wire is a list of integers and a list
|
|
54
|
+
of floats — no text, no filenames, no schema. A two-turn conversation is about
|
|
55
|
+
900 bytes of that.
|
|
56
|
+
|
|
57
|
+
You can check this yourself before sending anything:
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
datum = renderer.to_datum(messages)
|
|
61
|
+
print(datum.token_ids) # integers
|
|
62
|
+
print(datum.weights) # 0.0 for the prompt, 1.0 for what you train on
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
What we do keep is the adapter you write with `save_state`, plus token counts
|
|
66
|
+
and loss values for billing and the charts in the console.
|
|
67
|
+
|
|
68
|
+
## Why the loop is yours
|
|
69
|
+
|
|
70
|
+
There is no job to submit and no config to fill in. You decide the batching,
|
|
71
|
+
the schedule, when to stop, and what the loss means. That is more work than a
|
|
72
|
+
managed API and it is the point: it is the difference between tuning a model
|
|
73
|
+
and filling in a form.
|
|
74
|
+
|
|
75
|
+
`forward_backward` and `optim_step` both return immediately. Submit them back
|
|
76
|
+
to back and collect later — the worker applies calls for a session in the
|
|
77
|
+
order they arrive, so the round trip overlaps your next batch of tokenization
|
|
78
|
+
instead of stalling the GPU.
|
|
79
|
+
|
|
80
|
+
## What you get out
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
for checkpoint in sc.list_checkpoints():
|
|
84
|
+
print(checkpoint.name, checkpoint.size_bytes, checkpoint.base_model)
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Checkpoints are standard PEFT adapters — `adapter_config.json` and
|
|
88
|
+
`adapter_model.safetensors` — so they load in vLLM, SGLang or transformers
|
|
89
|
+
without conversion. Download them from the console, or serve them on Runara
|
|
90
|
+
alongside the base model.
|
|
91
|
+
|
|
92
|
+
## Errors worth catching
|
|
93
|
+
|
|
94
|
+
| | |
|
|
95
|
+
|---|---|
|
|
96
|
+
| `runara.InsufficientBalance` | The organization is out of credit. |
|
|
97
|
+
| `runara.SessionClosed` | The session ended, or is ending because the balance ran out. |
|
|
98
|
+
| `runara.RunaraError` | Everything else, with `.status` and `.code`. |
|
|
99
|
+
|
|
100
|
+
A training call is never retried automatically. One that timed out may already
|
|
101
|
+
have applied its gradients, and sending it again would train on the same batch
|
|
102
|
+
twice.
|
|
103
|
+
|
|
104
|
+
## Checked against the reference
|
|
105
|
+
|
|
106
|
+
The renderers are verified to produce byte-identical tokens, targets and loss
|
|
107
|
+
masks to `tinker-cookbook` for the same conversation and tokenizer. Chat
|
|
108
|
+
templating is the part of fine-tuning that goes wrong silently, so it is
|
|
109
|
+
checked against an independent implementation rather than only against itself.
|
|
110
|
+
|
|
111
|
+
## Installing
|
|
112
|
+
|
|
113
|
+
The client has no dependencies at all — it is stdlib, so installing it cannot
|
|
114
|
+
conflict with whatever `torch`, `transformers` or `numpy` versions your
|
|
115
|
+
training code already pins. The heavy dependencies live in `runara-cookbook`.
|
runara-0.1.0/README.md
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# runara
|
|
2
|
+
|
|
3
|
+
Fine-tune open models without hosting them. You write the training loop; we
|
|
4
|
+
hold the base weights and run the forward and backward passes.
|
|
5
|
+
|
|
6
|
+
```bash
|
|
7
|
+
pip install runara runara-cookbook
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
```python
|
|
11
|
+
import runara
|
|
12
|
+
from runara_cookbook import renderers
|
|
13
|
+
|
|
14
|
+
sc = runara.ServiceClient(api_key="rk_live_...")
|
|
15
|
+
tc = sc.create_lora_training_client(base_model="qwen3.5-4b", rank=32)
|
|
16
|
+
|
|
17
|
+
renderer = renderers.get("qwen3_5", tokenizer=tc.get_tokenizer())
|
|
18
|
+
rows = [json.loads(line) for line in open("my_tickets.jsonl")] # your disk
|
|
19
|
+
|
|
20
|
+
for epoch in range(3):
|
|
21
|
+
for chunk in batches(rows, 8):
|
|
22
|
+
data = [renderer.to_datum(row["messages"]) for row in chunk]
|
|
23
|
+
fb = tc.forward_backward(data) # submitted, not awaited
|
|
24
|
+
tc.optim_step(runara.AdamParams(1e-4)) # submitted straight after
|
|
25
|
+
print(fb.result().loss) # the loop blocks here
|
|
26
|
+
|
|
27
|
+
tc.save_state(name="support-v1")
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Your data does not leave your machine
|
|
31
|
+
|
|
32
|
+
`open(...)` runs on your computer. The cookbook tokenizes locally and builds a
|
|
33
|
+
per-token loss mask, so what crosses the wire is a list of integers and a list
|
|
34
|
+
of floats — no text, no filenames, no schema. A two-turn conversation is about
|
|
35
|
+
900 bytes of that.
|
|
36
|
+
|
|
37
|
+
You can check this yourself before sending anything:
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
datum = renderer.to_datum(messages)
|
|
41
|
+
print(datum.token_ids) # integers
|
|
42
|
+
print(datum.weights) # 0.0 for the prompt, 1.0 for what you train on
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
What we do keep is the adapter you write with `save_state`, plus token counts
|
|
46
|
+
and loss values for billing and the charts in the console.
|
|
47
|
+
|
|
48
|
+
## Why the loop is yours
|
|
49
|
+
|
|
50
|
+
There is no job to submit and no config to fill in. You decide the batching,
|
|
51
|
+
the schedule, when to stop, and what the loss means. That is more work than a
|
|
52
|
+
managed API and it is the point: it is the difference between tuning a model
|
|
53
|
+
and filling in a form.
|
|
54
|
+
|
|
55
|
+
`forward_backward` and `optim_step` both return immediately. Submit them back
|
|
56
|
+
to back and collect later — the worker applies calls for a session in the
|
|
57
|
+
order they arrive, so the round trip overlaps your next batch of tokenization
|
|
58
|
+
instead of stalling the GPU.
|
|
59
|
+
|
|
60
|
+
## What you get out
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
for checkpoint in sc.list_checkpoints():
|
|
64
|
+
print(checkpoint.name, checkpoint.size_bytes, checkpoint.base_model)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Checkpoints are standard PEFT adapters — `adapter_config.json` and
|
|
68
|
+
`adapter_model.safetensors` — so they load in vLLM, SGLang or transformers
|
|
69
|
+
without conversion. Download them from the console, or serve them on Runara
|
|
70
|
+
alongside the base model.
|
|
71
|
+
|
|
72
|
+
## Errors worth catching
|
|
73
|
+
|
|
74
|
+
| | |
|
|
75
|
+
|---|---|
|
|
76
|
+
| `runara.InsufficientBalance` | The organization is out of credit. |
|
|
77
|
+
| `runara.SessionClosed` | The session ended, or is ending because the balance ran out. |
|
|
78
|
+
| `runara.RunaraError` | Everything else, with `.status` and `.code`. |
|
|
79
|
+
|
|
80
|
+
A training call is never retried automatically. One that timed out may already
|
|
81
|
+
have applied its gradients, and sending it again would train on the same batch
|
|
82
|
+
twice.
|
|
83
|
+
|
|
84
|
+
## Checked against the reference
|
|
85
|
+
|
|
86
|
+
The renderers are verified to produce byte-identical tokens, targets and loss
|
|
87
|
+
masks to `tinker-cookbook` for the same conversation and tokenizer. Chat
|
|
88
|
+
templating is the part of fine-tuning that goes wrong silently, so it is
|
|
89
|
+
checked against an independent implementation rather than only against itself.
|
|
90
|
+
|
|
91
|
+
## Installing
|
|
92
|
+
|
|
93
|
+
The client has no dependencies at all — it is stdlib, so installing it cannot
|
|
94
|
+
conflict with whatever `torch`, `transformers` or `numpy` versions your
|
|
95
|
+
training code already pins. The heavy dependencies live in `runara-cookbook`.
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Without this, the package cannot be built as a wheel at all — pip falls
|
|
2
|
+
# back to legacy behaviour and `python -m build` refuses outright.
|
|
3
|
+
[build-system]
|
|
4
|
+
requires = ["setuptools>=68"]
|
|
5
|
+
build-backend = "setuptools.build_meta"
|
|
6
|
+
|
|
7
|
+
[project]
|
|
8
|
+
name = "runara"
|
|
9
|
+
version = "0.1.0"
|
|
10
|
+
description = "Fine-tune open models on Runara. You write the training loop."
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
readme = "README.md"
|
|
13
|
+
license = "Apache-2.0"
|
|
14
|
+
authors = [{ name = "Runara" }]
|
|
15
|
+
keywords = ["llm", "fine-tuning", "lora", "training"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 3 - Alpha",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
21
|
+
]
|
|
22
|
+
# No runtime dependencies. The client is stdlib so that installing it cannot
|
|
23
|
+
# conflict with whatever torch, transformers or numpy versions your training
|
|
24
|
+
# code already pins — the cookbook is where the heavy dependencies live.
|
|
25
|
+
dependencies = []
|
|
26
|
+
|
|
27
|
+
[project.optional-dependencies]
|
|
28
|
+
# The cookbook needs a tokenizer; the client deliberately does not.
|
|
29
|
+
cookbook = ["transformers>=4.45"]
|
|
30
|
+
dev = ["pytest>=8.0"]
|
|
31
|
+
|
|
32
|
+
# Both are shipped. The cookbook is a module inside this distribution rather
|
|
33
|
+
# than a separate one, so it installs as runara[cookbook] — "pip install
|
|
34
|
+
# runara-cookbook" names a package that does not and will not exist.
|
|
35
|
+
# Explicitly every package, subpackages included. An explicit list silently
|
|
36
|
+
# omits what it does not name: "packages = [runara, runara_cookbook]" built a
|
|
37
|
+
# wheel without runara_cookbook.renderers, so "from runara_cookbook import
|
|
38
|
+
# renderers" — the first line of the quickstart — would have failed on a
|
|
39
|
+
# version number that can never be re-uploaded.
|
|
40
|
+
[tool.setuptools.packages.find]
|
|
41
|
+
include = ["runara*", "runara_cookbook*"]
|
|
42
|
+
namespaces = false
|
|
43
|
+
|
|
44
|
+
[project.urls]
|
|
45
|
+
Homepage = "https://runara.ai"
|
|
46
|
+
Source = "https://github.com/Runaraai/runara-python"
|
|
47
|
+
|
|
48
|
+
[tool.pytest.ini_options]
|
|
49
|
+
testpaths = ["tests"]
|
|
50
|
+
pythonpath = [".", "tests"]
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Fine-tune open models on Runara. You write the training loop; we hold the weights.
|
|
2
|
+
|
|
3
|
+
import runara
|
|
4
|
+
|
|
5
|
+
sc = runara.ServiceClient(api_key="rk_live_...")
|
|
6
|
+
tc = sc.create_lora_training_client(base_model="qwen3.5-4b", rank=32)
|
|
7
|
+
|
|
8
|
+
Your data stays on your machine. Only token ids and per-token loss weights
|
|
9
|
+
cross the wire — see `runara_cookbook` for turning your files into those.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from ._futures import Future
|
|
13
|
+
from ._http import InsufficientBalance, RunaraError, SessionClosed
|
|
14
|
+
from .client import ServiceClient, TrainingClient
|
|
15
|
+
from .types import (
|
|
16
|
+
AdamParams,
|
|
17
|
+
Checkpoint,
|
|
18
|
+
Datum,
|
|
19
|
+
ForwardBackwardResult,
|
|
20
|
+
ModelInfo,
|
|
21
|
+
SampleResult,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
__version__ = "0.1.0"
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"AdamParams",
|
|
28
|
+
"Checkpoint",
|
|
29
|
+
"Datum",
|
|
30
|
+
"ForwardBackwardResult",
|
|
31
|
+
"Future",
|
|
32
|
+
"InsufficientBalance",
|
|
33
|
+
"ModelInfo",
|
|
34
|
+
"RunaraError",
|
|
35
|
+
"SampleResult",
|
|
36
|
+
"ServiceClient",
|
|
37
|
+
"SessionClosed",
|
|
38
|
+
"TrainingClient",
|
|
39
|
+
]
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Futures that let a training loop stay ahead of the network.
|
|
2
|
+
|
|
3
|
+
A step is two calls — ``forward_backward`` then ``optim_step`` — and a loop
|
|
4
|
+
that awaits each one spends most of its wall-clock time waiting for a round
|
|
5
|
+
trip while the GPU sits idle. The worker already guarantees that calls for one
|
|
6
|
+
session are applied in the order they arrive, so the client can submit both and
|
|
7
|
+
collect later.
|
|
8
|
+
|
|
9
|
+
That ordering guarantee is what makes this safe, and it is the worker's
|
|
10
|
+
per-session lock that provides it. Without it, two in-flight calls would race
|
|
11
|
+
on the same adapter and the optimizer could apply a step before the gradients
|
|
12
|
+
it was meant to apply.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from concurrent.futures import Future as _Future
|
|
18
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
19
|
+
from typing import Callable, Generic, TypeVar
|
|
20
|
+
|
|
21
|
+
T = TypeVar("T")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Future(Generic[T]):
|
|
25
|
+
"""A submitted request. Call ``result()`` when you actually need the value.
|
|
26
|
+
|
|
27
|
+
Deliberately thin: it is one outstanding HTTP request, not a task in a
|
|
28
|
+
scheduler. There is no polling, no result store and no reconnection
|
|
29
|
+
protocol, because there is nothing to reconnect to — the request either
|
|
30
|
+
returns or raises.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
def __init__(self, inner: _Future) -> None:
|
|
34
|
+
self._inner = inner
|
|
35
|
+
|
|
36
|
+
def result(self, timeout: float | None = None) -> T:
|
|
37
|
+
"""Block until the worker answers, re-raising whatever it raised."""
|
|
38
|
+
return self._inner.result(timeout)
|
|
39
|
+
|
|
40
|
+
def done(self) -> bool:
|
|
41
|
+
return self._inner.done()
|
|
42
|
+
|
|
43
|
+
def __repr__(self) -> str: # pragma: no cover - debugging convenience
|
|
44
|
+
state = "done" if self.done() else "pending"
|
|
45
|
+
return f"<runara.Future {state}>"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class Submitter:
|
|
49
|
+
"""Runs requests for one session, one at a time, in submission order.
|
|
50
|
+
|
|
51
|
+
A single worker thread rather than a pool: the ordering the protocol
|
|
52
|
+
promises is per session, and a pool would let two requests for the same
|
|
53
|
+
session overlap. Pipelining comes from the client not blocking, not from
|
|
54
|
+
the requests running concurrently.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
def __init__(self) -> None:
|
|
58
|
+
self._pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix="runara")
|
|
59
|
+
|
|
60
|
+
def submit(self, fn: Callable[..., T], *args, **kwargs) -> Future[T]:
|
|
61
|
+
return Future(self._pool.submit(fn, *args, **kwargs))
|
|
62
|
+
|
|
63
|
+
def close(self) -> None:
|
|
64
|
+
self._pool.shutdown(wait=True)
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""HTTP, with the retry policy spelled out.
|
|
2
|
+
|
|
3
|
+
Two different endpoints with different failure characters:
|
|
4
|
+
|
|
5
|
+
* The **control plane** creates sessions and lists checkpoints. Slow, rare,
|
|
6
|
+
and safe to retry only because session creation is idempotent on a request
|
|
7
|
+
id the client generates.
|
|
8
|
+
* The **worker** runs the training calls. Fast, frequent, and **not** safe to
|
|
9
|
+
retry: a ``forward_backward`` that timed out may well have applied its
|
|
10
|
+
gradients, and sending it again would train on the same batch twice. So the
|
|
11
|
+
worker path retries only on errors that prove the request never arrived.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
import ssl
|
|
19
|
+
import tempfile
|
|
20
|
+
import time
|
|
21
|
+
import urllib.error
|
|
22
|
+
import urllib.request
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
#: Only these are retried on the worker path: the request did not land.
|
|
26
|
+
_CONNECTION_ERRORS = (urllib.error.URLError, TimeoutError, ConnectionError)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class _PinnedWorker:
|
|
30
|
+
"""A TLS context that trusts one worker's certificate and nothing else.
|
|
31
|
+
|
|
32
|
+
A training worker is rented for minutes at an address we do not own, so
|
|
33
|
+
there is no public certificate for it and no DNS name to put in one. The
|
|
34
|
+
control plane, which we do authenticate normally, tells us the exact
|
|
35
|
+
certificate the worker will present; this trusts that and only that.
|
|
36
|
+
|
|
37
|
+
That is stronger than the usual path rather than weaker. A public CA would
|
|
38
|
+
vouch for anyone who could prove control of the address; this vouches for
|
|
39
|
+
the one machine the control plane placed our session on. The alternative
|
|
40
|
+
actually on offer — ``verify=False`` — would leave the training tokens on
|
|
41
|
+
the wire readable by anyone between here and the GPU.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def __init__(self, cert_pem: str):
|
|
45
|
+
if not cert_pem.startswith("-----BEGIN CERTIFICATE-----"):
|
|
46
|
+
raise ValueError("worker certificate is not PEM")
|
|
47
|
+
# NamedTemporaryFile because SSLContext loads CAs from a path. Written
|
|
48
|
+
# 0600 and removed when the client is done with it.
|
|
49
|
+
handle = tempfile.NamedTemporaryFile(
|
|
50
|
+
"w", suffix=".pem", prefix="runara-worker-", delete=False
|
|
51
|
+
)
|
|
52
|
+
try:
|
|
53
|
+
os.chmod(handle.name, 0o600)
|
|
54
|
+
handle.write(cert_pem)
|
|
55
|
+
finally:
|
|
56
|
+
handle.close()
|
|
57
|
+
self._path = handle.name
|
|
58
|
+
|
|
59
|
+
context = ssl.create_default_context(cafile=self._path)
|
|
60
|
+
context.minimum_version = ssl.TLSVersion.TLSv1_2
|
|
61
|
+
# The certificate carries the worker's IP in its SAN, so the hostname
|
|
62
|
+
# check is a real check and stays on.
|
|
63
|
+
context.check_hostname = True
|
|
64
|
+
context.verify_mode = ssl.CERT_REQUIRED
|
|
65
|
+
self.context = context
|
|
66
|
+
|
|
67
|
+
def close(self) -> None:
|
|
68
|
+
try:
|
|
69
|
+
os.unlink(self._path)
|
|
70
|
+
except OSError:
|
|
71
|
+
pass
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class RunaraError(Exception):
|
|
75
|
+
"""An error the service reported, with its code where there is one."""
|
|
76
|
+
|
|
77
|
+
def __init__(self, message: str, *, status: int | None = None, code: str | None = None):
|
|
78
|
+
super().__init__(message)
|
|
79
|
+
self.status = status
|
|
80
|
+
self.code = code
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class InsufficientBalance(RunaraError):
|
|
84
|
+
"""The organization is out of credit. Training stops until it is topped up."""
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class SessionClosed(RunaraError):
|
|
88
|
+
"""The session has ended, or is ending because the balance ran out."""
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _raise_for(status: int, body: Any, url: str) -> None:
|
|
92
|
+
error = body.get("error") if isinstance(body, dict) else None
|
|
93
|
+
code = error.get("code") if isinstance(error, dict) else (error if isinstance(error, str) else None)
|
|
94
|
+
message = error.get("message") if isinstance(error, dict) else str(error or body)
|
|
95
|
+
|
|
96
|
+
if status == 402 or code == "INSUFFICIENT_BALANCE":
|
|
97
|
+
raise InsufficientBalance(message, status=status, code=code)
|
|
98
|
+
if code in {"session_closed", "session_closing", "session_not_resident"}:
|
|
99
|
+
raise SessionClosed(message, status=status, code=code)
|
|
100
|
+
raise RunaraError(f"{message} ({url.rsplit('/', 1)[-1]})", status=status, code=code)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def request(
|
|
104
|
+
method: str,
|
|
105
|
+
url: str,
|
|
106
|
+
token: str,
|
|
107
|
+
body: dict | None = None,
|
|
108
|
+
*,
|
|
109
|
+
timeout: float = 600.0,
|
|
110
|
+
retries: int = 0,
|
|
111
|
+
ssl_context: ssl.SSLContext | None = None,
|
|
112
|
+
) -> Any:
|
|
113
|
+
"""One request, retried only when it provably did not arrive.
|
|
114
|
+
|
|
115
|
+
`retries` defaults to zero. The caller decides, because only the caller
|
|
116
|
+
knows whether repeating the call is safe — and for a training step it is
|
|
117
|
+
not: a timed-out forward_backward may have already applied its gradients.
|
|
118
|
+
"""
|
|
119
|
+
payload = json.dumps(body).encode() if body is not None else None
|
|
120
|
+
attempt = 0
|
|
121
|
+
while True:
|
|
122
|
+
http_request = urllib.request.Request(
|
|
123
|
+
url,
|
|
124
|
+
data=payload,
|
|
125
|
+
method=method,
|
|
126
|
+
headers={
|
|
127
|
+
"Authorization": f"Bearer {token}",
|
|
128
|
+
"Content-Type": "application/json",
|
|
129
|
+
"User-Agent": "runara-python",
|
|
130
|
+
},
|
|
131
|
+
)
|
|
132
|
+
try:
|
|
133
|
+
with urllib.request.urlopen(
|
|
134
|
+
http_request, timeout=timeout, context=ssl_context
|
|
135
|
+
) as response:
|
|
136
|
+
raw = response.read()
|
|
137
|
+
return json.loads(raw) if raw else {}
|
|
138
|
+
except urllib.error.HTTPError as error:
|
|
139
|
+
try:
|
|
140
|
+
parsed = json.loads(error.read())
|
|
141
|
+
except Exception: # noqa: BLE001 - a non-JSON error body is still an error
|
|
142
|
+
parsed = {"error": {"message": error.reason}}
|
|
143
|
+
# A 503 means the worker is still loading its weights; that is a
|
|
144
|
+
# wait, not a failure, and the request never reached the model.
|
|
145
|
+
if error.code == 503 and attempt < retries:
|
|
146
|
+
attempt += 1
|
|
147
|
+
time.sleep(min(2**attempt, 10))
|
|
148
|
+
continue
|
|
149
|
+
_raise_for(error.code, parsed, url)
|
|
150
|
+
except _CONNECTION_ERRORS as error:
|
|
151
|
+
if attempt < retries:
|
|
152
|
+
attempt += 1
|
|
153
|
+
time.sleep(min(2**attempt, 10))
|
|
154
|
+
continue
|
|
155
|
+
raise RunaraError(f"Could not reach {url.rsplit('/', 2)[0]}: {error}") from error
|