cortexgrid-infer 0.3.5__tar.gz → 0.3.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/PKG-INFO +15 -11
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/README.md +13 -9
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/anthropic.py +5 -4
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/base.py +3 -3
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/gemini/base.py +4 -4
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/image2mesh.py +4 -3
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/text2image.py +3 -3
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/text2text.py +5 -3
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/text_rewriter.py +3 -3
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/pyproject.toml +4 -2
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/.gitignore +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/LICENSE +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/__init__.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/compiling.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/core.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/device.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/importers/__init__.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/importers/base.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/importers/huggingface.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/protocols/__init__.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/protocols/completion.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/protocols/imaging.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/protocols/meshing.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/protocols/rewriting.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/py.typed +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/registry.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/__init__.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/gemini/__init__.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/gemini/text2image.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/gemini/text2text.py +0 -0
- {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cortexgrid-infer
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.6
|
|
4
4
|
Summary: Serve apps per task (text-to-text, text-to-image, image-to-mesh) and importers per source (HuggingFace) for running models on cortexgrid
|
|
5
5
|
Project-URL: Homepage, https://github.com/robodatalab/cortexgrid-infer
|
|
6
6
|
Project-URL: Repository, https://github.com/robodatalab/cortexgrid-infer
|
|
@@ -9,7 +9,7 @@ License-File: LICENSE
|
|
|
9
9
|
Requires-Python: <3.12,>=3.11
|
|
10
10
|
Requires-Dist: accelerate>=0.33
|
|
11
11
|
Requires-Dist: anthropic>=0.40.0
|
|
12
|
-
Requires-Dist: cortexgrid>=0.3.
|
|
12
|
+
Requires-Dist: cortexgrid>=0.3.12
|
|
13
13
|
Requires-Dist: diffusers>=0.38
|
|
14
14
|
Requires-Dist: fastapi>=0.133
|
|
15
15
|
Requires-Dist: google-genai>=1.0
|
|
@@ -132,7 +132,7 @@ async def run():
|
|
|
132
132
|
asyncio.run(run())
|
|
133
133
|
|
|
134
134
|
# 4. Teardown.
|
|
135
|
-
cortexgrid.undeploy_model(
|
|
135
|
+
cortexgrid.undeploy_model(deployment.key) # free the GPU
|
|
136
136
|
cortexgrid.delete_model(imp.family, imp.suffix, cortexgrid.IMPORTED) # and the weights
|
|
137
137
|
```
|
|
138
138
|
|
|
@@ -179,20 +179,24 @@ beyond the weights. (A `Hosted` entry asks for none — it holds no weights.)
|
|
|
179
179
|
## Compilation
|
|
180
180
|
|
|
181
181
|
Every serve app that runs weights can run them through `torch.compile`, and none
|
|
182
|
-
does unless asked: a replica is eager unless its
|
|
183
|
-
`compile: "true"`. Each `LocalModel` writes `compile: "false"` onto the card
|
|
184
|
-
import, so turning it on is
|
|
182
|
+
does unless asked: a replica is eager unless its deployment's config carries
|
|
183
|
+
`compile: "true"`. Each `LocalModel` writes `compile: "false"` onto the model card
|
|
184
|
+
at import, so turning it on is a deployment that asks for it:
|
|
185
185
|
|
|
186
186
|
```python
|
|
187
|
-
|
|
188
|
-
cortexgrid.set_model_config(family, suffix, cortexgrid.IMPORTED, {**config, "compile": "true"})
|
|
189
|
-
cortexgrid.deploy_model(family, suffix, cortexgrid.IMPORTED, wait=True)
|
|
187
|
+
cortexgrid.deploy_model(family, suffix, cortexgrid.IMPORTED, wait=True, config={"compile": "true"})
|
|
190
188
|
```
|
|
191
189
|
|
|
190
|
+
That is a separate, compiled deployment of the same model, alongside any uncompiled
|
|
191
|
+
one: a model gets one deployment per distinct config, each at its own
|
|
192
|
+
`deployment.url`. `Text2Text`'s `enable_thinking` and `max_total_tokens` are set per
|
|
193
|
+
deployment the same way. Whatever a deployment leaves out comes from the model card,
|
|
194
|
+
so the defaults `config()` writes there at import still apply.
|
|
195
|
+
|
|
192
196
|
It is off by default because it is not free: every input shape pays a warm-up on
|
|
193
197
|
the request that first brings it, and each CUDA graph holds memory of its own. It
|
|
194
198
|
pays back where traffic keeps to a few shapes and the model is small enough for
|
|
195
|
-
launch overhead to matter. A
|
|
199
|
+
launch overhead to matter. A deployment asking for it off CUDA is served eager.
|
|
196
200
|
|
|
197
201
|
Compiled, a serve app runs `torch.compile` over the model it loads, because
|
|
198
202
|
eager PyTorch bills per *operation* and a transformer forward is mostly glue —
|
|
@@ -233,7 +237,7 @@ Each serve app is tuned rather than defaulted:
|
|
|
233
237
|
1.2 GB of keys and values re-read per generated token, several times the
|
|
234
238
|
traffic of the model's own weights, to make room for a prompt nobody sent. It
|
|
235
239
|
is pinned instead to the prompt and reply a caller plausibly sends: 4096 tokens,
|
|
236
|
-
or `max_total_tokens`
|
|
240
|
+
or `max_total_tokens` from the deployment's config, never past the model's own
|
|
237
241
|
`max_position_embeddings`. A reply takes at most half of it, so an
|
|
238
242
|
over-optimistic `max_new_tokens` cannot squeeze the prompt to nothing, and a
|
|
239
243
|
prompt that will not fit in the rest is truncated — keeping the end, since a
|
|
@@ -104,7 +104,7 @@ async def run():
|
|
|
104
104
|
asyncio.run(run())
|
|
105
105
|
|
|
106
106
|
# 4. Teardown.
|
|
107
|
-
cortexgrid.undeploy_model(
|
|
107
|
+
cortexgrid.undeploy_model(deployment.key) # free the GPU
|
|
108
108
|
cortexgrid.delete_model(imp.family, imp.suffix, cortexgrid.IMPORTED) # and the weights
|
|
109
109
|
```
|
|
110
110
|
|
|
@@ -151,20 +151,24 @@ beyond the weights. (A `Hosted` entry asks for none — it holds no weights.)
|
|
|
151
151
|
## Compilation
|
|
152
152
|
|
|
153
153
|
Every serve app that runs weights can run them through `torch.compile`, and none
|
|
154
|
-
does unless asked: a replica is eager unless its
|
|
155
|
-
`compile: "true"`. Each `LocalModel` writes `compile: "false"` onto the card
|
|
156
|
-
import, so turning it on is
|
|
154
|
+
does unless asked: a replica is eager unless its deployment's config carries
|
|
155
|
+
`compile: "true"`. Each `LocalModel` writes `compile: "false"` onto the model card
|
|
156
|
+
at import, so turning it on is a deployment that asks for it:
|
|
157
157
|
|
|
158
158
|
```python
|
|
159
|
-
|
|
160
|
-
cortexgrid.set_model_config(family, suffix, cortexgrid.IMPORTED, {**config, "compile": "true"})
|
|
161
|
-
cortexgrid.deploy_model(family, suffix, cortexgrid.IMPORTED, wait=True)
|
|
159
|
+
cortexgrid.deploy_model(family, suffix, cortexgrid.IMPORTED, wait=True, config={"compile": "true"})
|
|
162
160
|
```
|
|
163
161
|
|
|
162
|
+
That is a separate, compiled deployment of the same model, alongside any uncompiled
|
|
163
|
+
one: a model gets one deployment per distinct config, each at its own
|
|
164
|
+
`deployment.url`. `Text2Text`'s `enable_thinking` and `max_total_tokens` are set per
|
|
165
|
+
deployment the same way. Whatever a deployment leaves out comes from the model card,
|
|
166
|
+
so the defaults `config()` writes there at import still apply.
|
|
167
|
+
|
|
164
168
|
It is off by default because it is not free: every input shape pays a warm-up on
|
|
165
169
|
the request that first brings it, and each CUDA graph holds memory of its own. It
|
|
166
170
|
pays back where traffic keeps to a few shapes and the model is small enough for
|
|
167
|
-
launch overhead to matter. A
|
|
171
|
+
launch overhead to matter. A deployment asking for it off CUDA is served eager.
|
|
168
172
|
|
|
169
173
|
Compiled, a serve app runs `torch.compile` over the model it loads, because
|
|
170
174
|
eager PyTorch bills per *operation* and a transformer forward is mostly glue —
|
|
@@ -205,7 +209,7 @@ Each serve app is tuned rather than defaulted:
|
|
|
205
209
|
1.2 GB of keys and values re-read per generated token, several times the
|
|
206
210
|
traffic of the model's own weights, to make room for a prompt nobody sent. It
|
|
207
211
|
is pinned instead to the prompt and reply a caller plausibly sends: 4096 tokens,
|
|
208
|
-
or `max_total_tokens`
|
|
212
|
+
or `max_total_tokens` from the deployment's config, never past the model's own
|
|
209
213
|
`max_position_embeddings`. A reply takes at most half of it, so an
|
|
210
214
|
over-optimistic `max_new_tokens` cannot squeeze the prompt to nothing, and a
|
|
211
215
|
prompt that will not fit in the rest is truncated — keeping the end, since a
|
|
@@ -131,13 +131,14 @@ class AnthropicText2Text(HostedModel):
|
|
|
131
131
|
def client(cls, url: str, name: str) -> ServedCompletingModel:
|
|
132
132
|
return ServedCompletingModel(url=url, model_id=name)
|
|
133
133
|
|
|
134
|
-
def __init__(self,
|
|
135
|
-
config = cortexgrid.model_config(
|
|
134
|
+
def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
|
|
135
|
+
config = cortexgrid.model_config(deployment)
|
|
136
136
|
missing = {MODEL_PARAM, API_KEY_SECRET_PARAM} - config.keys()
|
|
137
137
|
if missing:
|
|
138
138
|
raise RuntimeError(
|
|
139
|
-
f"{family}/{suffix}/{run_name}
|
|
140
|
-
"its config; set them on the
|
|
139
|
+
f"{deployment.family}/{deployment.suffix}/{deployment.run_name} "
|
|
140
|
+
f"is missing {sorted(missing)} from its config; set them on the "
|
|
141
|
+
"model card"
|
|
141
142
|
)
|
|
142
143
|
self._model = config[MODEL_PARAM]
|
|
143
144
|
self._client = AsyncAnthropic(
|
|
@@ -31,9 +31,9 @@ _GIB = float(1 << 30)
|
|
|
31
31
|
ENABLE_THINKING_PARAM = "enable_thinking"
|
|
32
32
|
|
|
33
33
|
# Whether a replica runs its model through `torch.compile` (see
|
|
34
|
-
# `cortexgrid_infer.compiling`). Off unless the
|
|
35
|
-
# serving pays a warm-up per input shape, which is worth it
|
|
36
|
-
# repeats those shapes.
|
|
34
|
+
# `cortexgrid_infer.compiling`). Off unless the deployment's config says
|
|
35
|
+
# "true": compiled serving pays a warm-up per input shape, which is worth it
|
|
36
|
+
# only where traffic repeats those shapes.
|
|
37
37
|
COMPILE_PARAM = "compile"
|
|
38
38
|
|
|
39
39
|
# Weights are only part of what a replica holds. The multiplier covers CUDA
|
{cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/gemini/base.py
RENAMED
|
@@ -36,13 +36,13 @@ class GeminiModel(HostedModel):
|
|
|
36
36
|
holding the key to send it with."""
|
|
37
37
|
return {MODEL_PARAM: model_id, API_KEY_SECRET_PARAM: api_key_secret}
|
|
38
38
|
|
|
39
|
-
def __init__(self,
|
|
40
|
-
config = cortexgrid.model_config(
|
|
39
|
+
def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
|
|
40
|
+
config = cortexgrid.model_config(deployment)
|
|
41
41
|
missing = {MODEL_PARAM, API_KEY_SECRET_PARAM} - config.keys()
|
|
42
42
|
if missing:
|
|
43
43
|
raise RuntimeError(
|
|
44
|
-
f"{family}/{suffix}/{run_name} is
|
|
45
|
-
"its config; set them on the model card"
|
|
44
|
+
f"{deployment.family}/{deployment.suffix}/{deployment.run_name} is "
|
|
45
|
+
f"missing {sorted(missing)} from its config; set them on the model card"
|
|
46
46
|
)
|
|
47
47
|
self._config = config
|
|
48
48
|
self._model = config[MODEL_PARAM]
|
|
@@ -54,10 +54,11 @@ class Image2Mesh(LocalModel):
|
|
|
54
54
|
def client(cls, url: str, name: str) -> ServedMeshingModel:
|
|
55
55
|
return ServedMeshingModel(url=url, model_id=name)
|
|
56
56
|
|
|
57
|
-
def __init__(self,
|
|
57
|
+
def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
|
|
58
58
|
self.device = detect_device()
|
|
59
|
-
|
|
60
|
-
|
|
59
|
+
path = cortexgrid.load_model(deployment.family, deployment.suffix, deployment.run_name)
|
|
60
|
+
self.load(Path(path), self.device)
|
|
61
|
+
settings = cortexgrid.model_config(deployment)
|
|
61
62
|
requested = settings.get(COMPILE_PARAM, "false") == "true"
|
|
62
63
|
self.compiled = requested and compiling.supported(self.device)
|
|
63
64
|
if self.compiled:
|
|
@@ -84,8 +84,8 @@ class Text2Image(LocalModel):
|
|
|
84
84
|
def client(cls, url: str, name: str) -> ServedGeneratingModel:
|
|
85
85
|
return ServedGeneratingModel(url=url, model_id=name)
|
|
86
86
|
|
|
87
|
-
def __init__(self,
|
|
88
|
-
path = cortexgrid.load_model(family, suffix, run_name)
|
|
87
|
+
def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
|
|
88
|
+
path = cortexgrid.load_model(deployment.family, deployment.suffix, deployment.run_name)
|
|
89
89
|
self._device = detect_device()
|
|
90
90
|
pipe = _pipeline_class(path).from_pretrained(path, torch_dtype=torch.bfloat16)
|
|
91
91
|
pipe = pipe.to(self._device)
|
|
@@ -95,7 +95,7 @@ class Text2Image(LocalModel):
|
|
|
95
95
|
vae.enable_tiling()
|
|
96
96
|
self._pipe = pipe
|
|
97
97
|
# A denoising schedule is a loop of identically shaped forwards: capture once, replay.
|
|
98
|
-
settings = cortexgrid.model_config(
|
|
98
|
+
settings = cortexgrid.model_config(deployment)
|
|
99
99
|
requested = settings.get(COMPILE_PARAM, "false") == "true"
|
|
100
100
|
self._compiled = (
|
|
101
101
|
compiling.compile_pipeline(pipe, self._device, compiling.Mode.GRAPHED)
|
|
@@ -72,8 +72,10 @@ class Text2Text(LocalModel):
|
|
|
72
72
|
def client(cls, url: str, name: str) -> ServedCompletingModel:
|
|
73
73
|
return ServedCompletingModel(url=url, model_id=name)
|
|
74
74
|
|
|
75
|
-
def __init__(self,
|
|
76
|
-
path = cortexgrid.load_model(
|
|
75
|
+
def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
|
|
76
|
+
path = cortexgrid.load_model(
|
|
77
|
+
deployment.family, deployment.suffix, deployment.run_name
|
|
78
|
+
)
|
|
77
79
|
self._device = detect_device()
|
|
78
80
|
self._tokenizer = AutoTokenizer.from_pretrained(path)
|
|
79
81
|
if self._tokenizer.pad_token is None:
|
|
@@ -82,7 +84,7 @@ class Text2Text(LocalModel):
|
|
|
82
84
|
str(path), torch_dtype=torch.float16
|
|
83
85
|
)
|
|
84
86
|
self._model.to(self._device) # type: ignore
|
|
85
|
-
settings = cortexgrid.model_config(
|
|
87
|
+
settings = cortexgrid.model_config(deployment)
|
|
86
88
|
context = getattr(
|
|
87
89
|
self._model.config, "max_position_embeddings", DEFAULT_MAX_TOTAL_TOKENS
|
|
88
90
|
)
|
{cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/text_rewriter.py
RENAMED
|
@@ -50,15 +50,15 @@ class TextRewriter(LocalModel):
|
|
|
50
50
|
def client(cls, url: str, name: str) -> ServedRewritingModel:
|
|
51
51
|
return ServedRewritingModel(url=url, model_id=name)
|
|
52
52
|
|
|
53
|
-
def __init__(self,
|
|
54
|
-
path = cortexgrid.load_model(family, suffix, run_name)
|
|
53
|
+
def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
|
|
54
|
+
path = cortexgrid.load_model(deployment.family, deployment.suffix, deployment.run_name)
|
|
55
55
|
self._device = detect_device()
|
|
56
56
|
self._tokenizer = AutoTokenizer.from_pretrained(path)
|
|
57
57
|
self._model = AutoModelForSeq2SeqLM.from_pretrained(
|
|
58
58
|
str(path), torch_dtype=torch.bfloat16
|
|
59
59
|
)
|
|
60
60
|
self._model.to(self._device)
|
|
61
|
-
settings = cortexgrid.model_config(
|
|
61
|
+
settings = cortexgrid.model_config(deployment)
|
|
62
62
|
# As in Text2Text: a static cache pins the decoder's self-attention to
|
|
63
63
|
# one shape, and transformers compiles the decode step itself. The
|
|
64
64
|
# cross-attention cache is pinned by padding the text; see `_tokenize`.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "cortexgrid-infer"
|
|
3
|
-
version = "0.3.
|
|
3
|
+
version = "0.3.6"
|
|
4
4
|
description = "Serve apps per task (text-to-text, text-to-image, image-to-mesh) and importers per source (HuggingFace) for running models on cortexgrid"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -15,7 +15,9 @@ dependencies = [
|
|
|
15
15
|
# 0.3.4: jobs return results (remote -> JobFuture), register_model for a
|
|
16
16
|
# model that stages no weights, model_config, and deploy_model skipping a
|
|
17
17
|
# redundant PUT - which is why this library has no reuse guard of its own.
|
|
18
|
-
|
|
18
|
+
# 0.3.12: DeploymentKey - a serve-app is constructed with its deployment's
|
|
19
|
+
# key and reads that deployment's config with model_config(key).
|
|
20
|
+
"cortexgrid>=0.3.12",
|
|
19
21
|
"torch>=2.0.0",
|
|
20
22
|
# 4.47: CompileConfig, which the completion serve app hands to `generate`
|
|
21
23
|
# alongside a static cache so transformers compiles the decode step itself.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/gemini/__init__.py
RENAMED
|
File without changes
|
{cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/gemini/text2image.py
RENAMED
|
File without changes
|
{cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/gemini/text2text.py
RENAMED
|
File without changes
|
|
File without changes
|