cortexgrid-infer 0.3.5__tar.gz → 0.3.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/PKG-INFO +15 -11
  2. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/README.md +13 -9
  3. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/anthropic.py +5 -4
  4. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/base.py +3 -3
  5. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/gemini/base.py +4 -4
  6. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/image2mesh.py +4 -3
  7. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/text2image.py +3 -3
  8. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/text2text.py +5 -3
  9. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/text_rewriter.py +3 -3
  10. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/pyproject.toml +4 -2
  11. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/.gitignore +0 -0
  12. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/LICENSE +0 -0
  13. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/__init__.py +0 -0
  14. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/compiling.py +0 -0
  15. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/core.py +0 -0
  16. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/device.py +0 -0
  17. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/importers/__init__.py +0 -0
  18. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/importers/base.py +0 -0
  19. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/importers/huggingface.py +0 -0
  20. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/protocols/__init__.py +0 -0
  21. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/protocols/completion.py +0 -0
  22. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/protocols/imaging.py +0 -0
  23. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/protocols/meshing.py +0 -0
  24. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/protocols/rewriting.py +0 -0
  25. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/py.typed +0 -0
  26. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/registry.py +0 -0
  27. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/__init__.py +0 -0
  28. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/gemini/__init__.py +0 -0
  29. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/gemini/text2image.py +0 -0
  30. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/serve_apps/gemini/text2text.py +0 -0
  31. {cortexgrid_infer-0.3.5 → cortexgrid_infer-0.3.6}/cortexgrid_infer/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: cortexgrid-infer
3
- Version: 0.3.5
3
+ Version: 0.3.6
4
4
  Summary: Serve apps per task (text-to-text, text-to-image, image-to-mesh) and importers per source (HuggingFace) for running models on cortexgrid
5
5
  Project-URL: Homepage, https://github.com/robodatalab/cortexgrid-infer
6
6
  Project-URL: Repository, https://github.com/robodatalab/cortexgrid-infer
@@ -9,7 +9,7 @@ License-File: LICENSE
9
9
  Requires-Python: <3.12,>=3.11
10
10
  Requires-Dist: accelerate>=0.33
11
11
  Requires-Dist: anthropic>=0.40.0
12
- Requires-Dist: cortexgrid>=0.3.4
12
+ Requires-Dist: cortexgrid>=0.3.12
13
13
  Requires-Dist: diffusers>=0.38
14
14
  Requires-Dist: fastapi>=0.133
15
15
  Requires-Dist: google-genai>=1.0
@@ -132,7 +132,7 @@ async def run():
132
132
  asyncio.run(run())
133
133
 
134
134
  # 4. Teardown.
135
- cortexgrid.undeploy_model(imp.family, imp.suffix, cortexgrid.IMPORTED) # free the GPU
135
+ cortexgrid.undeploy_model(deployment.key) # free the GPU
136
136
  cortexgrid.delete_model(imp.family, imp.suffix, cortexgrid.IMPORTED) # and the weights
137
137
  ```
138
138
 
@@ -179,20 +179,24 @@ beyond the weights. (A `Hosted` entry asks for none — it holds no weights.)
179
179
  ## Compilation
180
180
 
181
181
  Every serve app that runs weights can run them through `torch.compile`, and none
182
- does unless asked: a replica is eager unless its model card carries
183
- `compile: "true"`. Each `LocalModel` writes `compile: "false"` onto the card at
184
- import, so turning it on is an edit of the card and a redeploy:
182
+ does unless asked: a replica is eager unless its deployment's config carries
183
+ `compile: "true"`. Each `LocalModel` writes `compile: "false"` onto the model card
184
+ at import, so turning it on is a deployment that asks for it:
185
185
 
186
186
  ```python
187
- config = cortexgrid.model_config(family, suffix, cortexgrid.IMPORTED)
188
- cortexgrid.set_model_config(family, suffix, cortexgrid.IMPORTED, {**config, "compile": "true"})
189
- cortexgrid.deploy_model(family, suffix, cortexgrid.IMPORTED, wait=True)
187
+ cortexgrid.deploy_model(family, suffix, cortexgrid.IMPORTED, wait=True, config={"compile": "true"})
190
188
  ```
191
189
 
190
+ That is a separate, compiled deployment of the same model, alongside any uncompiled
191
+ one: a model gets one deployment per distinct config, each at its own
192
+ `deployment.url`. `Text2Text`'s `enable_thinking` and `max_total_tokens` are set per
193
+ deployment the same way. Whatever a deployment leaves out comes from the model card,
194
+ so the defaults `config()` writes there at import still apply.
195
+
192
196
  It is off by default because it is not free: every input shape pays a warm-up on
193
197
  the request that first brings it, and each CUDA graph holds memory of its own. It
194
198
  pays back where traffic keeps to a few shapes and the model is small enough for
195
- launch overhead to matter. A card asking for it off CUDA is served eager.
199
+ launch overhead to matter. A deployment asking for it off CUDA is served eager.
196
200
 
197
201
  Compiled, a serve app runs `torch.compile` over the model it loads, because
198
202
  eager PyTorch bills per *operation* and a transformer forward is mostly glue —
@@ -233,7 +237,7 @@ Each serve app is tuned rather than defaulted:
233
237
  1.2 GB of keys and values re-read per generated token, several times the
234
238
  traffic of the model's own weights, to make room for a prompt nobody sent. It
235
239
  is pinned instead to the prompt and reply a caller plausibly sends: 4096 tokens,
236
- or `max_total_tokens` on the model card, never past the model's own
240
+ or `max_total_tokens` from the deployment's config, never past the model's own
237
241
  `max_position_embeddings`. A reply takes at most half of it, so an
238
242
  over-optimistic `max_new_tokens` cannot squeeze the prompt to nothing, and a
239
243
  prompt that will not fit in the rest is truncated — keeping the end, since a
@@ -104,7 +104,7 @@ async def run():
104
104
  asyncio.run(run())
105
105
 
106
106
  # 4. Teardown.
107
- cortexgrid.undeploy_model(imp.family, imp.suffix, cortexgrid.IMPORTED) # free the GPU
107
+ cortexgrid.undeploy_model(deployment.key) # free the GPU
108
108
  cortexgrid.delete_model(imp.family, imp.suffix, cortexgrid.IMPORTED) # and the weights
109
109
  ```
110
110
 
@@ -151,20 +151,24 @@ beyond the weights. (A `Hosted` entry asks for none — it holds no weights.)
151
151
  ## Compilation
152
152
 
153
153
  Every serve app that runs weights can run them through `torch.compile`, and none
154
- does unless asked: a replica is eager unless its model card carries
155
- `compile: "true"`. Each `LocalModel` writes `compile: "false"` onto the card at
156
- import, so turning it on is an edit of the card and a redeploy:
154
+ does unless asked: a replica is eager unless its deployment's config carries
155
+ `compile: "true"`. Each `LocalModel` writes `compile: "false"` onto the model card
156
+ at import, so turning it on is a deployment that asks for it:
157
157
 
158
158
  ```python
159
- config = cortexgrid.model_config(family, suffix, cortexgrid.IMPORTED)
160
- cortexgrid.set_model_config(family, suffix, cortexgrid.IMPORTED, {**config, "compile": "true"})
161
- cortexgrid.deploy_model(family, suffix, cortexgrid.IMPORTED, wait=True)
159
+ cortexgrid.deploy_model(family, suffix, cortexgrid.IMPORTED, wait=True, config={"compile": "true"})
162
160
  ```
163
161
 
162
+ That is a separate, compiled deployment of the same model, alongside any uncompiled
163
+ one: a model gets one deployment per distinct config, each at its own
164
+ `deployment.url`. `Text2Text`'s `enable_thinking` and `max_total_tokens` are set per
165
+ deployment the same way. Whatever a deployment leaves out comes from the model card,
166
+ so the defaults `config()` writes there at import still apply.
167
+
164
168
  It is off by default because it is not free: every input shape pays a warm-up on
165
169
  the request that first brings it, and each CUDA graph holds memory of its own. It
166
170
  pays back where traffic keeps to a few shapes and the model is small enough for
167
- launch overhead to matter. A card asking for it off CUDA is served eager.
171
+ launch overhead to matter. A deployment asking for it off CUDA is served eager.
168
172
 
169
173
  Compiled, a serve app runs `torch.compile` over the model it loads, because
170
174
  eager PyTorch bills per *operation* and a transformer forward is mostly glue —
@@ -205,7 +209,7 @@ Each serve app is tuned rather than defaulted:
205
209
  1.2 GB of keys and values re-read per generated token, several times the
206
210
  traffic of the model's own weights, to make room for a prompt nobody sent. It
207
211
  is pinned instead to the prompt and reply a caller plausibly sends: 4096 tokens,
208
- or `max_total_tokens` on the model card, never past the model's own
212
+ or `max_total_tokens` from the deployment's config, never past the model's own
209
213
  `max_position_embeddings`. A reply takes at most half of it, so an
210
214
  over-optimistic `max_new_tokens` cannot squeeze the prompt to nothing, and a
211
215
  prompt that will not fit in the rest is truncated — keeping the end, since a
@@ -131,13 +131,14 @@ class AnthropicText2Text(HostedModel):
131
131
  def client(cls, url: str, name: str) -> ServedCompletingModel:
132
132
  return ServedCompletingModel(url=url, model_id=name)
133
133
 
134
- def __init__(self, family: str, suffix: str, run_name: str) -> None:
135
- config = cortexgrid.model_config(family, suffix, run_name)
134
+ def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
135
+ config = cortexgrid.model_config(deployment)
136
136
  missing = {MODEL_PARAM, API_KEY_SECRET_PARAM} - config.keys()
137
137
  if missing:
138
138
  raise RuntimeError(
139
- f"{family}/{suffix}/{run_name} is missing {sorted(missing)} from "
140
- "its config; set them on the model card"
139
+ f"{deployment.family}/{deployment.suffix}/{deployment.run_name} "
140
+ f"is missing {sorted(missing)} from its config; set them on the "
141
+ "model card"
141
142
  )
142
143
  self._model = config[MODEL_PARAM]
143
144
  self._client = AsyncAnthropic(
@@ -31,9 +31,9 @@ _GIB = float(1 << 30)
31
31
  ENABLE_THINKING_PARAM = "enable_thinking"
32
32
 
33
33
  # Whether a replica runs its model through `torch.compile` (see
34
- # `cortexgrid_infer.compiling`). Off unless the model card says "true": compiled
35
- # serving pays a warm-up per input shape, which is worth it only where traffic
36
- # repeats those shapes.
34
+ # `cortexgrid_infer.compiling`). Off unless the deployment's config says
35
+ # "true": compiled serving pays a warm-up per input shape, which is worth it
36
+ # only where traffic repeats those shapes.
37
37
  COMPILE_PARAM = "compile"
38
38
 
39
39
  # Weights are only part of what a replica holds. The multiplier covers CUDA
@@ -36,13 +36,13 @@ class GeminiModel(HostedModel):
36
36
  holding the key to send it with."""
37
37
  return {MODEL_PARAM: model_id, API_KEY_SECRET_PARAM: api_key_secret}
38
38
 
39
- def __init__(self, family: str, suffix: str, run_name: str) -> None:
40
- config = cortexgrid.model_config(family, suffix, run_name)
39
+ def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
40
+ config = cortexgrid.model_config(deployment)
41
41
  missing = {MODEL_PARAM, API_KEY_SECRET_PARAM} - config.keys()
42
42
  if missing:
43
43
  raise RuntimeError(
44
- f"{family}/{suffix}/{run_name} is missing {sorted(missing)} from "
45
- "its config; set them on the model card"
44
+ f"{deployment.family}/{deployment.suffix}/{deployment.run_name} is "
45
+ f"missing {sorted(missing)} from its config; set them on the model card"
46
46
  )
47
47
  self._config = config
48
48
  self._model = config[MODEL_PARAM]
@@ -54,10 +54,11 @@ class Image2Mesh(LocalModel):
54
54
  def client(cls, url: str, name: str) -> ServedMeshingModel:
55
55
  return ServedMeshingModel(url=url, model_id=name)
56
56
 
57
- def __init__(self, family: str, suffix: str, run_name: str) -> None:
57
+ def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
58
58
  self.device = detect_device()
59
- self.load(Path(cortexgrid.load_model(family, suffix, run_name)), self.device)
60
- settings = cortexgrid.model_config(family, suffix, run_name)
59
+ path = cortexgrid.load_model(deployment.family, deployment.suffix, deployment.run_name)
60
+ self.load(Path(path), self.device)
61
+ settings = cortexgrid.model_config(deployment)
61
62
  requested = settings.get(COMPILE_PARAM, "false") == "true"
62
63
  self.compiled = requested and compiling.supported(self.device)
63
64
  if self.compiled:
@@ -84,8 +84,8 @@ class Text2Image(LocalModel):
84
84
  def client(cls, url: str, name: str) -> ServedGeneratingModel:
85
85
  return ServedGeneratingModel(url=url, model_id=name)
86
86
 
87
- def __init__(self, family: str, suffix: str, run_name: str) -> None:
88
- path = cortexgrid.load_model(family, suffix, run_name)
87
+ def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
88
+ path = cortexgrid.load_model(deployment.family, deployment.suffix, deployment.run_name)
89
89
  self._device = detect_device()
90
90
  pipe = _pipeline_class(path).from_pretrained(path, torch_dtype=torch.bfloat16)
91
91
  pipe = pipe.to(self._device)
@@ -95,7 +95,7 @@ class Text2Image(LocalModel):
95
95
  vae.enable_tiling()
96
96
  self._pipe = pipe
97
97
  # A denoising schedule is a loop of identically shaped forwards: capture once, replay.
98
- settings = cortexgrid.model_config(family, suffix, run_name)
98
+ settings = cortexgrid.model_config(deployment)
99
99
  requested = settings.get(COMPILE_PARAM, "false") == "true"
100
100
  self._compiled = (
101
101
  compiling.compile_pipeline(pipe, self._device, compiling.Mode.GRAPHED)
@@ -72,8 +72,10 @@ class Text2Text(LocalModel):
72
72
  def client(cls, url: str, name: str) -> ServedCompletingModel:
73
73
  return ServedCompletingModel(url=url, model_id=name)
74
74
 
75
- def __init__(self, family: str, suffix: str, run_name: str) -> None:
76
- path = cortexgrid.load_model(family, suffix, run_name)
75
+ def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
76
+ path = cortexgrid.load_model(
77
+ deployment.family, deployment.suffix, deployment.run_name
78
+ )
77
79
  self._device = detect_device()
78
80
  self._tokenizer = AutoTokenizer.from_pretrained(path)
79
81
  if self._tokenizer.pad_token is None:
@@ -82,7 +84,7 @@ class Text2Text(LocalModel):
82
84
  str(path), torch_dtype=torch.float16
83
85
  )
84
86
  self._model.to(self._device) # type: ignore
85
- settings = cortexgrid.model_config(family, suffix, run_name)
87
+ settings = cortexgrid.model_config(deployment)
86
88
  context = getattr(
87
89
  self._model.config, "max_position_embeddings", DEFAULT_MAX_TOTAL_TOKENS
88
90
  )
@@ -50,15 +50,15 @@ class TextRewriter(LocalModel):
50
50
  def client(cls, url: str, name: str) -> ServedRewritingModel:
51
51
  return ServedRewritingModel(url=url, model_id=name)
52
52
 
53
- def __init__(self, family: str, suffix: str, run_name: str) -> None:
54
- path = cortexgrid.load_model(family, suffix, run_name)
53
+ def __init__(self, deployment: cortexgrid.DeploymentKey) -> None:
54
+ path = cortexgrid.load_model(deployment.family, deployment.suffix, deployment.run_name)
55
55
  self._device = detect_device()
56
56
  self._tokenizer = AutoTokenizer.from_pretrained(path)
57
57
  self._model = AutoModelForSeq2SeqLM.from_pretrained(
58
58
  str(path), torch_dtype=torch.bfloat16
59
59
  )
60
60
  self._model.to(self._device)
61
- settings = cortexgrid.model_config(family, suffix, run_name)
61
+ settings = cortexgrid.model_config(deployment)
62
62
  # As in Text2Text: a static cache pins the decoder's self-attention to
63
63
  # one shape, and transformers compiles the decode step itself. The
64
64
  # cross-attention cache is pinned by padding the text; see `_tokenize`.
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "cortexgrid-infer"
3
- version = "0.3.5"
3
+ version = "0.3.6"
4
4
  description = "Serve apps per task (text-to-text, text-to-image, image-to-mesh) and importers per source (HuggingFace) for running models on cortexgrid"
5
5
  readme = "README.md"
6
6
  license = "Apache-2.0"
@@ -15,7 +15,9 @@ dependencies = [
15
15
  # 0.3.4: jobs return results (remote -> JobFuture), register_model for a
16
16
  # model that stages no weights, model_config, and deploy_model skipping a
17
17
  # redundant PUT - which is why this library has no reuse guard of its own.
18
- "cortexgrid>=0.3.4",
18
+ # 0.3.12: DeploymentKey - a serve-app is constructed with its deployment's
19
+ # key and reads that deployment's config with model_config(key).
20
+ "cortexgrid>=0.3.12",
19
21
  "torch>=2.0.0",
20
22
  # 4.47: CompileConfig, which the completion serve app hands to `generate`
21
23
  # alongside a static cache so transformers compiles the decode step itself.