cortexgrid-infer 0.2.2__tar.gz → 0.2.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/.gitignore +2 -1
  2. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/PKG-INFO +1 -1
  3. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/providers/huggingface_complete_serve.py +5 -14
  4. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/pyproject.toml +1 -1
  5. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/LICENSE +0 -0
  6. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/README.md +0 -0
  7. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/__init__.py +0 -0
  8. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/compiling.py +0 -0
  9. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/completion.py +0 -0
  10. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/core.py +0 -0
  11. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/device.py +0 -0
  12. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/importing.py +0 -0
  13. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/providers/__init__.py +0 -0
  14. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/providers/anthropic.py +0 -0
  15. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/providers/anthropic_serve.py +0 -0
  16. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/providers/huggingface_complete.py +0 -0
  17. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/providers/huggingface_image.py +0 -0
  18. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/providers/huggingface_image_serve.py +0 -0
  19. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/py.typed +0 -0
  20. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/requirements.py +0 -0
  21. {cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/utils.py +0 -0
{cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/.gitignore RENAMED
@@ -9,4 +9,5 @@ build/
9
9
  venv/
10
10
  .env
11
11
  .claude
12
- .mypy_cache/
12
+ .mypy_cache/
13
+ uv.lock
{cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: cortexgrid-infer
3
- Version: 0.2.2
3
+ Version: 0.2.3
4
4
  Summary: Deploy and interact with models across providers (Anthropic, HuggingFace LLM + diffusion)
5
5
  Project-URL: Homepage, https://github.com/robodatalab/cortexgrid-infer
6
6
  Project-URL: Repository, https://github.com/robodatalab/cortexgrid-infer
{cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/cortexgrid_infer/providers/huggingface_complete_serve.py RENAMED
@@ -11,12 +11,7 @@ import cortexgrid
11
11
  from cortexgrid import serve
12
12
  from fastapi import FastAPI
13
13
  from fastapi.responses import StreamingResponse
14
- from transformers import (
15
- AutoModelForCausalLM,
16
- AutoTokenizer,
17
- CompileConfig,
18
- TextIteratorStreamer,
19
- )
14
+ from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
20
15
  import torch
21
16
 
22
17
  from cortexgrid_infer import compiling
@@ -49,14 +44,10 @@ class HuggingFaceCompletingDeployment:
49
44
  str(path), torch_dtype=torch.float16
50
45
  )
51
46
  self._model.to(self._device) # type: ignore
52
- # Decode has no stable shape to capture until the cache is static; given one,
53
- # transformers compiles `generate` itself, so compiling the module would double up.
54
- self._compiled = compiling.supported(self._device)
55
- if self._compiled:
56
- self._model.generation_config.cache_implementation = "static"
57
- self._model.generation_config.compile_config = CompileConfig(
58
- mode=compiling.Mode.GRAPHED
59
- )
47
+ # Decode's shapes move as the sequence grows, so fuse without capturing.
48
+ self._compiled = compiling.compile_module(
49
+ self._model, self._device, compiling.Mode.FUSED, label="model"
50
+ )
60
51
 
61
52
  @_app.post("/complete")
62
53
  async def complete(self, body: dict[str, Any]) -> StreamingResponse:
{cortexgrid_infer-0.2.2 → cortexgrid_infer-0.2.3}/pyproject.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "cortexgrid-infer"
3
- version = "0.2.2"
3
+ version = "0.2.3"
4
4
  description = "Deploy and interact with models across providers (Anthropic, HuggingFace LLM + diffusion)"
5
5
  readme = "README.md"
6
6
  license = "Apache-2.0"