litserve 0.2.0.dev0__tar.gz → 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. litserve-0.2.1/PKG-INFO +288 -0
  2. litserve-0.2.1/README.md +239 -0
  3. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/__about__.py +1 -1
  4. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/api.py +4 -15
  5. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/examples/openai_spec_example.py +8 -0
  6. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/server.py +1 -2
  7. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/specs/openai.py +27 -1
  8. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/utils.py +1 -21
  9. litserve-0.2.1/src/litserve.egg-info/PKG-INFO +288 -0
  10. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve.egg-info/requires.txt +1 -0
  11. litserve-0.2.0.dev0/PKG-INFO +0 -294
  12. litserve-0.2.0.dev0/README.md +0 -246
  13. litserve-0.2.0.dev0/src/litserve.egg-info/PKG-INFO +0 -294
  14. {litserve-0.2.0.dev0 → litserve-0.2.1}/LICENSE +0 -0
  15. {litserve-0.2.0.dev0 → litserve-0.2.1}/MANIFEST.in +0 -0
  16. {litserve-0.2.0.dev0 → litserve-0.2.1}/requirements.txt +0 -0
  17. {litserve-0.2.0.dev0 → litserve-0.2.1}/setup.cfg +0 -0
  18. {litserve-0.2.0.dev0 → litserve-0.2.1}/setup.py +0 -0
  19. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/__init__.py +0 -0
  20. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/connector.py +0 -0
  21. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/examples/__init__.py +0 -0
  22. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/examples/simple_example.py +0 -0
  23. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/python_client.py +0 -0
  24. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/specs/__init__.py +0 -0
  25. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/specs/base.py +0 -0
  26. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve.egg-info/SOURCES.txt +0 -0
  27. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve.egg-info/dependency_links.txt +0 -0
  28. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve.egg-info/not-zip-safe +0 -0
  29. {litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve.egg-info/top_level.txt +0 -0
litserve-0.2.1/PKG-INFO ADDED
@@ -0,0 +1,288 @@
1
+ Metadata-Version: 2.1
2
+ Name: litserve
3
+ Version: 0.2.1
4
+ Summary: Lightweight AI server.
5
+ Home-page: https://github.com/Lightning-AI/litserve
6
+ Download-URL: https://github.com/Lightning-AI/litserve
7
+ Author: Lightning-AI et al.
8
+ Author-email: community@lightning.ai
9
+ License: Apache-2.0
10
+ Project-URL: Bug Tracker, https://github.com/Lightning-AI/litserve/issues
11
+ Project-URL: Documentation, https://lightning-ai.github.io/litserve/
12
+ Project-URL: Source Code, https://github.com/Lightning-AI/litserve
13
+ Keywords: deep learning,pytorch,AI
14
+ Classifier: Environment :: Console
15
+ Classifier: Natural Language :: English
16
+ Classifier: Development Status :: 3 - Alpha
17
+ Classifier: Intended Audience :: Developers
18
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
19
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
20
+ Classifier: License :: OSI Approved :: Apache Software License
21
+ Classifier: Operating System :: OS Independent
22
+ Classifier: Programming Language :: Python :: 3
23
+ Classifier: Programming Language :: Python :: 3.8
24
+ Classifier: Programming Language :: Python :: 3.9
25
+ Classifier: Programming Language :: Python :: 3.10
26
+ Classifier: Programming Language :: Python :: 3.11
27
+ Requires-Python: >=3.8
28
+ Description-Content-Type: text/markdown
29
+ License-File: LICENSE
30
+ Requires-Dist: fastapi>=0.100
31
+ Requires-Dist: httpx
32
+ Requires-Dist: uvicorn>=0.29.0
33
+ Provides-Extra: test
34
+ Requires-Dist: asgi-lifespan; extra == "test"
35
+ Requires-Dist: coverage[toml]>=7.5.3; extra == "test"
36
+ Requires-Dist: lightning>2.0.0; extra == "test"
37
+ Requires-Dist: mypy==1.11.1; extra == "test"
38
+ Requires-Dist: numpy<2.0; extra == "test"
39
+ Requires-Dist: openai>=1.12.0; extra == "test"
40
+ Requires-Dist: pillow; extra == "test"
41
+ Requires-Dist: psutil; extra == "test"
42
+ Requires-Dist: pytest-asyncio; extra == "test"
43
+ Requires-Dist: pytest-cov; extra == "test"
44
+ Requires-Dist: pytest>=8.0; extra == "test"
45
+ Requires-Dist: python-multipart; extra == "test"
46
+ Requires-Dist: requests; extra == "test"
47
+ Requires-Dist: torch>2.0.0; extra == "test"
48
+ Requires-Dist: transformers; extra == "test"
49
+
50
+ <div align='center'>
51
+
52
+ # Easily serve AI models Lightning fast ⚡
53
+
54
+ <img alt="Lightning" src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/ls_banner2.png" width="800px" style="max-width: 100%;">
55
+
56
+ &nbsp;
57
+
58
+ <strong>Lightning-fast serving engine for AI models.</strong>
59
+ Easy. Flexible. Enterprise-scale.
60
+ </div>
61
+
62
+ ----
63
+
64
+ **LitServe** is an easy-to-use, flexible serving engine for AI models built on FastAPI. Features like batching, streaming, and GPU autoscaling eliminate the need to rebuild a FastAPI server per model.
65
+
66
+ LitServe is at least [2x faster](#performance) than plain FastAPI.
67
+
68
+ <div align='center'>
69
+
70
+ <pre>
71
+ ✅ (2x)+ faster serving ✅ Easy to use ✅ Batching, Streaming
72
+ ✅ Bring your own model ✅ PyTorch/JAX/TF/... ✅ Built on FastAPI
73
+ ✅ GPU autoscaling ✅ Multi-modal ✅ Self-host or ⚡️ managed
74
+ </pre>
75
+
76
+ <div align='center'>
77
+
78
+ [![Discord](https://img.shields.io/discord/1077906959069626439?label=Get%20help%20on%20Discord)](https://discord.gg/VptPCZkGNa)
79
+ ![cpu-tests](https://github.com/Lightning-AI/litserve/actions/workflows/ci-testing.yml/badge.svg)
80
+ [![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/Lightning-AI/litserve/blob/main/LICENSE)
81
+
82
+ </div>
83
+ </div>
84
+ <div align="center">
85
+ <div style="text-align: center;">
86
+ <a target="_blank" href="#quick-start" style="margin: 0 10px;">Quick start</a> •
87
+ <a target="_blank" href="#featured-examples" style="margin: 0 10px;">Examples</a> •
88
+ <a target="_blank" href="#features" style="margin: 0 10px;">Features</a> •
89
+ <a target="_blank" href="#performance" style="margin: 0 10px;">Performance</a> •
90
+ <a target="_blank" href="#hosting-options" style="margin: 0 10px;">Hosting</a> •
91
+ <a target="_blank" href="https://lightning.ai/docs/litserve" style="margin: 0 10px;">Docs</a>
92
+ </div>
93
+ </div>
94
+
95
+ &nbsp;
96
+
97
+ <div align="center">
98
+ <a target="_blank" href="https://lightning.ai/docs/litserve/home/get-started">
99
+ <img src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/get-started-badge.svg" height="36px" alt="Get started"/>
100
+ </a>
101
+ </div>
102
+
103
+ &nbsp;
104
+
105
+ # Quick start
106
+
107
+ Install LitServe via pip ([more options](https://lightning.ai/docs/litserve/home/install)):
108
+
109
+ ```bash
110
+ pip install litserve
111
+ ```
112
+
113
+ ### Define a server
114
+ This 2-model toy example (AI compound system) shows LitServe's flexibility ([see real examples](#featured-examples)):
115
+
116
+ ```python
117
+ # server.py
118
+ import litserve as ls
119
+
120
+ # (STEP 1) - DEFINE THE API (compound AI system)
121
+ class SimpleLitAPI(ls.LitAPI):
122
+ def setup(self, device):
123
+ # setup is called once at startup. Build a compound AI system (1+ models), connect DBs, load data, etc...
124
+ self.model1 = lambda x: x**2
125
+ self.model2 = lambda x: x**3
126
+
127
+ def decode_request(self, request):
128
+ # Convert the request payload to model input.
129
+ return request["input"]
130
+
131
+ def predict(self, x):
132
+ # Easily build compound systems. Run inference and return the output.
133
+ squared = self.model1(x)
134
+ cubed = self.model2(x)
135
+ output = squared + cubed
136
+ return {"output": output}
137
+
138
+ def encode_response(self, output):
139
+ # Convert the model output to a response payload.
140
+ return {"output": output}
141
+
142
+ # (STEP 2) - START THE SERVER
143
+ if __name__ == "__main__":
144
+ # serve with advanced features (GPUs, etc...)
145
+ server = ls.LitServer(SimpleLitAPI(), accelerator="auto", max_batch_size=1)
146
+ server.run(port=8000)
147
+ ```
148
+
149
+ Now run the server via the command-line
150
+
151
+ ```bash
152
+ python server.py
153
+ ```
154
+
155
+ - [LitAPI](https://lightning.ai/docs/litserve/api-reference/litapi) gives full control to build scalable compound AI systems (1 or more models).
156
+ - [LitServer](https://lightning.ai/docs/litserve/api-reference/litserver) handles optimizations like batching, auto-GPU scaling, etc...
157
+
158
+ ### Query the server
159
+
160
+ Use the auto-generated LitServe client:
161
+
162
+ ```bash
163
+ python client.py
164
+ ```
165
+
166
+ <details>
167
+ <summary>Write a custom client</summary>
168
+
169
+ ```python
170
+ import requests
171
+ response = requests.post(
172
+ "http://127.0.0.1:8000/predict",
173
+ json={"input": 4.0}
174
+ )
175
+ ```
176
+ </details>
177
+
178
+ &nbsp;
179
+
180
+
181
+ # Featured examples
182
+ Use LitServe to deploy any model or AI service: (Gen AI, classical ML, embedding servers, LLMs, vision, audio, multi-modal systems, etc...)
183
+
184
+ <div align='center'>
185
+ <div width='200px'>
186
+ <video src="https://github.com/user-attachments/assets/5e73549a-bc0f-47a9-9d9c-5b54389be5de" width='200px' controls></video>
187
+ </div>
188
+ </div>
189
+
190
+
191
+ <pre>
192
+ <strong>Featured examples</strong><br>
193
+ <strong>Toy model:</strong> <a target="_blank" href="#define-a-server">Hello world</a>
194
+ <strong>LLMs:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-private-llama-3-8b-api">Llama 3 (8B)</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/openai-fault-tolerant-proxy-server">LLM Proxy server</a>
195
+ <strong>NLP:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-any-hugging-face-model-instantly">Hugging face</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-hugging-face-bert-model">BERT</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-text-embedding-api-with-litserve">Text embedding API</a>
196
+ <strong>Multimodal:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-open-ai-clip-with-litserve">OpenAI Clip</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-multi-modal-llm-with-minicpm">MiniCPM</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-phi3-5-vision-api-with-litserve">Phi-3.5 Vision Instruct</a>
197
+ <strong>Audio:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-open-ai-s-whisper-model">Whisper</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-music-generation-api-with-meta-s-audio-craft">AudioCraft</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-audio-generation-api">StableAudio</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-noise-cancellation-api-with-deepfilternet">Noise cancellation (DeepFilterNet)</a>
198
+ <strong>Vision:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-private-api-for-stable-diffusion-2">Stable diffusion 2</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-image-generation-api-with-auraflow">AuraFlow</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-image-generation-api-with-flux">Flux</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-super-resolution-image-api-with-aura-sr">Image super resolution (Aura SR)</a>
199
+ <strong>Speech:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-voice-clone-api-coqui-xtts-v2-model">Text-speech (XTTS V2)</a>
200
+ <strong>Classical ML:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-random-forest-with-litserve">Random forest</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-xgboost-with-litserve">XGBoost</a>
201
+ <strong>Miscellaneous:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-media-conversion-api-with-ffmpeg">Media conversion API (ffmpeg)</a>
202
+ </pre>
203
+
204
+ [Browse 100+ community-built templates](https://lightning.ai/studios?section=serving)
205
+
206
+ &nbsp;
207
+
208
+ # Features
209
+ State-of-the-art features:
210
+
211
+ ✅ [(2x)+ faster than plain FastAPI](#performance)
212
+ ✅ [Bring your own model](https://lightning.ai/docs/litserve/features/full-control)
213
+ ✅ [Build compound systems (1+ models)](https://lightning.ai/docs/litserve/features/compound-ai-systems)
214
+ ✅ [GPU autoscaling](https://lightning.ai/docs/litserve/features/gpu-inference)
215
+ ✅ [Batching](https://lightning.ai/docs/litserve/features/batching)
216
+ ✅ [Streaming](https://lightning.ai/docs/litserve/features/streaming)
217
+ ✅ [Worker autoscaling](https://lightning.ai/docs/litserve/features/autoscaling)
218
+ ✅ [Self-host on your machines](https://lightning.ai/docs/litserve/features/hosting-methods#host-on-your-own)
219
+ ✅ [Host fully managed on Lightning AI](https://lightning.ai/docs/litserve/features/hosting-methods#host-on-lightning-studios)
220
+ ✅ [Serve all models: (LLMs, vision, etc.)](https://lightning.ai/docs/litserve/examples)
221
+ ✅ [Scale to zero (serverless)](https://lightning.ai/docs/litserve/features/streaming)
222
+ ✅ [Supports PyTorch, JAX, TF, etc...](https://lightning.ai/docs/litserve/features/full-control)
223
+ ✅ [OpenAPI compliant](https://www.openapis.org/)
224
+ ✅ [Open AI compatibility](https://lightning.ai/docs/litserve/features/open-ai-spec)
225
+ ✅ [Authentication](https://lightning.ai/docs/litserve/features/authentication)
226
+
227
+
228
+ [10+ features...](https://lightning.ai/docs/litserve/features)
229
+
230
+ **Note:** We prioritize scalable, enterprise-level features over hype.
231
+
232
+ &nbsp;
233
+
234
+ # Performance
235
+ LitServe is designed for AI workloads. Specialized multi-worker handling delivers a minimum **2x speedup over FastAPI**.
236
+
237
+ Additional features like batching and GPU autoscaling can drive performance well beyond 2x, scaling efficiently to handle more simultaneous requests than FastAPI and TorchServe.
238
+
239
+ Reproduce the full benchmarks [here](https://lightning.ai/docs/litserve/home/benchmarks) (higher is better).
240
+
241
+ <div align="center">
242
+ <img alt="LitServe" src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/ls_charts_v6.png" width="1000px" style="max-width: 100%;">
243
+ </div>
244
+
245
+ These results are for image and text classification ML tasks. The performance relationships hold for other ML tasks (embedding, LLM serving, audio, segmentation, object detection, summarization etc...).
246
+
247
+ ***💡 Note on LLM serving:*** For high-performance LLM serving (like Ollama/VLLM), use [LitGPT](https://github.com/Lightning-AI/litgpt?tab=readme-ov-file#deploy-an-llm) or build your custom VLLM-like server with LitServe. Optimizations like kv-caching, which can be done with LitServe, are needed to maximize LLM performance.
248
+
249
+ &nbsp;
250
+
251
+ # Hosting options
252
+ LitServe can be hosted independently on your own machines or fully managed via Lightning Studios.
253
+
254
+ Self-hosting is ideal for hackers, students, and DIY developers, while fully managed hosting is ideal for enterprise developers needing easy autoscaling, security, release management, and 99.995% uptime and observability.
255
+
256
+ &nbsp;
257
+
258
+ <div align="center">
259
+ <a target="_blank" href="https://lightning.ai/lightning-ai/studios/litserve-hello-world">
260
+ <img src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/host-on-lightning.svg" alt="Host on Lightning"/>
261
+ </a>
262
+ </div>
263
+
264
+ &nbsp;
265
+
266
+ <div align='center'>
267
+
268
+ | Feature | Self Managed | Fully Managed on Studios |
269
+ |----------------------------------|-----------------------------------|-------------------------------------|
270
+ | Deployment | ✅ Do it yourself deployment | ✅ One-button cloud deploy |
271
+ | Load balancing | ❌ | ✅ |
272
+ | Autoscaling | ❌ | ✅ |
273
+ | Scale to zero | ❌ | ✅ |
274
+ | Multi-machine inference | ❌ | ✅ |
275
+ | Authentication | ❌ | ✅ |
276
+ | Own VPC | ❌ | ✅ |
277
+ | AWS, GCP | ❌ | ✅ |
278
+ | Use your own cloud commits | ❌ | ✅ |
279
+
280
+ </div>
281
+
282
+ &nbsp;
283
+
284
+ # Community
285
+ LitServe is a [community project accepting contributions](https://lightning.ai/docs/litserve/community) - Let's make the world's most advanced AI inference engine.
286
+
287
+ 💬 [Get help on Discord](https://discord.com/invite/XncpTy7DSt)
288
+ 📋 [License: Apache 2.0](https://github.com/Lightning-AI/litserve/blob/main/LICENSE)
litserve-0.2.1/README.md ADDED
@@ -0,0 +1,239 @@
1
+ <div align='center'>
2
+
3
+ # Easily serve AI models Lightning fast ⚡
4
+
5
+ <img alt="Lightning" src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/ls_banner2.png" width="800px" style="max-width: 100%;">
6
+
7
+ &nbsp;
8
+
9
+ <strong>Lightning-fast serving engine for AI models.</strong>
10
+ Easy. Flexible. Enterprise-scale.
11
+ </div>
12
+
13
+ ----
14
+
15
+ **LitServe** is an easy-to-use, flexible serving engine for AI models built on FastAPI. Features like batching, streaming, and GPU autoscaling eliminate the need to rebuild a FastAPI server per model.
16
+
17
+ LitServe is at least [2x faster](#performance) than plain FastAPI.
18
+
19
+ <div align='center'>
20
+
21
+ <pre>
22
+ ✅ (2x)+ faster serving ✅ Easy to use ✅ Batching, Streaming
23
+ ✅ Bring your own model ✅ PyTorch/JAX/TF/... ✅ Built on FastAPI
24
+ ✅ GPU autoscaling ✅ Multi-modal ✅ Self-host or ⚡️ managed
25
+ </pre>
26
+
27
+ <div align='center'>
28
+
29
+ [![Discord](https://img.shields.io/discord/1077906959069626439?label=Get%20help%20on%20Discord)](https://discord.gg/VptPCZkGNa)
30
+ ![cpu-tests](https://github.com/Lightning-AI/litserve/actions/workflows/ci-testing.yml/badge.svg)
31
+ [![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/Lightning-AI/litserve/blob/main/LICENSE)
32
+
33
+ </div>
34
+ </div>
35
+ <div align="center">
36
+ <div style="text-align: center;">
37
+ <a target="_blank" href="#quick-start" style="margin: 0 10px;">Quick start</a> •
38
+ <a target="_blank" href="#featured-examples" style="margin: 0 10px;">Examples</a> •
39
+ <a target="_blank" href="#features" style="margin: 0 10px;">Features</a> •
40
+ <a target="_blank" href="#performance" style="margin: 0 10px;">Performance</a> •
41
+ <a target="_blank" href="#hosting-options" style="margin: 0 10px;">Hosting</a> •
42
+ <a target="_blank" href="https://lightning.ai/docs/litserve" style="margin: 0 10px;">Docs</a>
43
+ </div>
44
+ </div>
45
+
46
+ &nbsp;
47
+
48
+ <div align="center">
49
+ <a target="_blank" href="https://lightning.ai/docs/litserve/home/get-started">
50
+ <img src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/get-started-badge.svg" height="36px" alt="Get started"/>
51
+ </a>
52
+ </div>
53
+
54
+ &nbsp;
55
+
56
+ # Quick start
57
+
58
+ Install LitServe via pip ([more options](https://lightning.ai/docs/litserve/home/install)):
59
+
60
+ ```bash
61
+ pip install litserve
62
+ ```
63
+
64
+ ### Define a server
65
+ This 2-model toy example (AI compound system) shows LitServe's flexibility ([see real examples](#featured-examples)):
66
+
67
+ ```python
68
+ # server.py
69
+ import litserve as ls
70
+
71
+ # (STEP 1) - DEFINE THE API (compound AI system)
72
+ class SimpleLitAPI(ls.LitAPI):
73
+ def setup(self, device):
74
+ # setup is called once at startup. Build a compound AI system (1+ models), connect DBs, load data, etc...
75
+ self.model1 = lambda x: x**2
76
+ self.model2 = lambda x: x**3
77
+
78
+ def decode_request(self, request):
79
+ # Convert the request payload to model input.
80
+ return request["input"]
81
+
82
+ def predict(self, x):
83
+ # Easily build compound systems. Run inference and return the output.
84
+ squared = self.model1(x)
85
+ cubed = self.model2(x)
86
+ output = squared + cubed
87
+ return {"output": output}
88
+
89
+ def encode_response(self, output):
90
+ # Convert the model output to a response payload.
91
+ return {"output": output}
92
+
93
+ # (STEP 2) - START THE SERVER
94
+ if __name__ == "__main__":
95
+ # serve with advanced features (GPUs, etc...)
96
+ server = ls.LitServer(SimpleLitAPI(), accelerator="auto", max_batch_size=1)
97
+ server.run(port=8000)
98
+ ```
99
+
100
+ Now run the server via the command-line
101
+
102
+ ```bash
103
+ python server.py
104
+ ```
105
+
106
+ - [LitAPI](https://lightning.ai/docs/litserve/api-reference/litapi) gives full control to build scalable compound AI systems (1 or more models).
107
+ - [LitServer](https://lightning.ai/docs/litserve/api-reference/litserver) handles optimizations like batching, auto-GPU scaling, etc...
108
+
109
+ ### Query the server
110
+
111
+ Use the auto-generated LitServe client:
112
+
113
+ ```bash
114
+ python client.py
115
+ ```
116
+
117
+ <details>
118
+ <summary>Write a custom client</summary>
119
+
120
+ ```python
121
+ import requests
122
+ response = requests.post(
123
+ "http://127.0.0.1:8000/predict",
124
+ json={"input": 4.0}
125
+ )
126
+ ```
127
+ </details>
128
+
129
+ &nbsp;
130
+
131
+
132
+ # Featured examples
133
+ Use LitServe to deploy any model or AI service: (Gen AI, classical ML, embedding servers, LLMs, vision, audio, multi-modal systems, etc...)
134
+
135
+ <div align='center'>
136
+ <div width='200px'>
137
+ <video src="https://github.com/user-attachments/assets/5e73549a-bc0f-47a9-9d9c-5b54389be5de" width='200px' controls></video>
138
+ </div>
139
+ </div>
140
+
141
+
142
+ <pre>
143
+ <strong>Featured examples</strong><br>
144
+ <strong>Toy model:</strong> <a target="_blank" href="#define-a-server">Hello world</a>
145
+ <strong>LLMs:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-private-llama-3-8b-api">Llama 3 (8B)</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/openai-fault-tolerant-proxy-server">LLM Proxy server</a>
146
+ <strong>NLP:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-any-hugging-face-model-instantly">Hugging face</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-hugging-face-bert-model">BERT</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-text-embedding-api-with-litserve">Text embedding API</a>
147
+ <strong>Multimodal:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-open-ai-clip-with-litserve">OpenAI Clip</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-multi-modal-llm-with-minicpm">MiniCPM</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-phi3-5-vision-api-with-litserve">Phi-3.5 Vision Instruct</a>
148
+ <strong>Audio:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-open-ai-s-whisper-model">Whisper</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-music-generation-api-with-meta-s-audio-craft">AudioCraft</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-audio-generation-api">StableAudio</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-noise-cancellation-api-with-deepfilternet">Noise cancellation (DeepFilterNet)</a>
149
+ <strong>Vision:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-private-api-for-stable-diffusion-2">Stable diffusion 2</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-image-generation-api-with-auraflow">AuraFlow</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-image-generation-api-with-flux">Flux</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-super-resolution-image-api-with-aura-sr">Image super resolution (Aura SR)</a>
150
+ <strong>Speech:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-voice-clone-api-coqui-xtts-v2-model">Text-speech (XTTS V2)</a>
151
+ <strong>Classical ML:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-random-forest-with-litserve">Random forest</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-xgboost-with-litserve">XGBoost</a>
152
+ <strong>Miscellaneous:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-media-conversion-api-with-ffmpeg">Media conversion API (ffmpeg)</a>
153
+ </pre>
154
+
155
+ [Browse 100+ community-built templates](https://lightning.ai/studios?section=serving)
156
+
157
+ &nbsp;
158
+
159
+ # Features
160
+ State-of-the-art features:
161
+
162
+ ✅ [(2x)+ faster than plain FastAPI](#performance)
163
+ ✅ [Bring your own model](https://lightning.ai/docs/litserve/features/full-control)
164
+ ✅ [Build compound systems (1+ models)](https://lightning.ai/docs/litserve/features/compound-ai-systems)
165
+ ✅ [GPU autoscaling](https://lightning.ai/docs/litserve/features/gpu-inference)
166
+ ✅ [Batching](https://lightning.ai/docs/litserve/features/batching)
167
+ ✅ [Streaming](https://lightning.ai/docs/litserve/features/streaming)
168
+ ✅ [Worker autoscaling](https://lightning.ai/docs/litserve/features/autoscaling)
169
+ ✅ [Self-host on your machines](https://lightning.ai/docs/litserve/features/hosting-methods#host-on-your-own)
170
+ ✅ [Host fully managed on Lightning AI](https://lightning.ai/docs/litserve/features/hosting-methods#host-on-lightning-studios)
171
+ ✅ [Serve all models: (LLMs, vision, etc.)](https://lightning.ai/docs/litserve/examples)
172
+ ✅ [Scale to zero (serverless)](https://lightning.ai/docs/litserve/features/streaming)
173
+ ✅ [Supports PyTorch, JAX, TF, etc...](https://lightning.ai/docs/litserve/features/full-control)
174
+ ✅ [OpenAPI compliant](https://www.openapis.org/)
175
+ ✅ [Open AI compatibility](https://lightning.ai/docs/litserve/features/open-ai-spec)
176
+ ✅ [Authentication](https://lightning.ai/docs/litserve/features/authentication)
177
+
178
+
179
+ [10+ features...](https://lightning.ai/docs/litserve/features)
180
+
181
+ **Note:** We prioritize scalable, enterprise-level features over hype.
182
+
183
+ &nbsp;
184
+
185
+ # Performance
186
+ LitServe is designed for AI workloads. Specialized multi-worker handling delivers a minimum **2x speedup over FastAPI**.
187
+
188
+ Additional features like batching and GPU autoscaling can drive performance well beyond 2x, scaling efficiently to handle more simultaneous requests than FastAPI and TorchServe.
189
+
190
+ Reproduce the full benchmarks [here](https://lightning.ai/docs/litserve/home/benchmarks) (higher is better).
191
+
192
+ <div align="center">
193
+ <img alt="LitServe" src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/ls_charts_v6.png" width="1000px" style="max-width: 100%;">
194
+ </div>
195
+
196
+ These results are for image and text classification ML tasks. The performance relationships hold for other ML tasks (embedding, LLM serving, audio, segmentation, object detection, summarization etc...).
197
+
198
+ ***💡 Note on LLM serving:*** For high-performance LLM serving (like Ollama/VLLM), use [LitGPT](https://github.com/Lightning-AI/litgpt?tab=readme-ov-file#deploy-an-llm) or build your custom VLLM-like server with LitServe. Optimizations like kv-caching, which can be done with LitServe, are needed to maximize LLM performance.
199
+
200
+ &nbsp;
201
+
202
+ # Hosting options
203
+ LitServe can be hosted independently on your own machines or fully managed via Lightning Studios.
204
+
205
+ Self-hosting is ideal for hackers, students, and DIY developers, while fully managed hosting is ideal for enterprise developers needing easy autoscaling, security, release management, and 99.995% uptime and observability.
206
+
207
+ &nbsp;
208
+
209
+ <div align="center">
210
+ <a target="_blank" href="https://lightning.ai/lightning-ai/studios/litserve-hello-world">
211
+ <img src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/host-on-lightning.svg" alt="Host on Lightning"/>
212
+ </a>
213
+ </div>
214
+
215
+ &nbsp;
216
+
217
+ <div align='center'>
218
+
219
+ | Feature | Self Managed | Fully Managed on Studios |
220
+ |----------------------------------|-----------------------------------|-------------------------------------|
221
+ | Deployment | ✅ Do it yourself deployment | ✅ One-button cloud deploy |
222
+ | Load balancing | ❌ | ✅ |
223
+ | Autoscaling | ❌ | ✅ |
224
+ | Scale to zero | ❌ | ✅ |
225
+ | Multi-machine inference | ❌ | ✅ |
226
+ | Authentication | ❌ | ✅ |
227
+ | Own VPC | ❌ | ✅ |
228
+ | AWS, GCP | ❌ | ✅ |
229
+ | Use your own cloud commits | ❌ | ✅ |
230
+
231
+ </div>
232
+
233
+ &nbsp;
234
+
235
+ # Community
236
+ LitServe is a [community project accepting contributions](https://lightning.ai/docs/litserve/community) - Let's make the world's most advanced AI inference engine.
237
+
238
+ 💬 [Get help on Discord](https://discord.com/invite/XncpTy7DSt)
239
+ 📋 [License: Apache 2.0](https://github.com/Lightning-AI/litserve/blob/main/LICENSE)
{litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/__about__.py RENAMED
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = "0.2.0.dev0"
14
+ __version__ = "0.2.1"
15
15
  __author__ = "Lightning-AI et al."
16
16
  __author_email__ = "community@lightning.ai"
17
17
  __license__ = "Apache-2.0"
{litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/api.py RENAMED
@@ -85,11 +85,7 @@ class LitAPI(ABC):
85
85
 
86
86
  return numpy.stack(inputs)
87
87
 
88
- if self.stream:
89
- message = no_batch_unbatch_message_stream(self, inputs)
90
- else:
91
- message = no_batch_unbatch_message_no_stream(self, inputs)
92
- raise NotImplementedError(message)
88
+ return inputs
93
89
 
94
90
  @abstractmethod
95
91
  def predict(self, x, **kwargs):
@@ -97,18 +93,11 @@ class LitAPI(ABC):
97
93
  pass
98
94
 
99
95
  def _unbatch_no_stream(self, output):
100
- if hasattr(output, "__torch_function__") or output.__class__.__name__ == "ndarray":
101
- return list(output)
102
- message = no_batch_unbatch_message_no_stream(self, output)
103
- raise NotImplementedError(message)
96
+ return list(output)
104
97
 
105
98
  def _unbatch_stream(self, output_stream):
106
99
  for output in output_stream:
107
- if hasattr(output, "__torch_function__") or output.__class__.__name__ == "ndarray":
108
- yield list(output)
109
- else:
110
- message = no_batch_unbatch_message_no_stream(self, output)
111
- raise NotImplementedError(message)
100
+ yield list(output)
112
101
 
113
102
  def unbatch(self, output):
114
103
  """Convert a batched output to a list of outputs."""
@@ -147,7 +136,7 @@ class LitAPI(ABC):
147
136
  def device(self, value):
148
137
  self._device = value
149
138
 
150
- def sanitize(self, max_batch_size: int, spec: LitSpec):
139
+ def _sanitize(self, max_batch_size: int, spec: LitSpec):
151
140
  if self.stream:
152
141
  self._default_unbatch = self._unbatch_stream
153
142
  else:
{litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/examples/openai_spec_example.py RENAMED
@@ -45,6 +45,14 @@ class TestAPIWithToolCalls(TestAPI):
45
45
  )
46
46
 
47
47
 
48
+ class TestAPIWithStructuredOutput(TestAPI):
49
+ def encode_response(self, output):
50
+ yield ChatMessage(
51
+ role="assistant",
52
+ content='{"name": "Science Fair", "date": "Friday", "participants": ["Alice", "Bob"]}',
53
+ )
54
+
55
+
48
56
  class OpenAIBatchContext(ls.LitAPI):
49
57
  def setup(self, device: str) -> None:
50
58
  self.model = None
{litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/server.py RENAMED
@@ -449,7 +449,7 @@ class LitServer:
449
449
  self.api_path = api_path
450
450
  lit_api.stream = stream
451
451
  lit_api.request_timeout = timeout
452
- lit_api.sanitize(max_batch_size, spec=spec)
452
+ lit_api._sanitize(max_batch_size, spec=spec)
453
453
  self.app = FastAPI(lifespan=self.lifespan)
454
454
  self.app.response_queue_id = None
455
455
  self.response_queue_id = None
@@ -463,7 +463,6 @@ class LitServer:
463
463
  self.lit_spec = spec
464
464
  self.workers_per_device = workers_per_device
465
465
  self.max_batch_size = max_batch_size
466
- self.timeout = timeout
467
466
  self.batch_timeout = batch_timeout
468
467
  self.stream = stream
469
468
  self._connector = _Connector(accelerator=accelerator, devices=devices)
{litserve-0.2.0.dev0 → litserve-0.2.1}/src/litserve/specs/openai.py RENAMED
@@ -20,7 +20,7 @@ import typing
20
20
  import uuid
21
21
  from collections import deque
22
22
  from enum import Enum
23
- from typing import AsyncGenerator, Dict, Iterator, List, Literal, Optional, Union
23
+ from typing import Annotated, AsyncGenerator, Dict, Iterator, List, Literal, Optional, Union
24
24
 
25
25
  from fastapi import BackgroundTasks, HTTPException, Request, Response
26
26
  from fastapi.responses import StreamingResponse
@@ -105,6 +105,31 @@ class ToolCall(BaseModel):
105
105
  function: FunctionCall
106
106
 
107
107
 
108
+ class ResponseFormatText(BaseModel):
109
+ type: Literal["text"]
110
+
111
+
112
+ class ResponseFormatJSONObject(BaseModel):
113
+ type: Literal["json_object"]
114
+
115
+
116
+ class JSONSchema(BaseModel):
117
+ name: str
118
+ description: Optional[str] = None
119
+ schema_def: Optional[Dict[str, object]] = Field(None, alias="schema")
120
+ strict: Optional[bool] = False
121
+
122
+
123
+ class ResponseFormatJSONSchema(BaseModel):
124
+ json_schema: JSONSchema
125
+ type: Literal["json_schema"]
126
+
127
+
128
+ ResponseFormat = Annotated[
129
+ Union[ResponseFormatText, ResponseFormatJSONObject, ResponseFormatJSONSchema], "ResponseFormat"
130
+ ]
131
+
132
+
108
133
  class ChatMessage(BaseModel):
109
134
  role: str
110
135
  content: Union[str, List[Union[TextContent, ImageContent]]]
@@ -138,6 +163,7 @@ class ChatCompletionRequest(BaseModel):
138
163
  user: Optional[str] = None
139
164
  tools: Optional[List[Tool]] = None
140
165
  tool_choice: Optional[ToolChoice] = ToolChoice.auto
166
+ response_format: Optional[ResponseFormat] = None
141
167
 
142
168
 
143
169
  class ChatCompletionResponseChoice(BaseModel):