litserve 0.2.0.dev0__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. litserve-0.2.2/PKG-INFO +294 -0
  2. litserve-0.2.2/README.md +245 -0
  3. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve/__about__.py +1 -1
  4. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve/api.py +4 -48
  5. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve/examples/openai_spec_example.py +8 -0
  6. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve/server.py +22 -10
  7. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve/specs/openai.py +27 -1
  8. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve/utils.py +1 -21
  9. litserve-0.2.2/src/litserve.egg-info/PKG-INFO +294 -0
  10. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve.egg-info/requires.txt +1 -0
  11. litserve-0.2.0.dev0/PKG-INFO +0 -294
  12. litserve-0.2.0.dev0/README.md +0 -246
  13. litserve-0.2.0.dev0/src/litserve.egg-info/PKG-INFO +0 -294
  14. {litserve-0.2.0.dev0 → litserve-0.2.2}/LICENSE +0 -0
  15. {litserve-0.2.0.dev0 → litserve-0.2.2}/MANIFEST.in +0 -0
  16. {litserve-0.2.0.dev0 → litserve-0.2.2}/requirements.txt +0 -0
  17. {litserve-0.2.0.dev0 → litserve-0.2.2}/setup.cfg +0 -0
  18. {litserve-0.2.0.dev0 → litserve-0.2.2}/setup.py +0 -0
  19. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve/__init__.py +0 -0
  20. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve/connector.py +0 -0
  21. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve/examples/__init__.py +0 -0
  22. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve/examples/simple_example.py +0 -0
  23. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve/python_client.py +0 -0
  24. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve/specs/__init__.py +0 -0
  25. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve/specs/base.py +0 -0
  26. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve.egg-info/SOURCES.txt +0 -0
  27. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve.egg-info/dependency_links.txt +0 -0
  28. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve.egg-info/not-zip-safe +0 -0
  29. {litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve.egg-info/top_level.txt +0 -0
litserve-0.2.2/PKG-INFO ADDED
@@ -0,0 +1,294 @@
1
+ Metadata-Version: 2.1
2
+ Name: litserve
3
+ Version: 0.2.2
4
+ Summary: Lightweight AI server.
5
+ Home-page: https://github.com/Lightning-AI/litserve
6
+ Download-URL: https://github.com/Lightning-AI/litserve
7
+ Author: Lightning-AI et al.
8
+ Author-email: community@lightning.ai
9
+ License: Apache-2.0
10
+ Project-URL: Bug Tracker, https://github.com/Lightning-AI/litserve/issues
11
+ Project-URL: Documentation, https://lightning-ai.github.io/litserve/
12
+ Project-URL: Source Code, https://github.com/Lightning-AI/litserve
13
+ Keywords: deep learning,pytorch,AI
14
+ Classifier: Environment :: Console
15
+ Classifier: Natural Language :: English
16
+ Classifier: Development Status :: 3 - Alpha
17
+ Classifier: Intended Audience :: Developers
18
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
19
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
20
+ Classifier: License :: OSI Approved :: Apache Software License
21
+ Classifier: Operating System :: OS Independent
22
+ Classifier: Programming Language :: Python :: 3
23
+ Classifier: Programming Language :: Python :: 3.8
24
+ Classifier: Programming Language :: Python :: 3.9
25
+ Classifier: Programming Language :: Python :: 3.10
26
+ Classifier: Programming Language :: Python :: 3.11
27
+ Requires-Python: >=3.8
28
+ Description-Content-Type: text/markdown
29
+ License-File: LICENSE
30
+ Requires-Dist: fastapi>=0.100
31
+ Requires-Dist: httpx
32
+ Requires-Dist: uvicorn>=0.29.0
33
+ Provides-Extra: test
34
+ Requires-Dist: asgi-lifespan; extra == "test"
35
+ Requires-Dist: coverage[toml]>=7.5.3; extra == "test"
36
+ Requires-Dist: lightning>2.0.0; extra == "test"
37
+ Requires-Dist: mypy==1.11.1; extra == "test"
38
+ Requires-Dist: numpy<2.0; extra == "test"
39
+ Requires-Dist: openai>=1.12.0; extra == "test"
40
+ Requires-Dist: pillow; extra == "test"
41
+ Requires-Dist: psutil; extra == "test"
42
+ Requires-Dist: pytest-asyncio; extra == "test"
43
+ Requires-Dist: pytest-cov; extra == "test"
44
+ Requires-Dist: pytest>=8.0; extra == "test"
45
+ Requires-Dist: python-multipart; extra == "test"
46
+ Requires-Dist: requests; extra == "test"
47
+ Requires-Dist: torch>2.0.0; extra == "test"
48
+ Requires-Dist: transformers; extra == "test"
49
+
50
+ <div align='center'>
51
+
52
+ # Easily serve AI models Lightning fast ⚡
53
+
54
+ <img alt="Lightning" src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/ls_banner2.png" width="800px" style="max-width: 100%;">
55
+
56
+ &nbsp;
57
+
58
+ <strong>Lightning-fast serving engine for AI models.</strong>
59
+ Easy. Flexible. Enterprise-scale.
60
+ </div>
61
+
62
+ ----
63
+
64
+ **LitServe** is an easy-to-use, flexible serving engine for AI models built on FastAPI. Features like batching, streaming, and GPU autoscaling eliminate the need to rebuild a FastAPI server per model.
65
+
66
+ LitServe is at least [2x faster](#performance) than plain FastAPI due to AI-specific multi-worker handling.
67
+
68
+ <div align='center'>
69
+
70
+ <pre>
71
+ ✅ (2x)+ faster serving ✅ Easy to use ✅ Batching, Streaming
72
+ ✅ Bring your own model ✅ PyTorch/JAX/TF/... ✅ Built on FastAPI
73
+ ✅ GPU autoscaling ✅ Multi-modal ✅ Self-host or ⚡️ managed
74
+ </pre>
75
+
76
+ <div align='center'>
77
+
78
+ [![Discord](https://img.shields.io/discord/1077906959069626439?label=Get%20help%20on%20Discord)](https://discord.gg/VptPCZkGNa)
79
+ ![cpu-tests](https://github.com/Lightning-AI/litserve/actions/workflows/ci-testing.yml/badge.svg)
80
+ [![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/Lightning-AI/litserve/blob/main/LICENSE)
81
+
82
+ </div>
83
+ </div>
84
+ <div align="center">
85
+ <div style="text-align: center;">
86
+ <a target="_blank" href="#quick-start" style="margin: 0 10px;">Quick start</a> •
87
+ <a target="_blank" href="#featured-examples" style="margin: 0 10px;">Examples</a> •
88
+ <a target="_blank" href="#features" style="margin: 0 10px;">Features</a> •
89
+ <a target="_blank" href="#performance" style="margin: 0 10px;">Performance</a> •
90
+ <a target="_blank" href="#hosting-options" style="margin: 0 10px;">Hosting</a> •
91
+ <a target="_blank" href="https://lightning.ai/docs/litserve" style="margin: 0 10px;">Docs</a>
92
+ </div>
93
+ </div>
94
+
95
+ &nbsp;
96
+
97
+ <div align="center">
98
+ <a target="_blank" href="https://lightning.ai/docs/litserve/home/get-started">
99
+ <img src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/get-started-badge.svg" height="36px" alt="Get started"/>
100
+ </a>
101
+ </div>
102
+
103
+ &nbsp;
104
+
105
+ # Quick start
106
+
107
+ Install LitServe via pip ([more options](https://lightning.ai/docs/litserve/home/install)):
108
+
109
+ ```bash
110
+ pip install litserve
111
+ ```
112
+
113
+ ### Define a server
114
+ This toy example with 2 models (AI compound system) shows LitServe's flexibility ([see real examples](#examples)):
115
+
116
+ ```python
117
+ # server.py
118
+ import litserve as ls
119
+
120
+ # (STEP 1) - DEFINE THE API (compound AI system)
121
+ class SimpleLitAPI(ls.LitAPI):
122
+ def setup(self, device):
123
+ # setup is called once at startup. Build a compound AI system (1+ models), connect DBs, load data, etc...
124
+ self.model1 = lambda x: x**2
125
+ self.model2 = lambda x: x**3
126
+
127
+ def decode_request(self, request):
128
+ # Convert the request payload to model input.
129
+ return request["input"]
130
+
131
+ def predict(self, x):
132
+ # Easily build compound systems. Run inference and return the output.
133
+ squared = self.model1(x)
134
+ cubed = self.model2(x)
135
+ output = squared + cubed
136
+ return {"output": output}
137
+
138
+ def encode_response(self, output):
139
+ # Convert the model output to a response payload.
140
+ return {"output": output}
141
+
142
+ # (STEP 2) - START THE SERVER
143
+ if __name__ == "__main__":
144
+ # scale with advanced features (batching, GPUs, etc...)
145
+ server = ls.LitServer(SimpleLitAPI(), accelerator="auto", max_batch_size=1)
146
+ server.run(port=8000)
147
+ ```
148
+
149
+ Now run the server via the command-line
150
+
151
+ ```bash
152
+ python server.py
153
+ ```
154
+
155
+ ### Test the server
156
+
157
+ Run this command on your terminal to test the server:
158
+
159
+ ```bash
160
+ curl -X POST http://127.0.0.1:8000/predict -H "Content-Type: application/json" -d '{"input": 4.0}'
161
+ ```
162
+
163
+ <details>
164
+ <summary>Example of a custom Python client</summary>
165
+
166
+ ```python
167
+ import requests
168
+ response = requests.post(
169
+ "http://127.0.0.1:8000/predict",
170
+ json={"input": 4.0}
171
+ )
172
+ ```
173
+ </details>
174
+
175
+ ### Summary
176
+ - LitAPI lets you easily build complex AI systems with one or more models ([docs](https://lightning.ai/docs/litserve/api-reference/litapi)).
177
+ - Use the setup method for one-time tasks like connecting models, DBs, and loading data ([docs](https://lightning.ai/docs/litserve/api-reference/litapi#setup)).
178
+ - LitServer handles optimizations like batching, GPU autoscaling, streaming, etc... ([docs](https://lightning.ai/docs/litserve/api-reference/litserver)).
179
+ - Self host on your own machines or use Lightning Studios for a fully managed deployment ([learn more](#hosting-options)).
180
+
181
+ [Learn how to make this server 200x faster](https://lightning.ai/docs/litserve/home/speed-up-serving-by-200x).
182
+
183
+ &nbsp;
184
+
185
+ # Featured examples
186
+ Use LitServe to deploy any model or AI service: (Gen AI, classical ML, embedding servers, LLMs, vision, audio, multi-modal systems, etc...)
187
+
188
+ <div align='center'>
189
+ <div width='200px'>
190
+ <video src="https://github.com/user-attachments/assets/5e73549a-bc0f-47a9-9d9c-5b54389be5de" width='200px' controls></video>
191
+ </div>
192
+ </div>
193
+
194
+ ## Examples
195
+ <pre>
196
+ <strong>Toy model:</strong> <a target="_blank" href="#define-a-server">Hello world</a>
197
+ <strong>LLMs:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-private-llama-3-8b-api">Llama 3 (8B)</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/openai-fault-tolerant-proxy-server">LLM Proxy server</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-ai-agent-with-tool-use">Agent with tool use</a>
198
+ <strong>RAG:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-private-llama-3-1-rag-api">RAG API (LlamaIndex)</a>
199
+ <strong>NLP:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-any-hugging-face-model-instantly">Hugging face</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-hugging-face-bert-model">BERT</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-text-embedding-api-with-litserve">Text embedding API</a>
200
+ <strong>Multimodal:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-open-ai-clip-with-litserve">OpenAI Clip</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-multi-modal-llm-with-minicpm">MiniCPM</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-phi3-5-vision-api-with-litserve">Phi-3.5 Vision Instruct</a>
201
+ <strong>Audio:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-open-ai-s-whisper-model">Whisper</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-music-generation-api-with-meta-s-audio-craft">AudioCraft</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-audio-generation-api">StableAudio</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-noise-cancellation-api-with-deepfilternet">Noise cancellation (DeepFilterNet)</a>
202
+ <strong>Vision:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-private-api-for-stable-diffusion-2">Stable diffusion 2</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-image-generation-api-with-auraflow">AuraFlow</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-image-generation-api-with-flux">Flux</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-super-resolution-image-api-with-aura-sr">Image Super Resolution (Aura SR)</a>,
203
+ <a target="_blank" href="https://lightning.ai/bhimrajyadav/studios/deploy-background-removal-api-with-litserve">Background Removal</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-controlled-image-generation-api-controlnet">Control Stable Diffusion (ControlNet)</a>
204
+ <strong>Speech:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-voice-clone-api-coqui-xtts-v2-model">Text-speech (XTTS V2)</a>
205
+ <strong>Classical ML:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-random-forest-with-litserve">Random forest</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-xgboost-with-litserve">XGBoost</a>
206
+ <strong>Miscellaneous:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-media-conversion-api-with-ffmpeg">Media conversion API (ffmpeg)</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-both-pytorch-and-tensorflow-in-a-single-api">PyTorch + TensorFlow in one API</a>
207
+ </pre>
208
+ </pre>
209
+
210
+ [Browse 100+ community-built templates](https://lightning.ai/studios?section=serving)
211
+
212
+ &nbsp;
213
+
214
+ # Features
215
+ State-of-the-art features:
216
+
217
+ ✅ [(2x)+ faster than plain FastAPI](#performance)
218
+ ✅ [Bring your own model](https://lightning.ai/docs/litserve/features/full-control)
219
+ ✅ [Build compound systems (1+ models)](https://lightning.ai/docs/litserve/features/compound-ai-systems)
220
+ ✅ [GPU autoscaling](https://lightning.ai/docs/litserve/features/gpu-inference)
221
+ ✅ [Batching](https://lightning.ai/docs/litserve/features/batching)
222
+ ✅ [Streaming](https://lightning.ai/docs/litserve/features/streaming)
223
+ ✅ [Worker autoscaling](https://lightning.ai/docs/litserve/features/autoscaling)
224
+ ✅ [Self-host on your machines](https://lightning.ai/docs/litserve/features/hosting-methods#host-on-your-own)
225
+ ✅ [Host fully managed on Lightning AI](https://lightning.ai/docs/litserve/features/hosting-methods#host-on-lightning-studios)
226
+ ✅ [Serve all models: (LLMs, vision, etc.)](https://lightning.ai/docs/litserve/examples)
227
+ ✅ [Scale to zero (serverless)](https://lightning.ai/docs/litserve/features/streaming)
228
+ ✅ [Supports PyTorch, JAX, TF, etc...](https://lightning.ai/docs/litserve/features/full-control)
229
+ ✅ [OpenAPI compliant](https://www.openapis.org/)
230
+ ✅ [Open AI compatibility](https://lightning.ai/docs/litserve/features/open-ai-spec)
231
+ ✅ [Authentication](https://lightning.ai/docs/litserve/features/authentication)
232
+
233
+
234
+ [10+ features...](https://lightning.ai/docs/litserve/features)
235
+
236
+ **Note:** We prioritize scalable, enterprise-level features over hype.
237
+
238
+ &nbsp;
239
+
240
+ # Performance
241
+ LitServe is designed for AI workloads. Specialized multi-worker handling delivers a minimum **2x speedup over FastAPI**.
242
+
243
+ Additional features like batching and GPU autoscaling can drive performance well beyond 2x, scaling efficiently to handle more simultaneous requests than FastAPI and TorchServe.
244
+
245
+ Reproduce the full benchmarks [here](https://lightning.ai/docs/litserve/home/benchmarks) (higher is better).
246
+
247
+ <div align="center">
248
+ <img alt="LitServe" src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/ls_charts_v6.png" width="1000px" style="max-width: 100%;">
249
+ </div>
250
+
251
+ These results are for image and text classification ML tasks. The performance relationships hold for other ML tasks (embedding, LLM serving, audio, segmentation, object detection, summarization etc...).
252
+
253
+ ***💡 Note on LLM serving:*** For high-performance LLM serving (like Ollama/VLLM), use [LitGPT](https://github.com/Lightning-AI/litgpt?tab=readme-ov-file#deploy-an-llm) or build your custom VLLM-like server with LitServe. Optimizations like kv-caching, which can be done with LitServe, are needed to maximize LLM performance.
254
+
255
+ &nbsp;
256
+
257
+ # Hosting options
258
+ LitServe can be hosted independently on your own machines or fully managed via Lightning Studios.
259
+
260
+ Self-hosting is ideal for hackers, students, and DIY developers, while fully managed hosting is ideal for enterprise developers needing easy autoscaling, security, release management, and 99.995% uptime and observability.
261
+
262
+ &nbsp;
263
+
264
+ <div align="center">
265
+ <a target="_blank" href="https://lightning.ai/lightning-ai/studios/litserve-hello-world">
266
+ <img src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/host-on-lightning.svg" alt="Host on Lightning"/>
267
+ </a>
268
+ </div>
269
+
270
+ &nbsp;
271
+
272
+ <div align='center'>
273
+
274
+ | Feature | Self Managed | Fully Managed on Studios |
275
+ |----------------------------------|-----------------------------------|-------------------------------------|
276
+ | Deployment | ✅ Do it yourself deployment | ✅ One-button cloud deploy |
277
+ | Load balancing | ❌ | ✅ |
278
+ | Autoscaling | ❌ | ✅ |
279
+ | Scale to zero | ❌ | ✅ |
280
+ | Multi-machine inference | ❌ | ✅ |
281
+ | Authentication | ❌ | ✅ |
282
+ | Own VPC | ❌ | ✅ |
283
+ | AWS, GCP | ❌ | ✅ |
284
+ | Use your own cloud commits | ❌ | ✅ |
285
+
286
+ </div>
287
+
288
+ &nbsp;
289
+
290
+ # Community
291
+ LitServe is a [community project accepting contributions](https://lightning.ai/docs/litserve/community) - Let's make the world's most advanced AI inference engine.
292
+
293
+ 💬 [Get help on Discord](https://discord.com/invite/XncpTy7DSt)
294
+ 📋 [License: Apache 2.0](https://github.com/Lightning-AI/litserve/blob/main/LICENSE)
litserve-0.2.2/README.md ADDED
@@ -0,0 +1,245 @@
1
+ <div align='center'>
2
+
3
+ # Easily serve AI models Lightning fast ⚡
4
+
5
+ <img alt="Lightning" src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/ls_banner2.png" width="800px" style="max-width: 100%;">
6
+
7
+ &nbsp;
8
+
9
+ <strong>Lightning-fast serving engine for AI models.</strong>
10
+ Easy. Flexible. Enterprise-scale.
11
+ </div>
12
+
13
+ ----
14
+
15
+ **LitServe** is an easy-to-use, flexible serving engine for AI models built on FastAPI. Features like batching, streaming, and GPU autoscaling eliminate the need to rebuild a FastAPI server per model.
16
+
17
+ LitServe is at least [2x faster](#performance) than plain FastAPI due to AI-specific multi-worker handling.
18
+
19
+ <div align='center'>
20
+
21
+ <pre>
22
+ ✅ (2x)+ faster serving ✅ Easy to use ✅ Batching, Streaming
23
+ ✅ Bring your own model ✅ PyTorch/JAX/TF/... ✅ Built on FastAPI
24
+ ✅ GPU autoscaling ✅ Multi-modal ✅ Self-host or ⚡️ managed
25
+ </pre>
26
+
27
+ <div align='center'>
28
+
29
+ [![Discord](https://img.shields.io/discord/1077906959069626439?label=Get%20help%20on%20Discord)](https://discord.gg/VptPCZkGNa)
30
+ ![cpu-tests](https://github.com/Lightning-AI/litserve/actions/workflows/ci-testing.yml/badge.svg)
31
+ [![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/Lightning-AI/litserve/blob/main/LICENSE)
32
+
33
+ </div>
34
+ </div>
35
+ <div align="center">
36
+ <div style="text-align: center;">
37
+ <a target="_blank" href="#quick-start" style="margin: 0 10px;">Quick start</a> •
38
+ <a target="_blank" href="#featured-examples" style="margin: 0 10px;">Examples</a> •
39
+ <a target="_blank" href="#features" style="margin: 0 10px;">Features</a> •
40
+ <a target="_blank" href="#performance" style="margin: 0 10px;">Performance</a> •
41
+ <a target="_blank" href="#hosting-options" style="margin: 0 10px;">Hosting</a> •
42
+ <a target="_blank" href="https://lightning.ai/docs/litserve" style="margin: 0 10px;">Docs</a>
43
+ </div>
44
+ </div>
45
+
46
+ &nbsp;
47
+
48
+ <div align="center">
49
+ <a target="_blank" href="https://lightning.ai/docs/litserve/home/get-started">
50
+ <img src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/get-started-badge.svg" height="36px" alt="Get started"/>
51
+ </a>
52
+ </div>
53
+
54
+ &nbsp;
55
+
56
+ # Quick start
57
+
58
+ Install LitServe via pip ([more options](https://lightning.ai/docs/litserve/home/install)):
59
+
60
+ ```bash
61
+ pip install litserve
62
+ ```
63
+
64
+ ### Define a server
65
+ This toy example with 2 models (AI compound system) shows LitServe's flexibility ([see real examples](#examples)):
66
+
67
+ ```python
68
+ # server.py
69
+ import litserve as ls
70
+
71
+ # (STEP 1) - DEFINE THE API (compound AI system)
72
+ class SimpleLitAPI(ls.LitAPI):
73
+ def setup(self, device):
74
+ # setup is called once at startup. Build a compound AI system (1+ models), connect DBs, load data, etc...
75
+ self.model1 = lambda x: x**2
76
+ self.model2 = lambda x: x**3
77
+
78
+ def decode_request(self, request):
79
+ # Convert the request payload to model input.
80
+ return request["input"]
81
+
82
+ def predict(self, x):
83
+ # Easily build compound systems. Run inference and return the output.
84
+ squared = self.model1(x)
85
+ cubed = self.model2(x)
86
+ output = squared + cubed
87
+ return {"output": output}
88
+
89
+ def encode_response(self, output):
90
+ # Convert the model output to a response payload.
91
+ return {"output": output}
92
+
93
+ # (STEP 2) - START THE SERVER
94
+ if __name__ == "__main__":
95
+ # scale with advanced features (batching, GPUs, etc...)
96
+ server = ls.LitServer(SimpleLitAPI(), accelerator="auto", max_batch_size=1)
97
+ server.run(port=8000)
98
+ ```
99
+
100
+ Now run the server via the command-line
101
+
102
+ ```bash
103
+ python server.py
104
+ ```
105
+
106
+ ### Test the server
107
+
108
+ Run this command on your terminal to test the server:
109
+
110
+ ```bash
111
+ curl -X POST http://127.0.0.1:8000/predict -H "Content-Type: application/json" -d '{"input": 4.0}'
112
+ ```
113
+
114
+ <details>
115
+ <summary>Example of a custom Python client</summary>
116
+
117
+ ```python
118
+ import requests
119
+ response = requests.post(
120
+ "http://127.0.0.1:8000/predict",
121
+ json={"input": 4.0}
122
+ )
123
+ ```
124
+ </details>
125
+
126
+ ### Summary
127
+ - LitAPI lets you easily build complex AI systems with one or more models ([docs](https://lightning.ai/docs/litserve/api-reference/litapi)).
128
+ - Use the setup method for one-time tasks like connecting models, DBs, and loading data ([docs](https://lightning.ai/docs/litserve/api-reference/litapi#setup)).
129
+ - LitServer handles optimizations like batching, GPU autoscaling, streaming, etc... ([docs](https://lightning.ai/docs/litserve/api-reference/litserver)).
130
+ - Self host on your own machines or use Lightning Studios for a fully managed deployment ([learn more](#hosting-options)).
131
+
132
+ [Learn how to make this server 200x faster](https://lightning.ai/docs/litserve/home/speed-up-serving-by-200x).
133
+
134
+ &nbsp;
135
+
136
+ # Featured examples
137
+ Use LitServe to deploy any model or AI service: (Gen AI, classical ML, embedding servers, LLMs, vision, audio, multi-modal systems, etc...)
138
+
139
+ <div align='center'>
140
+ <div width='200px'>
141
+ <video src="https://github.com/user-attachments/assets/5e73549a-bc0f-47a9-9d9c-5b54389be5de" width='200px' controls></video>
142
+ </div>
143
+ </div>
144
+
145
+ ## Examples
146
+ <pre>
147
+ <strong>Toy model:</strong> <a target="_blank" href="#define-a-server">Hello world</a>
148
+ <strong>LLMs:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-private-llama-3-8b-api">Llama 3 (8B)</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/openai-fault-tolerant-proxy-server">LLM Proxy server</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-ai-agent-with-tool-use">Agent with tool use</a>
149
+ <strong>RAG:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-private-llama-3-1-rag-api">RAG API (LlamaIndex)</a>
150
+ <strong>NLP:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-any-hugging-face-model-instantly">Hugging face</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-hugging-face-bert-model">BERT</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-text-embedding-api-with-litserve">Text embedding API</a>
151
+ <strong>Multimodal:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-open-ai-clip-with-litserve">OpenAI Clip</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-multi-modal-llm-with-minicpm">MiniCPM</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-phi3-5-vision-api-with-litserve">Phi-3.5 Vision Instruct</a>
152
+ <strong>Audio:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-open-ai-s-whisper-model">Whisper</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-music-generation-api-with-meta-s-audio-craft">AudioCraft</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-audio-generation-api">StableAudio</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-noise-cancellation-api-with-deepfilternet">Noise cancellation (DeepFilterNet)</a>
153
+ <strong>Vision:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-private-api-for-stable-diffusion-2">Stable diffusion 2</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-image-generation-api-with-auraflow">AuraFlow</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-image-generation-api-with-flux">Flux</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-super-resolution-image-api-with-aura-sr">Image Super Resolution (Aura SR)</a>,
154
+ <a target="_blank" href="https://lightning.ai/bhimrajyadav/studios/deploy-background-removal-api-with-litserve">Background Removal</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-controlled-image-generation-api-controlnet">Control Stable Diffusion (ControlNet)</a>
155
+ <strong>Speech:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-voice-clone-api-coqui-xtts-v2-model">Text-speech (XTTS V2)</a>
156
+ <strong>Classical ML:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-random-forest-with-litserve">Random forest</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-xgboost-with-litserve">XGBoost</a>
157
+ <strong>Miscellaneous:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-media-conversion-api-with-ffmpeg">Media conversion API (ffmpeg)</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-both-pytorch-and-tensorflow-in-a-single-api">PyTorch + TensorFlow in one API</a>
158
+ </pre>
159
+ </pre>
160
+
161
+ [Browse 100+ community-built templates](https://lightning.ai/studios?section=serving)
162
+
163
+ &nbsp;
164
+
165
+ # Features
166
+ State-of-the-art features:
167
+
168
+ ✅ [(2x)+ faster than plain FastAPI](#performance)
169
+ ✅ [Bring your own model](https://lightning.ai/docs/litserve/features/full-control)
170
+ ✅ [Build compound systems (1+ models)](https://lightning.ai/docs/litserve/features/compound-ai-systems)
171
+ ✅ [GPU autoscaling](https://lightning.ai/docs/litserve/features/gpu-inference)
172
+ ✅ [Batching](https://lightning.ai/docs/litserve/features/batching)
173
+ ✅ [Streaming](https://lightning.ai/docs/litserve/features/streaming)
174
+ ✅ [Worker autoscaling](https://lightning.ai/docs/litserve/features/autoscaling)
175
+ ✅ [Self-host on your machines](https://lightning.ai/docs/litserve/features/hosting-methods#host-on-your-own)
176
+ ✅ [Host fully managed on Lightning AI](https://lightning.ai/docs/litserve/features/hosting-methods#host-on-lightning-studios)
177
+ ✅ [Serve all models: (LLMs, vision, etc.)](https://lightning.ai/docs/litserve/examples)
178
+ ✅ [Scale to zero (serverless)](https://lightning.ai/docs/litserve/features/streaming)
179
+ ✅ [Supports PyTorch, JAX, TF, etc...](https://lightning.ai/docs/litserve/features/full-control)
180
+ ✅ [OpenAPI compliant](https://www.openapis.org/)
181
+ ✅ [Open AI compatibility](https://lightning.ai/docs/litserve/features/open-ai-spec)
182
+ ✅ [Authentication](https://lightning.ai/docs/litserve/features/authentication)
183
+
184
+
185
+ [10+ features...](https://lightning.ai/docs/litserve/features)
186
+
187
+ **Note:** We prioritize scalable, enterprise-level features over hype.
188
+
189
+ &nbsp;
190
+
191
+ # Performance
192
+ LitServe is designed for AI workloads. Specialized multi-worker handling delivers a minimum **2x speedup over FastAPI**.
193
+
194
+ Additional features like batching and GPU autoscaling can drive performance well beyond 2x, scaling efficiently to handle more simultaneous requests than FastAPI and TorchServe.
195
+
196
+ Reproduce the full benchmarks [here](https://lightning.ai/docs/litserve/home/benchmarks) (higher is better).
197
+
198
+ <div align="center">
199
+ <img alt="LitServe" src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/ls_charts_v6.png" width="1000px" style="max-width: 100%;">
200
+ </div>
201
+
202
+ These results are for image and text classification ML tasks. The performance relationships hold for other ML tasks (embedding, LLM serving, audio, segmentation, object detection, summarization etc...).
203
+
204
+ ***💡 Note on LLM serving:*** For high-performance LLM serving (like Ollama/VLLM), use [LitGPT](https://github.com/Lightning-AI/litgpt?tab=readme-ov-file#deploy-an-llm) or build your custom VLLM-like server with LitServe. Optimizations like kv-caching, which can be done with LitServe, are needed to maximize LLM performance.
205
+
206
+ &nbsp;
207
+
208
+ # Hosting options
209
+ LitServe can be hosted independently on your own machines or fully managed via Lightning Studios.
210
+
211
+ Self-hosting is ideal for hackers, students, and DIY developers, while fully managed hosting is ideal for enterprise developers needing easy autoscaling, security, release management, and 99.995% uptime and observability.
212
+
213
+ &nbsp;
214
+
215
+ <div align="center">
216
+ <a target="_blank" href="https://lightning.ai/lightning-ai/studios/litserve-hello-world">
217
+ <img src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/host-on-lightning.svg" alt="Host on Lightning"/>
218
+ </a>
219
+ </div>
220
+
221
+ &nbsp;
222
+
223
+ <div align='center'>
224
+
225
+ | Feature | Self Managed | Fully Managed on Studios |
226
+ |----------------------------------|-----------------------------------|-------------------------------------|
227
+ | Deployment | ✅ Do it yourself deployment | ✅ One-button cloud deploy |
228
+ | Load balancing | ❌ | ✅ |
229
+ | Autoscaling | ❌ | ✅ |
230
+ | Scale to zero | ❌ | ✅ |
231
+ | Multi-machine inference | ❌ | ✅ |
232
+ | Authentication | ❌ | ✅ |
233
+ | Own VPC | ❌ | ✅ |
234
+ | AWS, GCP | ❌ | ✅ |
235
+ | Use your own cloud commits | ❌ | ✅ |
236
+
237
+ </div>
238
+
239
+ &nbsp;
240
+
241
+ # Community
242
+ LitServe is a [community project accepting contributions](https://lightning.ai/docs/litserve/community) - Let's make the world's most advanced AI inference engine.
243
+
244
+ 💬 [Get help on Discord](https://discord.com/invite/XncpTy7DSt)
245
+ 📋 [License: Apache 2.0](https://github.com/Lightning-AI/litserve/blob/main/LICENSE)
{litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve/__about__.py RENAMED
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = "0.2.0.dev0"
14
+ __version__ = "0.2.2"
15
15
  __author__ = "Lightning-AI et al."
16
16
  __author_email__ = "community@lightning.ai"
17
17
  __license__ = "Apache-2.0"
{litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve/api.py RENAMED
@@ -21,39 +21,6 @@ from pydantic import BaseModel
21
21
  from litserve.specs.base import LitSpec
22
22
 
23
23
 
24
- def no_batch_unbatch_message_no_stream(obj, data):
25
- return f"""
26
- You set `max_batch_size > 1`, but the default implementation for batch() and unbatch() only supports
27
- PyTorch tensors or NumPy ndarrays, while we found {type(data)}.
28
- Please implement these two methods in {obj.__class__.__name__}.
29
-
30
- Example:
31
-
32
- def batch(self, inputs):
33
- return np.stack(inputs)
34
-
35
- def unbatch(self, output):
36
- return list(output)
37
- """
38
-
39
-
40
- def no_batch_unbatch_message_stream(obj, data):
41
- return f"""
42
- You set `max_batch_size > 1`, but the default implementation for batch() and unbatch() only supports
43
- PyTorch tensors or NumPy ndarrays, while we found {type(data)}.
44
- Please implement these two methods in {obj.__class__.__name__}.
45
-
46
- Example:
47
-
48
- def batch(self, inputs):
49
- return np.stack(inputs)
50
-
51
- def unbatch(self, output):
52
- for out in output:
53
- yield list(out)
54
- """
55
-
56
-
57
24
  class LitAPI(ABC):
58
25
  _stream: bool = False
59
26
  _default_unbatch: callable = None
@@ -85,11 +52,7 @@ class LitAPI(ABC):
85
52
 
86
53
  return numpy.stack(inputs)
87
54
 
88
- if self.stream:
89
- message = no_batch_unbatch_message_stream(self, inputs)
90
- else:
91
- message = no_batch_unbatch_message_no_stream(self, inputs)
92
- raise NotImplementedError(message)
55
+ return inputs
93
56
 
94
57
  @abstractmethod
95
58
  def predict(self, x, **kwargs):
@@ -97,18 +60,11 @@ class LitAPI(ABC):
97
60
  pass
98
61
 
99
62
  def _unbatch_no_stream(self, output):
100
- if hasattr(output, "__torch_function__") or output.__class__.__name__ == "ndarray":
101
- return list(output)
102
- message = no_batch_unbatch_message_no_stream(self, output)
103
- raise NotImplementedError(message)
63
+ return list(output)
104
64
 
105
65
  def _unbatch_stream(self, output_stream):
106
66
  for output in output_stream:
107
- if hasattr(output, "__torch_function__") or output.__class__.__name__ == "ndarray":
108
- yield list(output)
109
- else:
110
- message = no_batch_unbatch_message_no_stream(self, output)
111
- raise NotImplementedError(message)
67
+ yield list(output)
112
68
 
113
69
  def unbatch(self, output):
114
70
  """Convert a batched output to a list of outputs."""
@@ -147,7 +103,7 @@ class LitAPI(ABC):
147
103
  def device(self, value):
148
104
  self._device = value
149
105
 
150
- def sanitize(self, max_batch_size: int, spec: LitSpec):
106
+ def _sanitize(self, max_batch_size: int, spec: LitSpec):
151
107
  if self.stream:
152
108
  self._default_unbatch = self._unbatch_stream
153
109
  else:
{litserve-0.2.0.dev0 → litserve-0.2.2}/src/litserve/examples/openai_spec_example.py RENAMED
@@ -45,6 +45,14 @@ class TestAPIWithToolCalls(TestAPI):
45
45
  )
46
46
 
47
47
 
48
+ class TestAPIWithStructuredOutput(TestAPI):
49
+ def encode_response(self, output):
50
+ yield ChatMessage(
51
+ role="assistant",
52
+ content='{"name": "Science Fair", "date": "Friday", "participants": ["Alice", "Bob"]}',
53
+ )
54
+
55
+
48
56
  class OpenAIBatchContext(ls.LitAPI):
49
57
  def setup(self, device: str) -> None:
50
58
  self.model = None