litserve 0.0.0.dev2__tar.gz → 0.2.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. litserve-0.2.6/PKG-INFO +317 -0
  2. litserve-0.2.6/README.md +248 -0
  3. litserve-0.2.6/requirements.txt +3 -0
  4. {litserve-0.0.0.dev2 → litserve-0.2.6}/setup.py +18 -0
  5. litserve-0.2.6/src/litserve/__about__.py +30 -0
  6. litserve-0.2.6/src/litserve/__init__.py +35 -0
  7. litserve-0.2.6/src/litserve/__main__.py +64 -0
  8. litserve-0.2.6/src/litserve/api.py +145 -0
  9. litserve-0.2.6/src/litserve/callbacks/__init__.py +3 -0
  10. litserve-0.2.6/src/litserve/callbacks/base.py +90 -0
  11. litserve-0.2.6/src/litserve/callbacks/defaults/__init__.py +3 -0
  12. litserve-0.2.6/src/litserve/callbacks/defaults/metric_callback.py +26 -0
  13. litserve-0.2.6/src/litserve/connector.py +123 -0
  14. litserve-0.2.6/src/litserve/docker_builder.py +174 -0
  15. litserve-0.2.6/src/litserve/loggers.py +170 -0
  16. litserve-0.2.6/src/litserve/loops/__init__.py +38 -0
  17. litserve-0.2.6/src/litserve/loops/base.py +362 -0
  18. litserve-0.2.6/src/litserve/loops/continuous_batching_loop.py +286 -0
  19. litserve-0.2.6/src/litserve/loops/loops.py +96 -0
  20. litserve-0.2.6/src/litserve/loops/simple_loops.py +253 -0
  21. litserve-0.2.6/src/litserve/loops/streaming_loops.py +234 -0
  22. litserve-0.2.6/src/litserve/middlewares.py +70 -0
  23. litserve-0.2.6/src/litserve/python_client.py +19 -0
  24. litserve-0.2.6/src/litserve/schema/__init__.py +0 -0
  25. litserve-0.2.6/src/litserve/schema/image.py +75 -0
  26. litserve-0.2.6/src/litserve/server.py +635 -0
  27. litserve-0.2.6/src/litserve/specs/__init__.py +4 -0
  28. litserve-0.2.6/src/litserve/specs/base.py +59 -0
  29. litserve-0.2.6/src/litserve/specs/openai.py +456 -0
  30. litserve-0.2.6/src/litserve/specs/openai_embedding.py +181 -0
  31. litserve-0.2.6/src/litserve/test_examples/__init__.py +20 -0
  32. litserve-0.2.6/src/litserve/test_examples/openai_embedding_spec_example.py +49 -0
  33. litserve-0.2.6/src/litserve/test_examples/openai_spec_example.py +132 -0
  34. litserve-0.2.6/src/litserve/test_examples/simple_example.py +102 -0
  35. litserve-0.2.6/src/litserve/utils.py +148 -0
  36. litserve-0.2.6/src/litserve/zmq_queue.py +179 -0
  37. litserve-0.2.6/src/litserve.egg-info/PKG-INFO +317 -0
  38. litserve-0.2.6/src/litserve.egg-info/SOURCES.txt +44 -0
  39. litserve-0.2.6/src/litserve.egg-info/entry_points.txt +2 -0
  40. litserve-0.2.6/src/litserve.egg-info/requires.txt +27 -0
  41. litserve-0.0.0.dev2/PKG-INFO +0 -259
  42. litserve-0.0.0.dev2/README.md +0 -219
  43. litserve-0.0.0.dev2/requirements.txt +0 -2
  44. litserve-0.0.0.dev2/src/litserve/__about__.py +0 -17
  45. litserve-0.0.0.dev2/src/litserve/__init__.py +0 -5
  46. litserve-0.0.0.dev2/src/litserve/api.py +0 -25
  47. litserve-0.0.0.dev2/src/litserve/server.py +0 -174
  48. litserve-0.0.0.dev2/src/litserve.egg-info/PKG-INFO +0 -259
  49. litserve-0.0.0.dev2/src/litserve.egg-info/SOURCES.txt +0 -15
  50. litserve-0.0.0.dev2/src/litserve.egg-info/requires.txt +0 -11
  51. {litserve-0.0.0.dev2 → litserve-0.2.6}/LICENSE +0 -0
  52. {litserve-0.0.0.dev2 → litserve-0.2.6}/MANIFEST.in +0 -0
  53. {litserve-0.0.0.dev2 → litserve-0.2.6}/setup.cfg +0 -0
  54. {litserve-0.0.0.dev2 → litserve-0.2.6}/src/litserve.egg-info/dependency_links.txt +0 -0
  55. {litserve-0.0.0.dev2 → litserve-0.2.6}/src/litserve.egg-info/not-zip-safe +0 -0
  56. {litserve-0.0.0.dev2 → litserve-0.2.6}/src/litserve.egg-info/top_level.txt +0 -0
litserve-0.2.6/PKG-INFO ADDED
@@ -0,0 +1,317 @@
1
+ Metadata-Version: 2.2
2
+ Name: litserve
3
+ Version: 0.2.6
4
+ Summary: Lightweight AI server.
5
+ Home-page: https://github.com/Lightning-AI/litserve
6
+ Download-URL: https://github.com/Lightning-AI/litserve
7
+ Author: Lightning-AI et al.
8
+ Author-email: community@lightning.ai
9
+ License: Apache-2.0
10
+ Project-URL: Bug Tracker, https://github.com/Lightning-AI/litserve/issues
11
+ Project-URL: Documentation, https://lightning-ai.github.io/litserve/
12
+ Project-URL: Source Code, https://github.com/Lightning-AI/litserve
13
+ Keywords: deep learning,pytorch,AI
14
+ Classifier: Environment :: Console
15
+ Classifier: Natural Language :: English
16
+ Classifier: Development Status :: 3 - Alpha
17
+ Classifier: Intended Audience :: Developers
18
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
19
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
20
+ Classifier: License :: OSI Approved :: Apache Software License
21
+ Classifier: Operating System :: OS Independent
22
+ Classifier: Programming Language :: Python :: 3
23
+ Classifier: Programming Language :: Python :: 3.8
24
+ Classifier: Programming Language :: Python :: 3.9
25
+ Classifier: Programming Language :: Python :: 3.10
26
+ Classifier: Programming Language :: Python :: 3.11
27
+ Requires-Python: >=3.8
28
+ Description-Content-Type: text/markdown
29
+ License-File: LICENSE
30
+ Requires-Dist: fastapi>=0.100
31
+ Requires-Dist: uvicorn[standard]>=0.29.0
32
+ Requires-Dist: pyzmq>=22.0.0
33
+ Provides-Extra: perf
34
+ Requires-Dist: jsonargparse; extra == "perf"
35
+ Requires-Dist: tenacity; extra == "perf"
36
+ Requires-Dist: uvloop; extra == "perf"
37
+ Provides-Extra: test
38
+ Requires-Dist: asgi-lifespan; extra == "test"
39
+ Requires-Dist: coverage[toml]>=7.5.3; extra == "test"
40
+ Requires-Dist: httpx>=0.27.0; extra == "test"
41
+ Requires-Dist: lightning>2.0.0; extra == "test"
42
+ Requires-Dist: mypy==1.11.2; extra == "test"
43
+ Requires-Dist: numpy<2.0; extra == "test"
44
+ Requires-Dist: openai>=1.12.0; extra == "test"
45
+ Requires-Dist: pillow; extra == "test"
46
+ Requires-Dist: psutil; extra == "test"
47
+ Requires-Dist: pytest-asyncio; extra == "test"
48
+ Requires-Dist: pytest-cov; extra == "test"
49
+ Requires-Dist: pytest-retry>=1.6.3; extra == "test"
50
+ Requires-Dist: pytest>=8.0; extra == "test"
51
+ Requires-Dist: python-multipart; extra == "test"
52
+ Requires-Dist: requests; extra == "test"
53
+ Requires-Dist: torch>2.0.0; extra == "test"
54
+ Requires-Dist: transformers; extra == "test"
55
+ Dynamic: author
56
+ Dynamic: author-email
57
+ Dynamic: classifier
58
+ Dynamic: description
59
+ Dynamic: description-content-type
60
+ Dynamic: download-url
61
+ Dynamic: home-page
62
+ Dynamic: keywords
63
+ Dynamic: license
64
+ Dynamic: project-url
65
+ Dynamic: provides-extra
66
+ Dynamic: requires-dist
67
+ Dynamic: requires-python
68
+ Dynamic: summary
69
+
70
+ <div align='center'>
71
+
72
+ # Easily serve AI models Lightning fast ⚡
73
+
74
+ <img alt="Lightning" src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/ls_banner2.png" width="800px" style="max-width: 100%;">
75
+
76
+ &nbsp;
77
+
78
+ <strong>Lightning-fast serving engine for AI models.</strong>
79
+ Easy. Flexible. Enterprise-scale.
80
+ </div>
81
+
82
+ ----
83
+
84
+ **LitServe** is an easy-to-use, flexible serving engine for AI models built on FastAPI. It augments FastAPI with features like batching, streaming, and GPU autoscaling eliminate the need to rebuild a FastAPI server per model.
85
+
86
+ LitServe is at least [2x faster](#performance) than plain FastAPI due to AI-specific multi-worker handling.
87
+
88
+ <div align='center'>
89
+
90
+ <pre>
91
+ ✅ (2x)+ faster serving ✅ Easy to use ✅ LLMs, non LLMs and more
92
+ ✅ Bring your own model ✅ PyTorch/JAX/TF/... ✅ Built on FastAPI
93
+ ✅ GPU autoscaling ✅ Batching, Streaming ✅ Self-host or ⚡️ managed
94
+ ✅ Compound AI ✅ Integrate with vLLM and more
95
+ </pre>
96
+
97
+ <div align='center'>
98
+
99
+ [![Discord](https://img.shields.io/discord/1077906959069626439?label=Get%20help%20on%20Discord)](https://discord.gg/WajDThKAur)
100
+ ![cpu-tests](https://github.com/Lightning-AI/litserve/actions/workflows/ci-testing.yml/badge.svg)
101
+ [![codecov](https://codecov.io/gh/Lightning-AI/litserve/graph/badge.svg?token=SmzX8mnKlA)](https://codecov.io/gh/Lightning-AI/litserve)
102
+ [![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/Lightning-AI/litserve/blob/main/LICENSE)
103
+
104
+ </div>
105
+ </div>
106
+ <div align="center">
107
+ <div style="text-align: center;">
108
+ <a target="_blank" href="#quick-start" style="margin: 0 10px;">Quick start</a> •
109
+ <a target="_blank" href="#featured-examples" style="margin: 0 10px;">Examples</a> •
110
+ <a target="_blank" href="#features" style="margin: 0 10px;">Features</a> •
111
+ <a target="_blank" href="#performance" style="margin: 0 10px;">Performance</a> •
112
+ <a target="_blank" href="#hosting-options" style="margin: 0 10px;">Hosting</a> •
113
+ <a target="_blank" href="https://lightning.ai/docs/litserve" style="margin: 0 10px;">Docs</a>
114
+ </div>
115
+ </div>
116
+
117
+ &nbsp;
118
+
119
+ <div align="center">
120
+ <a target="_blank" href="https://lightning.ai/docs/litserve/home/get-started">
121
+ <img src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/get-started-badge.svg" height="36px" alt="Get started"/>
122
+ </a>
123
+ </div>
124
+
125
+ &nbsp;
126
+
127
+ # Quick start
128
+
129
+ Install LitServe via pip ([more options](https://lightning.ai/docs/litserve/home/install)):
130
+
131
+ ```bash
132
+ pip install litserve
133
+ ```
134
+
135
+ ### Define a server
136
+ This toy example with 2 models (AI compound system) shows LitServe's flexibility ([see real examples](#examples)):
137
+
138
+ ```python
139
+ # server.py
140
+ import litserve as ls
141
+
142
+ # (STEP 1) - DEFINE THE API (compound AI system)
143
+ class SimpleLitAPI(ls.LitAPI):
144
+ def setup(self, device):
145
+ # setup is called once at startup. Build a compound AI system (1+ models), connect DBs, load data, etc...
146
+ self.model1 = lambda x: x**2
147
+ self.model2 = lambda x: x**3
148
+
149
+ def decode_request(self, request):
150
+ # Convert the request payload to model input.
151
+ return request["input"]
152
+
153
+ def predict(self, x):
154
+ # Easily build compound systems. Run inference and return the output.
155
+ squared = self.model1(x)
156
+ cubed = self.model2(x)
157
+ output = squared + cubed
158
+ return {"output": output}
159
+
160
+ def encode_response(self, output):
161
+ # Convert the model output to a response payload.
162
+ return {"output": output}
163
+
164
+ # (STEP 2) - START THE SERVER
165
+ if __name__ == "__main__":
166
+ # scale with advanced features (batching, GPUs, etc...)
167
+ server = ls.LitServer(SimpleLitAPI(), accelerator="auto", max_batch_size=1)
168
+ server.run(port=8000)
169
+ ```
170
+
171
+ Now run the server via the command-line
172
+
173
+ ```bash
174
+ python server.py
175
+ ```
176
+
177
+ ### Test the server
178
+ Run the auto-generated test client:
179
+ ```bash
180
+ python client.py
181
+ ```
182
+
183
+ Or use this terminal command:
184
+ ```bash
185
+ curl -X POST http://127.0.0.1:8000/predict -H "Content-Type: application/json" -d '{"input": 4.0}'
186
+ ```
187
+
188
+ ### LLM serving
189
+ LitServe isn’t *just* for LLMs like vLLM or Ollama; it serves any AI model with full control over internals ([learn more](https://lightning.ai/docs/litserve/features/serve-llms)).
190
+ For easy LLM serving, integrate [vLLM with LitServe](https://lightning.ai/lightning-ai/studios/deploy-a-private-llama-3-2-rag-api), or use [LitGPT](https://github.com/Lightning-AI/litgpt?tab=readme-ov-file#deploy-an-llm) (built on LitServe).
191
+
192
+ ```
193
+ litgpt serve microsoft/phi-2
194
+ ```
195
+
196
+ ### Summary
197
+ - LitAPI lets you easily build complex AI systems with one or more models ([docs](https://lightning.ai/docs/litserve/api-reference/litapi)).
198
+ - Use the setup method for one-time tasks like connecting models, DBs, and loading data ([docs](https://lightning.ai/docs/litserve/api-reference/litapi#setup)).
199
+ - LitServer handles optimizations like batching, GPU autoscaling, streaming, etc... ([docs](https://lightning.ai/docs/litserve/api-reference/litserver)).
200
+ - Self host on your own machines or use Lightning Studios for a fully managed deployment ([learn more](#hosting-options)).
201
+
202
+ [Learn how to make this server 200x faster](https://lightning.ai/docs/litserve/home/speed-up-serving-by-200x).
203
+
204
+ &nbsp;
205
+
206
+ # Featured examples
207
+ Use LitServe to deploy any model or AI service: (Compound AI, Gen AI, classic ML, embeddings, LLMs, vision, audio, etc...)
208
+
209
+ <div align='center'>
210
+ <div width='200px'>
211
+ <video src="https://github.com/user-attachments/assets/5e73549a-bc0f-47a9-9d9c-5b54389be5de" width='200px' controls></video>
212
+ </div>
213
+ </div>
214
+
215
+ ## Examples
216
+ <pre>
217
+ <strong>Toy model:</strong> <a target="_blank" href="#define-a-server">Hello world</a>
218
+ <strong>LLMs:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-llama-3-2-vision-with-litserve">Llama 3.2</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/openai-fault-tolerant-proxy-server">LLM Proxy server</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-ai-agent-with-tool-use">Agent with tool use</a>
219
+ <strong>RAG:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-private-llama-3-2-rag-api">vLLM RAG (Llama 3.2)</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-private-llama-3-1-rag-api">RAG API (LlamaIndex)</a>
220
+ <strong>NLP:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-any-hugging-face-model-instantly">Hugging face</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-hugging-face-bert-model">BERT</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-text-embedding-api-with-litserve">Text embedding API</a>
221
+ <strong>Multimodal:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-open-ai-clip-with-litserve">OpenAI Clip</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-multi-modal-llm-with-minicpm">MiniCPM</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-phi3-5-vision-api-with-litserve">Phi-3.5 Vision Instruct</a>, <a target="_blank" href="https://lightning.ai/bhimrajyadav/studios/deploy-and-chat-with-qwen2-vl-using-litserve">Qwen2-VL</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-multi-modal-llm-with-pixtral">Pixtral</a>
222
+ <strong>Audio:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-open-ai-s-whisper-model">Whisper</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-music-generation-api-with-meta-s-audio-craft">AudioCraft</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-audio-generation-api">StableAudio</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-noise-cancellation-api-with-deepfilternet">Noise cancellation (DeepFilterNet)</a>
223
+ <strong>Vision:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-private-api-for-stable-diffusion-2">Stable diffusion 2</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-image-generation-api-with-auraflow">AuraFlow</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-image-generation-api-with-flux">Flux</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-super-resolution-image-api-with-aura-sr">Image Super Resolution (Aura SR)</a>,
224
+ <a target="_blank" href="https://lightning.ai/bhimrajyadav/studios/deploy-background-removal-api-with-litserve">Background Removal</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-controlled-image-generation-api-controlnet">Control Stable Diffusion (ControlNet)</a>
225
+ <strong>Speech:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-voice-clone-api-coqui-xtts-v2-model">Text-speech (XTTS V2)</a>, <a target="_blank" href="https://lightning.ai/bhimrajyadav/studios/deploy-a-speech-generation-api-using-parler-tts-powered-by-litserve">Parler-TTS</a>
226
+ <strong>Classical ML:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-random-forest-with-litserve">Random forest</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-xgboost-with-litserve">XGBoost</a>
227
+ <strong>Miscellaneous:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-media-conversion-api-with-ffmpeg">Media conversion API (ffmpeg)</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-both-pytorch-and-tensorflow-in-a-single-api">PyTorch + TensorFlow in one API</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/openai-fault-tolerant-proxy-server">LLM proxy server</a>
228
+ </pre>
229
+ </pre>
230
+
231
+ [Browse 100+ community-built templates](https://lightning.ai/studios?section=serving)
232
+
233
+ &nbsp;
234
+
235
+ # Features
236
+ State-of-the-art features:
237
+
238
+ ✅ [(2x)+ faster than plain FastAPI](#performance)
239
+ ✅ [Bring your own model](https://lightning.ai/docs/litserve/features/full-control)
240
+ ✅ [Build compound systems (1+ models)](https://lightning.ai/docs/litserve/home)
241
+ ✅ [GPU autoscaling](https://lightning.ai/docs/litserve/features/gpu-inference)
242
+ ✅ [Batching](https://lightning.ai/docs/litserve/features/batching)
243
+ ✅ [Streaming](https://lightning.ai/docs/litserve/features/streaming)
244
+ ✅ [Worker autoscaling](https://lightning.ai/docs/litserve/features/autoscaling)
245
+ ✅ [Self-host on your machines](https://lightning.ai/docs/litserve/features/hosting-methods#host-on-your-own)
246
+ ✅ [Host fully managed on Lightning AI](https://lightning.ai/docs/litserve/features/hosting-methods#host-on-lightning-studios)
247
+ ✅ [Serve all models: (LLMs, vision, etc.)](https://lightning.ai/docs/litserve/examples)
248
+ ✅ [Scale to zero (serverless)](https://lightning.ai/docs/litserve/features/streaming)
249
+ ✅ [Supports PyTorch, JAX, TF, etc...](https://lightning.ai/docs/litserve/features/full-control)
250
+ ✅ [OpenAPI compliant](https://www.openapis.org/)
251
+ ✅ [Open AI compatibility](https://lightning.ai/docs/litserve/features/open-ai-spec)
252
+ ✅ [Authentication](https://lightning.ai/docs/litserve/features/authentication)
253
+ ✅ [Dockerization](https://lightning.ai/docs/litserve/features/dockerization-deployment)
254
+
255
+
256
+
257
+ [10+ features...](https://lightning.ai/docs/litserve/features)
258
+
259
+ **Note:** We prioritize scalable, enterprise-level features over hype.
260
+
261
+ &nbsp;
262
+
263
+ # Performance
264
+ LitServe is designed for AI workloads. Specialized multi-worker handling delivers a minimum **2x speedup over FastAPI**.
265
+
266
+ Additional features like batching and GPU autoscaling can drive performance well beyond 2x, scaling efficiently to handle more simultaneous requests than FastAPI and TorchServe.
267
+
268
+ Reproduce the full benchmarks [here](https://lightning.ai/docs/litserve/home/benchmarks) (higher is better).
269
+
270
+ <div align="center">
271
+ <img alt="LitServe" src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/ls_charts_v6.png" width="1000px" style="max-width: 100%;">
272
+ </div>
273
+
274
+ These results are for image and text classification ML tasks. The performance relationships hold for other ML tasks (embedding, LLM serving, audio, segmentation, object detection, summarization etc...).
275
+
276
+ ***💡 Note on LLM serving:*** For high-performance LLM serving (like Ollama/vLLM), integrate [vLLM with LitServe](https://lightning.ai/lightning-ai/studios/deploy-a-private-llama-3-2-rag-api), use [LitGPT](https://github.com/Lightning-AI/litgpt?tab=readme-ov-file#deploy-an-llm), or build your custom vLLM-like server with LitServe. Optimizations like kv-caching, which can be done with LitServe, are needed to maximize LLM performance.
277
+
278
+ &nbsp;
279
+
280
+ # Hosting options
281
+ LitServe can be hosted independently on your own machines or fully managed via Lightning Studios.
282
+
283
+ Self-hosting is ideal for hackers, students, and DIY developers, while fully managed hosting is ideal for enterprise developers needing easy autoscaling, security, release management, and 99.995% uptime and observability.
284
+
285
+ &nbsp;
286
+
287
+ <div align="center">
288
+ <a target="_blank" href="https://lightning.ai/lightning-ai/studios/litserve-hello-world">
289
+ <img src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/host-on-lightning.svg" alt="Host on Lightning"/>
290
+ </a>
291
+ </div>
292
+
293
+ &nbsp;
294
+
295
+ <div align='center'>
296
+
297
+ | Feature | Self Managed | Fully Managed on Studios |
298
+ |----------------------------------|-----------------------------------|-------------------------------------|
299
+ | Deployment | ✅ Do it yourself deployment | ✅ One-button cloud deploy |
300
+ | Load balancing | ❌ | ✅ |
301
+ | Autoscaling | ❌ | ✅ |
302
+ | Scale to zero | ❌ | ✅ |
303
+ | Multi-machine inference | ❌ | ✅ |
304
+ | Authentication | ❌ | ✅ |
305
+ | Own VPC | ❌ | ✅ |
306
+ | AWS, GCP | ❌ | ✅ |
307
+ | Use your own cloud commits | ❌ | ✅ |
308
+
309
+ </div>
310
+
311
+ &nbsp;
312
+
313
+ # Community
314
+ LitServe is a [community project accepting contributions](https://lightning.ai/docs/litserve/community) - Let's make the world's most advanced AI inference engine.
315
+
316
+ 💬 [Get help on Discord](https://discord.com/invite/XncpTy7DSt)
317
+ 📋 [License: Apache 2.0](https://github.com/Lightning-AI/litserve/blob/main/LICENSE)
litserve-0.2.6/README.md ADDED
@@ -0,0 +1,248 @@
1
+ <div align='center'>
2
+
3
+ # Easily serve AI models Lightning fast ⚡
4
+
5
+ <img alt="Lightning" src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/ls_banner2.png" width="800px" style="max-width: 100%;">
6
+
7
+ &nbsp;
8
+
9
+ <strong>Lightning-fast serving engine for AI models.</strong>
10
+ Easy. Flexible. Enterprise-scale.
11
+ </div>
12
+
13
+ ----
14
+
15
+ **LitServe** is an easy-to-use, flexible serving engine for AI models built on FastAPI. It augments FastAPI with features like batching, streaming, and GPU autoscaling eliminate the need to rebuild a FastAPI server per model.
16
+
17
+ LitServe is at least [2x faster](#performance) than plain FastAPI due to AI-specific multi-worker handling.
18
+
19
+ <div align='center'>
20
+
21
+ <pre>
22
+ ✅ (2x)+ faster serving ✅ Easy to use ✅ LLMs, non LLMs and more
23
+ ✅ Bring your own model ✅ PyTorch/JAX/TF/... ✅ Built on FastAPI
24
+ ✅ GPU autoscaling ✅ Batching, Streaming ✅ Self-host or ⚡️ managed
25
+ ✅ Compound AI ✅ Integrate with vLLM and more
26
+ </pre>
27
+
28
+ <div align='center'>
29
+
30
+ [![Discord](https://img.shields.io/discord/1077906959069626439?label=Get%20help%20on%20Discord)](https://discord.gg/WajDThKAur)
31
+ ![cpu-tests](https://github.com/Lightning-AI/litserve/actions/workflows/ci-testing.yml/badge.svg)
32
+ [![codecov](https://codecov.io/gh/Lightning-AI/litserve/graph/badge.svg?token=SmzX8mnKlA)](https://codecov.io/gh/Lightning-AI/litserve)
33
+ [![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/Lightning-AI/litserve/blob/main/LICENSE)
34
+
35
+ </div>
36
+ </div>
37
+ <div align="center">
38
+ <div style="text-align: center;">
39
+ <a target="_blank" href="#quick-start" style="margin: 0 10px;">Quick start</a> •
40
+ <a target="_blank" href="#featured-examples" style="margin: 0 10px;">Examples</a> •
41
+ <a target="_blank" href="#features" style="margin: 0 10px;">Features</a> •
42
+ <a target="_blank" href="#performance" style="margin: 0 10px;">Performance</a> •
43
+ <a target="_blank" href="#hosting-options" style="margin: 0 10px;">Hosting</a> •
44
+ <a target="_blank" href="https://lightning.ai/docs/litserve" style="margin: 0 10px;">Docs</a>
45
+ </div>
46
+ </div>
47
+
48
+ &nbsp;
49
+
50
+ <div align="center">
51
+ <a target="_blank" href="https://lightning.ai/docs/litserve/home/get-started">
52
+ <img src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/get-started-badge.svg" height="36px" alt="Get started"/>
53
+ </a>
54
+ </div>
55
+
56
+ &nbsp;
57
+
58
+ # Quick start
59
+
60
+ Install LitServe via pip ([more options](https://lightning.ai/docs/litserve/home/install)):
61
+
62
+ ```bash
63
+ pip install litserve
64
+ ```
65
+
66
+ ### Define a server
67
+ This toy example with 2 models (AI compound system) shows LitServe's flexibility ([see real examples](#examples)):
68
+
69
+ ```python
70
+ # server.py
71
+ import litserve as ls
72
+
73
+ # (STEP 1) - DEFINE THE API (compound AI system)
74
+ class SimpleLitAPI(ls.LitAPI):
75
+ def setup(self, device):
76
+ # setup is called once at startup. Build a compound AI system (1+ models), connect DBs, load data, etc...
77
+ self.model1 = lambda x: x**2
78
+ self.model2 = lambda x: x**3
79
+
80
+ def decode_request(self, request):
81
+ # Convert the request payload to model input.
82
+ return request["input"]
83
+
84
+ def predict(self, x):
85
+ # Easily build compound systems. Run inference and return the output.
86
+ squared = self.model1(x)
87
+ cubed = self.model2(x)
88
+ output = squared + cubed
89
+ return {"output": output}
90
+
91
+ def encode_response(self, output):
92
+ # Convert the model output to a response payload.
93
+ return {"output": output}
94
+
95
+ # (STEP 2) - START THE SERVER
96
+ if __name__ == "__main__":
97
+ # scale with advanced features (batching, GPUs, etc...)
98
+ server = ls.LitServer(SimpleLitAPI(), accelerator="auto", max_batch_size=1)
99
+ server.run(port=8000)
100
+ ```
101
+
102
+ Now run the server via the command-line
103
+
104
+ ```bash
105
+ python server.py
106
+ ```
107
+
108
+ ### Test the server
109
+ Run the auto-generated test client:
110
+ ```bash
111
+ python client.py
112
+ ```
113
+
114
+ Or use this terminal command:
115
+ ```bash
116
+ curl -X POST http://127.0.0.1:8000/predict -H "Content-Type: application/json" -d '{"input": 4.0}'
117
+ ```
118
+
119
+ ### LLM serving
120
+ LitServe isn’t *just* for LLMs like vLLM or Ollama; it serves any AI model with full control over internals ([learn more](https://lightning.ai/docs/litserve/features/serve-llms)).
121
+ For easy LLM serving, integrate [vLLM with LitServe](https://lightning.ai/lightning-ai/studios/deploy-a-private-llama-3-2-rag-api), or use [LitGPT](https://github.com/Lightning-AI/litgpt?tab=readme-ov-file#deploy-an-llm) (built on LitServe).
122
+
123
+ ```
124
+ litgpt serve microsoft/phi-2
125
+ ```
126
+
127
+ ### Summary
128
+ - LitAPI lets you easily build complex AI systems with one or more models ([docs](https://lightning.ai/docs/litserve/api-reference/litapi)).
129
+ - Use the setup method for one-time tasks like connecting models, DBs, and loading data ([docs](https://lightning.ai/docs/litserve/api-reference/litapi#setup)).
130
+ - LitServer handles optimizations like batching, GPU autoscaling, streaming, etc... ([docs](https://lightning.ai/docs/litserve/api-reference/litserver)).
131
+ - Self host on your own machines or use Lightning Studios for a fully managed deployment ([learn more](#hosting-options)).
132
+
133
+ [Learn how to make this server 200x faster](https://lightning.ai/docs/litserve/home/speed-up-serving-by-200x).
134
+
135
+ &nbsp;
136
+
137
+ # Featured examples
138
+ Use LitServe to deploy any model or AI service: (Compound AI, Gen AI, classic ML, embeddings, LLMs, vision, audio, etc...)
139
+
140
+ <div align='center'>
141
+ <div width='200px'>
142
+ <video src="https://github.com/user-attachments/assets/5e73549a-bc0f-47a9-9d9c-5b54389be5de" width='200px' controls></video>
143
+ </div>
144
+ </div>
145
+
146
+ ## Examples
147
+ <pre>
148
+ <strong>Toy model:</strong> <a target="_blank" href="#define-a-server">Hello world</a>
149
+ <strong>LLMs:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-llama-3-2-vision-with-litserve">Llama 3.2</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/openai-fault-tolerant-proxy-server">LLM Proxy server</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-ai-agent-with-tool-use">Agent with tool use</a>
150
+ <strong>RAG:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-private-llama-3-2-rag-api">vLLM RAG (Llama 3.2)</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-private-llama-3-1-rag-api">RAG API (LlamaIndex)</a>
151
+ <strong>NLP:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-any-hugging-face-model-instantly">Hugging face</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-hugging-face-bert-model">BERT</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-text-embedding-api-with-litserve">Text embedding API</a>
152
+ <strong>Multimodal:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-open-ai-clip-with-litserve">OpenAI Clip</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-multi-modal-llm-with-minicpm">MiniCPM</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-phi3-5-vision-api-with-litserve">Phi-3.5 Vision Instruct</a>, <a target="_blank" href="https://lightning.ai/bhimrajyadav/studios/deploy-and-chat-with-qwen2-vl-using-litserve">Qwen2-VL</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-multi-modal-llm-with-pixtral">Pixtral</a>
153
+ <strong>Audio:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-open-ai-s-whisper-model">Whisper</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-music-generation-api-with-meta-s-audio-craft">AudioCraft</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-audio-generation-api">StableAudio</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-noise-cancellation-api-with-deepfilternet">Noise cancellation (DeepFilterNet)</a>
154
+ <strong>Vision:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-private-api-for-stable-diffusion-2">Stable diffusion 2</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-image-generation-api-with-auraflow">AuraFlow</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-image-generation-api-with-flux">Flux</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-super-resolution-image-api-with-aura-sr">Image Super Resolution (Aura SR)</a>,
155
+ <a target="_blank" href="https://lightning.ai/bhimrajyadav/studios/deploy-background-removal-api-with-litserve">Background Removal</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-controlled-image-generation-api-controlnet">Control Stable Diffusion (ControlNet)</a>
156
+ <strong>Speech:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-a-voice-clone-api-coqui-xtts-v2-model">Text-speech (XTTS V2)</a>, <a target="_blank" href="https://lightning.ai/bhimrajyadav/studios/deploy-a-speech-generation-api-using-parler-tts-powered-by-litserve">Parler-TTS</a>
157
+ <strong>Classical ML:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-random-forest-with-litserve">Random forest</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-xgboost-with-litserve">XGBoost</a>
158
+ <strong>Miscellaneous:</strong> <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-an-media-conversion-api-with-ffmpeg">Media conversion API (ffmpeg)</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/deploy-both-pytorch-and-tensorflow-in-a-single-api">PyTorch + TensorFlow in one API</a>, <a target="_blank" href="https://lightning.ai/lightning-ai/studios/openai-fault-tolerant-proxy-server">LLM proxy server</a>
159
+ </pre>
160
+ </pre>
161
+
162
+ [Browse 100+ community-built templates](https://lightning.ai/studios?section=serving)
163
+
164
+ &nbsp;
165
+
166
+ # Features
167
+ State-of-the-art features:
168
+
169
+ ✅ [(2x)+ faster than plain FastAPI](#performance)
170
+ ✅ [Bring your own model](https://lightning.ai/docs/litserve/features/full-control)
171
+ ✅ [Build compound systems (1+ models)](https://lightning.ai/docs/litserve/home)
172
+ ✅ [GPU autoscaling](https://lightning.ai/docs/litserve/features/gpu-inference)
173
+ ✅ [Batching](https://lightning.ai/docs/litserve/features/batching)
174
+ ✅ [Streaming](https://lightning.ai/docs/litserve/features/streaming)
175
+ ✅ [Worker autoscaling](https://lightning.ai/docs/litserve/features/autoscaling)
176
+ ✅ [Self-host on your machines](https://lightning.ai/docs/litserve/features/hosting-methods#host-on-your-own)
177
+ ✅ [Host fully managed on Lightning AI](https://lightning.ai/docs/litserve/features/hosting-methods#host-on-lightning-studios)
178
+ ✅ [Serve all models: (LLMs, vision, etc.)](https://lightning.ai/docs/litserve/examples)
179
+ ✅ [Scale to zero (serverless)](https://lightning.ai/docs/litserve/features/streaming)
180
+ ✅ [Supports PyTorch, JAX, TF, etc...](https://lightning.ai/docs/litserve/features/full-control)
181
+ ✅ [OpenAPI compliant](https://www.openapis.org/)
182
+ ✅ [Open AI compatibility](https://lightning.ai/docs/litserve/features/open-ai-spec)
183
+ ✅ [Authentication](https://lightning.ai/docs/litserve/features/authentication)
184
+ ✅ [Dockerization](https://lightning.ai/docs/litserve/features/dockerization-deployment)
185
+
186
+
187
+
188
+ [10+ features...](https://lightning.ai/docs/litserve/features)
189
+
190
+ **Note:** We prioritize scalable, enterprise-level features over hype.
191
+
192
+ &nbsp;
193
+
194
+ # Performance
195
+ LitServe is designed for AI workloads. Specialized multi-worker handling delivers a minimum **2x speedup over FastAPI**.
196
+
197
+ Additional features like batching and GPU autoscaling can drive performance well beyond 2x, scaling efficiently to handle more simultaneous requests than FastAPI and TorchServe.
198
+
199
+ Reproduce the full benchmarks [here](https://lightning.ai/docs/litserve/home/benchmarks) (higher is better).
200
+
201
+ <div align="center">
202
+ <img alt="LitServe" src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/ls_charts_v6.png" width="1000px" style="max-width: 100%;">
203
+ </div>
204
+
205
+ These results are for image and text classification ML tasks. The performance relationships hold for other ML tasks (embedding, LLM serving, audio, segmentation, object detection, summarization etc...).
206
+
207
+ ***💡 Note on LLM serving:*** For high-performance LLM serving (like Ollama/vLLM), integrate [vLLM with LitServe](https://lightning.ai/lightning-ai/studios/deploy-a-private-llama-3-2-rag-api), use [LitGPT](https://github.com/Lightning-AI/litgpt?tab=readme-ov-file#deploy-an-llm), or build your custom vLLM-like server with LitServe. Optimizations like kv-caching, which can be done with LitServe, are needed to maximize LLM performance.
208
+
209
+ &nbsp;
210
+
211
+ # Hosting options
212
+ LitServe can be hosted independently on your own machines or fully managed via Lightning Studios.
213
+
214
+ Self-hosting is ideal for hackers, students, and DIY developers, while fully managed hosting is ideal for enterprise developers needing easy autoscaling, security, release management, and 99.995% uptime and observability.
215
+
216
+ &nbsp;
217
+
218
+ <div align="center">
219
+ <a target="_blank" href="https://lightning.ai/lightning-ai/studios/litserve-hello-world">
220
+ <img src="https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/host-on-lightning.svg" alt="Host on Lightning"/>
221
+ </a>
222
+ </div>
223
+
224
+ &nbsp;
225
+
226
+ <div align='center'>
227
+
228
+ | Feature | Self Managed | Fully Managed on Studios |
229
+ |----------------------------------|-----------------------------------|-------------------------------------|
230
+ | Deployment | ✅ Do it yourself deployment | ✅ One-button cloud deploy |
231
+ | Load balancing | ❌ | ✅ |
232
+ | Autoscaling | ❌ | ✅ |
233
+ | Scale to zero | ❌ | ✅ |
234
+ | Multi-machine inference | ❌ | ✅ |
235
+ | Authentication | ❌ | ✅ |
236
+ | Own VPC | ❌ | ✅ |
237
+ | AWS, GCP | ❌ | ✅ |
238
+ | Use your own cloud commits | ❌ | ✅ |
239
+
240
+ </div>
241
+
242
+ &nbsp;
243
+
244
+ # Community
245
+ LitServe is a [community project accepting contributions](https://lightning.ai/docs/litserve/community) - Let's make the world's most advanced AI inference engine.
246
+
247
+ 💬 [Get help on Discord](https://discord.com/invite/XncpTy7DSt)
248
+ 📋 [License: Apache 2.0](https://github.com/Lightning-AI/litserve/blob/main/LICENSE)
litserve-0.2.6/requirements.txt ADDED
@@ -0,0 +1,3 @@
1
+ fastapi >=0.100
2
+ uvicorn[standard] >=0.29.0
3
+ pyzmq >=22.0.0
{litserve-0.0.0.dev2 → litserve-0.2.6}/setup.py RENAMED
@@ -1,4 +1,17 @@
1
1
  #!/usr/bin/env python
2
+ # Copyright The Lightning AI team.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
2
15
  import glob
3
16
  import os
4
17
  from importlib.util import module_from_spec, spec_from_file_location
@@ -97,4 +110,9 @@ setup(
97
110
  "Programming Language :: Python :: 3.10",
98
111
  "Programming Language :: Python :: 3.11",
99
112
  ],
113
+ entry_points={
114
+ "console_scripts": [
115
+ "litserve=litserve.__main__:main",
116
+ ],
117
+ },
100
118
  )
litserve-0.2.6/src/litserve/__about__.py ADDED
@@ -0,0 +1,30 @@
1
+ # Copyright The Lightning AI team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ __version__ = "0.2.6"
15
+ __author__ = "Lightning-AI et al."
16
+ __author_email__ = "community@lightning.ai"
17
+ __license__ = "Apache-2.0"
18
+ __copyright__ = f"Copyright (c) 2024, {__author__}."
19
+ __homepage__ = "https://github.com/Lightning-AI/litserve"
20
+ __docs__ = "Lightweight AI server."
21
+
22
+ __all__ = [
23
+ "__author__",
24
+ "__author_email__",
25
+ "__copyright__",
26
+ "__docs__",
27
+ "__homepage__",
28
+ "__license__",
29
+ "__version__",
30
+ ]