litserve 0.2.4__tar.gz → 0.2.5.dev0__tar.gz

{litserve-0.2.4/src/litserve.egg-info → litserve-0.2.5.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: litserve
-Version: 0.2.4
+Version: 0.2.5.dev0
 Summary: Lightweight AI server.
 Home-page: https://github.com/Lightning-AI/litserve
 Download-URL: https://github.com/Lightning-AI/litserve

{litserve-0.2.4 → litserve-0.2.5.dev0}/src/litserve/__about__.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-__version__ = "0.2.4"
+__version__ = "0.2.5.dev0"
 __author__ = "Lightning-AI et al."
 __author_email__ = "community@lightning.ai"
 __license__ = "Apache-2.0"

{litserve-0.2.4 → litserve-0.2.5.dev0}/src/litserve/loops.py

@@ -26,7 +26,7 @@ from starlette.formparsers import MultiPartParser
 from litserve import LitAPI
 from litserve.callbacks import CallbackRunner, EventTypes
 from litserve.specs.base import LitSpec
-from litserve.utils import LitAPIStatus, PickleableHTTPException
+from litserve.utils import LitAPIStatus, PickleableHTTPException, WorkerSetupStatus
 
 mp.allow_connection_pickling()
 
@@ -399,18 +399,23 @@ def inference_worker(
     max_batch_size: int,
     batch_timeout: float,
     stream: bool,
-    workers_setup_status: Dict[str, bool],
+    workers_setup_status: Dict[int, str],
     callback_runner: CallbackRunner,
 ):
     callback_runner.trigger_event(EventTypes.BEFORE_SETUP, lit_api=lit_api)
-    lit_api.setup(device)
+    try:
+        lit_api.setup(device)
+    except Exception:
+        logger.exception(f"Error setting up worker {worker_id}.")
+        workers_setup_status[worker_id] = WorkerSetupStatus.ERROR
+        return
     lit_api.device = device
     callback_runner.trigger_event(EventTypes.AFTER_SETUP, lit_api=lit_api)
 
-    print(f"Setup complete for worker {worker_id}.")
+    logger.info(f"Setup complete for worker {worker_id}.")
 
     if workers_setup_status:
-        workers_setup_status[worker_id] = True
+        workers_setup_status[worker_id] = WorkerSetupStatus.READY
 
     if lit_spec:
         logging.info(f"LitServe will use {lit_spec.__class__.__name__} spec")

{litserve-0.2.4 → litserve-0.2.5.dev0}/src/litserve/server.py

@@ -44,18 +44,10 @@ from litserve.middlewares import MaxSizeMiddleware, RequestCountMiddleware
 from litserve.python_client import client_template
 from litserve.specs import OpenAISpec
 from litserve.specs.base import LitSpec
-from litserve.utils import LitAPIStatus, call_after_stream
+from litserve.utils import LitAPIStatus, WorkerSetupStatus, call_after_stream
 
 mp.allow_connection_pickling()
 
-try:
-    import uvloop
-
-    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
-
-except ImportError:
-    print("uvloop is not installed. Falling back to the default asyncio event loop.")
-
 logger = logging.getLogger(__name__)
 
 # if defined, it will require clients to auth with X-API-Key in the header
@@ -233,7 +225,7 @@ class LitServer:
             if len(device) == 1:
                 device = device[0]
 
-            self.workers_setup_status[worker_id] = False
+            self.workers_setup_status[worker_id] = WorkerSetupStatus.STARTING
 
             ctx = mp.get_context("spawn")
             process = ctx.Process(
@@ -328,7 +320,7 @@ class LitServer:
         async def health(request: Request) -> Response:
             nonlocal workers_ready
             if not workers_ready:
-                workers_ready = all(self.workers_setup_status.values())
+                workers_ready = all(v == WorkerSetupStatus.READY for v in self.workers_setup_status.values())
 
             if workers_ready:
                 return Response(content="ok", status_code=200)
@@ -436,6 +428,13 @@ class LitServer:
         except Exception as e:
             logger.exception(f"Error copying file: {e}")
 
+    def verify_worker_status(self):
+        while not any(v == WorkerSetupStatus.READY for v in self.workers_setup_status.values()):
+            if any(v == WorkerSetupStatus.ERROR for v in self.workers_setup_status.values()):
+                raise RuntimeError("One or more workers failed to start. Shutting down LitServe")
+            time.sleep(0.05)
+        logger.debug("One or more workers are ready to serve requests")
+
     def run(
         self,
         host: str = "0.0.0.0",
@@ -473,7 +472,7 @@ class LitServer:
 
         if sys.platform == "win32":
             warnings.warn(
-                "Windows does not support forking. Using threads" " api_server_worker_type will be set to 'thread'"
+                "Windows does not support forking. Using threads api_server_worker_type will be set to 'thread'"
             )
             api_server_worker_type = "thread"
         elif api_server_worker_type is None:
@@ -481,6 +480,7 @@ class LitServer:
 
         manager, litserve_workers = self.launch_inference_worker(num_api_servers)
 
+        self.verify_worker_status()
         try:
             servers = self._start_server(port, num_api_servers, log_level, sockets, api_server_worker_type, **kwargs)
             print(f"Swagger UI is available at http://0.0.0.0:{port}/docs")

{litserve-0.2.4 → litserve-0.2.5.dev0}/src/litserve/test_examples/openai_spec_example.py

@@ -66,7 +66,7 @@ class OpenAIBatchContext(LitAPI):
         for ctx in context:
             ctx["temperature"] = 1.0
         output = (
-            "Hi! It's nice to meet you. Is there something I can help you with " "or would you like to chat?"
+            "Hi! It's nice to meet you. Is there something I can help you with or would you like to chat?"
         ).split()
         for out in output:
             time.sleep(0.01)  # fake delay

{litserve-0.2.4 → litserve-0.2.5.dev0}/src/litserve/utils.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import asyncio
+import dataclasses
 import logging
 import pickle
 from contextlib import contextmanager
@@ -78,3 +79,11 @@ async def call_after_stream(streamer: AsyncIterator, callback, *args, **kwargs):
         logger.exception(f"Error in streamer: {e}")
     finally:
         callback(*args, **kwargs)
+
+
+@dataclasses.dataclass
+class WorkerSetupStatus:
+    STARTING: str = "starting"
+    READY: str = "ready"
+    ERROR: str = "error"
+    FINISHED: str = "finished"

{litserve-0.2.4 → litserve-0.2.5.dev0/src/litserve.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: litserve
-Version: 0.2.4
+Version: 0.2.5.dev0
 Summary: Lightweight AI server.
 Home-page: https://github.com/Lightning-AI/litserve
 Download-URL: https://github.com/Lightning-AI/litserve