litserve 0.2.6__tar.gz → 0.2.6.dev0__tar.gz

{litserve-0.2.6/src/litserve.egg-info → litserve-0.2.6.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.1
 Name: litserve
-Version: 0.2.6
+Version: 0.2.6.dev0
 Summary: Lightweight AI server.
 Home-page: https://github.com/Lightning-AI/litserve
 Download-URL: https://github.com/Lightning-AI/litserve
@@ -29,11 +29,6 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: fastapi>=0.100
 Requires-Dist: uvicorn[standard]>=0.29.0
-Requires-Dist: pyzmq>=22.0.0
-Provides-Extra: perf
-Requires-Dist: jsonargparse; extra == "perf"
-Requires-Dist: tenacity; extra == "perf"
-Requires-Dist: uvloop; extra == "perf"
 Provides-Extra: test
 Requires-Dist: asgi-lifespan; extra == "test"
 Requires-Dist: coverage[toml]>=7.5.3; extra == "test"
@@ -52,20 +47,10 @@ Requires-Dist: python-multipart; extra == "test"
 Requires-Dist: requests; extra == "test"
 Requires-Dist: torch>2.0.0; extra == "test"
 Requires-Dist: transformers; extra == "test"
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: download-url
-Dynamic: home-page
-Dynamic: keywords
-Dynamic: license
-Dynamic: project-url
-Dynamic: provides-extra
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary
+Provides-Extra: perf
+Requires-Dist: jsonargparse; extra == "perf"
+Requires-Dist: tenacity; extra == "perf"
+Requires-Dist: uvloop; extra == "perf"
 
 <div align='center'>
 

{litserve-0.2.6 → litserve-0.2.6.dev0}/requirements.txt

@@ -1,3 +1,2 @@
 fastapi >=0.100
 uvicorn[standard] >=0.29.0
-pyzmq >=22.0.0

{litserve-0.2.6 → litserve-0.2.6.dev0}/src/litserve/__about__.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-__version__ = "0.2.6"
+__version__ = "0.2.6.dev0"
 __author__ = "Lightning-AI et al."
 __author_email__ = "community@lightning.ai"
 __license__ = "Apache-2.0"

{litserve-0.2.6 → litserve-0.2.6.dev0}/src/litserve/__init__.py

@@ -18,9 +18,6 @@ from litserve.callbacks import Callback
 from litserve.loggers import Logger
 from litserve.server import LitServer, Request, Response
 from litserve.specs import OpenAIEmbeddingSpec, OpenAISpec
-from litserve.utils import configure_logging
-
-configure_logging()
 
 __all__ = [
     "LitAPI",

{litserve-0.2.6 → litserve-0.2.6.dev0}/src/litserve/api.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import inspect
 import json
 import warnings
 from abc import ABC, abstractmethod
@@ -56,9 +57,10 @@ class LitAPI(ABC):
 
         return inputs
 
+    @abstractmethod
     def predict(self, x, **kwargs):
         """Run the model on the input and return or yield the output."""
-        raise NotImplementedError("predict is not implemented")
+        pass
 
     def _unbatch_no_stream(self, output):
         if isinstance(output, str):
@@ -111,16 +113,75 @@ class LitAPI(ABC):
     def device(self, value):
         self._device = value
 
-    def pre_setup(self, max_batch_size: int, spec: Optional[LitSpec]):
-        self.max_batch_size = max_batch_size
+    def _sanitize(self, max_batch_size: int, spec: Optional[LitSpec]):
         if self.stream:
             self._default_unbatch = self._unbatch_stream
         else:
             self._default_unbatch = self._unbatch_no_stream
 
+        # we will sanitize regularly if no spec
+        # in case, we have spec then:
+        # case 1: spec implements a streaming API
+        # Case 2: spec implements a non-streaming API
         if spec:
+            # TODO: Implement sanitization
             self._spec = spec
-            spec.pre_setup(self)
+            return
+
+        original = self.unbatch.__code__ is LitAPI.unbatch.__code__
+        if (
+            self.stream
+            and max_batch_size > 1
+            and not all([
+                inspect.isgeneratorfunction(self.predict),
+                inspect.isgeneratorfunction(self.encode_response),
+                (original or inspect.isgeneratorfunction(self.unbatch)),
+            ])
+        ):
+            raise ValueError(
+                """When `stream=True` with max_batch_size > 1, `lit_api.predict`, `lit_api.encode_response` and
+                `lit_api.unbatch` must generate values using `yield`.
+
+             Example:
+
+                def predict(self, inputs):
+                    ...
+                    for i in range(max_token_length):
+                        yield prediction
+
+                def encode_response(self, outputs):
+                    for output in outputs:
+                        encoded_output = ...
+                        yield encoded_output
+
+                def unbatch(self, outputs):
+                    for output in outputs:
+                        unbatched_output = ...
+                        yield unbatched_output
+             """
+            )
+
+        if self.stream and not all([
+            inspect.isgeneratorfunction(self.predict),
+            inspect.isgeneratorfunction(self.encode_response),
+        ]):
+            raise ValueError(
+                """When `stream=True` both `lit_api.predict` and
+             `lit_api.encode_response` must generate values using `yield`.
+
+             Example:
+
+                def predict(self, inputs):
+                    ...
+                    for i in range(max_token_length):
+                        yield prediction
+
+                def encode_response(self, outputs):
+                    for output in outputs:
+                        encoded_output = ...
+                        yield encoded_output
+             """
+            )
 
     def set_logger_queue(self, queue: Queue):
         """Set the queue for logging events."""
@@ -137,9 +198,3 @@ class LitAPI(ABC):
             )
             return
         self._logger_queue.put((key, value))
-
-    def has_active_requests(self) -> bool:
-        raise NotImplementedError("has_active_requests is not implemented")
-
-    def has_capacity(self) -> bool:
-        raise NotImplementedError("has_capacity is not implemented")