truss 0.10.0rc1__py3-none-any.whl → 0.60.0__py3-none-any.whl

truss/tests/test_data/happy.ipynb

@@ -0,0 +1,56 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "7a4aad7e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import random\n",
+    "import tempfile\n",
+    "\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from sklearn.linear_model import LogisticRegression\n",
+    "\n",
+    "import truss\n",
+    "\n",
+    "# 10 rows of 10 features\n",
+    "features = random.choices(range(10), k=10)\n",
+    "values = np.random.randint(0, 100, size=(10, 10))\n",
+    "\n",
+    "df = pd.DataFrame(values, columns=features)\n",
+    "\n",
+    "lr = LogisticRegression(random_state=0, max_iter=1000)\n",
+    "lr.fit(df.values, df.iloc[:, 0])\n",
+    "\n",
+    "with tempfile.TemporaryDirectory() as tmp:\n",
+    "    tr = truss.create(lr, target_directory=tmp)\n",
+    "\n",
+    "    assert truss.load(tmp).spec.yaml_string == tr.spec.yaml_string"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

truss/tests/test_data/model_load_failure_test/config.yaml

@@ -0,0 +1,2 @@
+model_name: Test Load Failure
+python_version: py39

truss/{test_data → tests/test_data}/patch_ping_test_server/app.py

@@ -22,9 +22,7 @@ def _inc_fn_call_stat():
 @app.route("/hash_is_current", methods=["POST"])
 def hash_is_current():
     _inc_fn_call_stat()
-    return {
-        "is_current": True,
-    }
+    return {"is_current": True}
 
 
 @app.route("/hash_is_current_but_only_every_third_call_succeeds", methods=["POST"])
@@ -33,18 +31,14 @@ def hash_is_current_but_only_every_third_call_succeeds():
     global _stats
     fn_name = inspect.stack()[0][3]
     if _stats[f"{fn_name}_called_count"] % 3 == 0:
-        return {
-            "is_current": True,
-        }
+        return {"is_current": True}
     return "simulated failure", 503
 
 
 @app.route("/accepted", methods=["POST"])
 def accepted():
     _inc_fn_call_stat()
-    return {
-        "accepted": True,
-    }
+    return {"accepted": True}
 
 
 @app.route("/health")

truss/{test_data → tests/test_data}/server.Dockerfile

@@ -1,27 +1,18 @@
 ARG PYVERSION=py39
-FROM baseten/truss-server-base:3.9-v0.4.3 as truss_server
+FROM baseten/truss-server-base:3.9-v0.4.3 AS truss_server
 
-ENV PYTHON_EXECUTABLE /usr/local/bin/python3
-ENV JSON_LOG True
+ENV PYTHON_EXECUTABLE="/usr/local/bin/python3"
 
 RUN grep -w 'ID=debian\|ID_LIKE=debian' /etc/os-release || { echo "ERROR: Supplied base image is not a debian image"; exit 1; }
-RUN $PYTHON_EXECUTABLE -c "import sys; sys.exit(0) if sys.version_info.major == 3 and sys.version_info.minor >=8 and sys.version_info.minor <=11 else sys.exit(1)" \
-    || { echo "ERROR: Supplied base image does not have 3.8 <= python <= 3.11"; exit 1; }
+RUN $PYTHON_EXECUTABLE -c "import sys; sys.exit(0) if sys.version_info.major == 3 and sys.version_info.minor >=8 and sys.version_info.minor <=12 else sys.exit(1)" \
+    || { echo "ERROR: Supplied base image does not have 3.8 <= python <= 3.12"; exit 1; }
 
 RUN pip install --upgrade pip --no-cache-dir \
     && rm -rf /root/.cache/pip
 
-
-# Always install the truss package
-COPY ./truss/ /lib/truss_pkg/truss
-COPY ./pyproject.toml /lib/truss_pkg/
-COPY ./README.md /lib/truss_pkg/
-RUN pip install /lib/truss_pkg --no-cache-dir && rm -rf /root/.cache/pip
-
-
 # If user base image is supplied in config, apply build commands from truss base image
-ENV PYTHONUNBUFFERED True
-ENV DEBIAN_FRONTEND=noninteractive
+ENV PYTHONUNBUFFERED="True"
+ENV DEBIAN_FRONTEND="noninteractive"
 
 RUN apt update && \
     apt install -y bash \
@@ -34,16 +25,24 @@ RUN apt update && \
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/*
 
-ENV APP_HOME /app
+COPY ./base_server_requirements.txt base_server_requirements.txt
+RUN pip install -r base_server_requirements.txt --no-cache-dir && rm -rf /root/.cache/pip
+
+COPY ./requirements.txt requirements.txt
+RUN cat requirements.txt
+RUN pip install -r requirements.txt --no-cache-dir && rm -rf /root/.cache/pip
+
+ENV APP_HOME="/app"
 WORKDIR $APP_HOME
 
 # Copy data before code for better caching
 COPY ./data /app/data
-COPY ./model /app/model
+COPY ./server /app
 COPY ./config.yaml /app/config.yaml
+COPY ./model /app/model
 
-COPY ./packages /app/packages
+COPY ./packages /packages
 
-ENV INFERENCE_SERVER_PORT 8080
-ENV SERVER_START_CMD="/usr/local/bin/python3 -m truss.server.inference_server"
-ENTRYPOINT ["/usr/local/bin/python3", "-m", "truss.server.inference_server"]
+ENV INFERENCE_SERVER_PORT="8080"
+ENV SERVER_START_CMD="/usr/local/bin/python3 /app/main.py"
+ENTRYPOINT ["/usr/local/bin/python3", "/app/main.py"]

truss/{test_data → tests/test_data}/server_conformance_test_truss/model/model.py

@@ -10,11 +10,9 @@ class Model:
         self._model = None
 
     def load(self):
-        # Load model here and assign to self._model.
-        print("Taking 20 seconds to load")
+        print("Starting loading over 20 seconds.")
         time.sleep(20)
 
     def predict(self, model_input: Any) -> Dict[str, List]:
-        # Invoke model on model_input and calculate predictions here.
         print("Taking 20 seconds to predict")
         time.sleep(20)

truss/tests/test_data/test_basic_truss/config.yaml

@@ -0,0 +1,16 @@
+environment_variables:
+  OTEL_TRACING_NDJSON_FILE: "/tmp/otel_traces.ndjson"
+external_package_dirs: []
+model_metadata: {}
+model_name: basic truss
+python_version: py39
+requirements: []
+resources:
+  accelerator: null
+  cpu: '1'
+  memory: 2Gi
+  use_gpu: false
+secrets:
+  foo: "foo"
+  bar: "bar"
+system_packages: []

truss/tests/test_data/test_build_commands/config.yaml

@@ -0,0 +1,13 @@
+build_commands:
+  - mkdir example_dir
+  - cd example_dir && touch testing.py
+model_metadata: {}
+model_name: null
+model_type: custom
+python_version: py39
+requirements: []
+resources:
+  accelerator: null
+  cpu: 500m
+  memory: 512Mi
+  use_gpu: false

truss/{test_data/test_streaming_async_generator_truss → tests/test_data/test_build_commands}/model/model.py

@@ -12,7 +12,6 @@ class Model:
         # Load model here and assign to self._model.
         pass
 
-    async def predict(self, model_input: Any) -> Dict[str, List]:
+    def predict(self, model_input: Any) -> Dict[str, List]:
         # Invoke model on model_input and calculate predictions here.
-        for i in range(5):
-            yield str(i)
+        return {"predictions": [1, 2]}

truss/tests/test_data/test_build_commands_failure/config.yaml

@@ -0,0 +1,14 @@
+build_commands:
+  - mkdir example_dir
+  - cd example_dir && touch testing.py
+  - haha lol
+model_metadata: {}
+model_name: null
+model_type: custom
+python_version: py39
+requirements: []
+resources:
+  accelerator: null
+  cpu: 500m
+  memory: 512Mi
+  use_gpu: false

truss/tests/test_data/test_build_commands_failure/model/model.py

@@ -0,0 +1,17 @@
+from typing import Any, Dict, List
+
+
+class Model:
+    def __init__(self, **kwargs) -> None:
+        self._data_dir = kwargs["data_dir"]
+        self._config = kwargs["config"]
+        self._secrets = kwargs["secrets"]
+        self._model = None
+
+    def load(self):
+        # Load model here and assign to self._model.
+        pass
+
+    def predict(self, model_input: Any) -> Dict[str, List]:
+        # Invoke model on model_input and calculate predictions here.
+        return {"predictions": [1, 2]}

truss/tests/test_data/test_concurrency_truss/config.yaml

@@ -0,0 +1,4 @@
+runtime:
+  predict_concurrency: 2
+environment_variables:
+  OTEL_TRACING_NDJSON_FILE: "/tmp/otel_traces.ndjson"

truss/tests/test_data/test_custom_server_truss/config.yaml

@@ -0,0 +1,20 @@
+base_image:
+  image: baseten/fastapi-test:0.1.2
+docker_server:
+  start_command: python main.py
+  predict_endpoint: /predict
+  server_port: 8000
+  readiness_endpoint: /
+  liveness_endpoint: /
+resources:
+  accelerator: null
+  cpu: '1'
+  memory: 2Gi
+  use_gpu: false
+model_name: Test Docker Server Truss
+secrets:
+  hf_access_token: null
+environment_variables:
+  HF_TOKEN: 123456
+runtime:
+  predict_concurrency: 1

truss/tests/test_data/test_custom_server_truss/test_docker_image/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.11-slim
+
+# Update package lists and install curl
+RUN apt-get update && apt-get install -y curl
+
+# Install pip dependencies
+RUN pip install fastapi[standard]
+
+# Copy the FastAPI application code
+COPY app.py /home/app.py
+COPY main.py /home/main.py
+
+# Set the working directory
+WORKDIR /home
+
+# Command to run FastAPI directly
+ENTRYPOINT ["python", "main.py"]

truss/tests/test_data/test_custom_server_truss/test_docker_image/README.md

@@ -0,0 +1,10 @@
+We built this minimal fastapi docker image to be used in integration test `test_custom_server_truss.py::test_custom_server_truss`
+
+Steps to update testing docker image
+
+1. run `docker login`
+2. cd into this directory
+3. update version number in VERSION file
+(before running the next step, make sure you meet with the [prerequisites](https://docs.docker.com/build/building/multi-platform/#prerequisites) here)
+4. run `sh build_upload_new_image.sh`
+5. update image tag to latest version in config.yaml

truss/tests/test_data/test_custom_server_truss/test_docker_image/VERSION

@@ -0,0 +1 @@
+0.1.1

truss/tests/test_data/test_custom_server_truss/test_docker_image/app.py

@@ -0,0 +1,19 @@
+import os
+
+from fastapi import FastAPI
+
+app = FastAPI()
+
+
+@app.get("/health")
+async def health():
+    return {"message": "OK"}
+
+
+@app.post("/predict")
+async def root():
+    return {
+        "message": "Hello World",
+        "is_env_var_passed": os.environ.get("HF_TOKEN") is not None,
+        "is_secret_mounted": os.path.exists("/secrets/hf_access_token"),
+    }

truss/tests/test_data/test_custom_server_truss/test_docker_image/build_upload_new_image.sh

@@ -0,0 +1,6 @@
+# Read the version from the VERSION file
+VERSION=$(cat VERSION)
+
+docker buildx build --platform linux/amd64,linux/arm64 -f ./Dockerfile -t fastapi-test:$VERSION . || { echo "Failed to build the Docker image"; exit 1; }
+docker tag fastapi-test:$VERSION baseten/fastapi-test:$VERSION || { echo "Failed to tag the Docker image."; exit 1; }
+docker push baseten/fastapi-test:$VERSION || { echo "Failed to push the Docker image."; exit 1; }

truss/{test_data/test_basic_truss → tests/test_data/test_openai}/config.yaml

@@ -1,9 +1,8 @@
 environment_variables: {}
 external_package_dirs: []
 model_metadata: {}
-model_name: basic truss
+model_name: Test OpenAI Compatibility
 python_version: py39
-requirements: []
 resources:
   accelerator: null
   cpu: '1'

truss/tests/test_data/test_openai/model/model.py

@@ -0,0 +1,15 @@
+from typing import Dict
+
+
+class Model:
+    def __init__(self, **kwargs):
+        pass
+
+    def chat_completions(self, input: Dict) -> str:
+        return "chat_completions"
+
+    def completions(self, input: Dict) -> str:
+        return "completions"
+
+    def predict(self, input: Dict) -> str:
+        return "predict"

truss/tests/test_data/test_pyantic_v1/model/model.py

@@ -0,0 +1,28 @@
+"""
+The `Model` class is an interface between the ML model that you're packaging and the model
+server that you're running it on.
+
+The main methods to implement here are:
+* `load`: runs exactly once when the model server is spun up or patched and loads the
+   model onto the model server. Include any logic for initializing your model, such
+   as downloading model weights and loading the model into memory.
+* `predict`: runs every time the model server is called. Include any logic for model
+  inference and return the model output.
+
+See https://truss.baseten.co/quickstart for more.
+"""
+
+import pydantic
+
+
+class DummyData(pydantic.BaseModel):
+    foo: str
+    bar: int
+
+
+class Model:
+    def __init__(self, **kwargs):
+        self._model = DummyData(foo="bla", bar=123)
+
+    def predict(self, model_input):
+        return self._model.json(indent=4)

truss/tests/test_data/test_pyantic_v1/requirements.txt

@@ -0,0 +1 @@
+pydantic<2.0

truss/tests/test_data/test_pyantic_v2/config.yaml

@@ -0,0 +1,13 @@
+environment_variables: {}
+external_package_dirs: []
+model_metadata: {}
+model_name: Test Requirements File
+python_version: py39
+requirements_file: ./requirements.txt
+resources:
+  accelerator: null
+  cpu: '1'
+  memory: 2Gi
+  use_gpu: false
+secrets: {}
+system_packages: []

truss/tests/test_data/test_pyantic_v2/model/model.py

@@ -0,0 +1,30 @@
+"""
+The `Model` class is an interface between the ML model that you're packaging and the model
+server that you're running it on.
+
+The main methods to implement here are:
+* `load`: runs exactly once when the model server is spun up or patched and loads the
+   model onto the model server. Include any logic for initializing your model, such
+   as downloading model weights and loading the model into memory.
+* `predict`: runs every time the model server is called. Include any logic for model
+  inference and return the model output.
+
+See https://truss.baseten.co/quickstart for more.
+"""
+
+import pydantic
+
+
+class DummyData(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict()
+
+    foo: str
+    bar: int
+
+
+class Model:
+    def __init__(self, **kwargs):
+        self._model = DummyData(foo="bla", bar=123)
+
+    def predict(self, model_input):
+        return self._model.model_dump_json(indent=4)

truss/tests/test_data/test_pyantic_v2/requirements.txt

@@ -0,0 +1 @@
+pydantic>=2.0

truss/tests/test_data/test_requirements_file_truss/config.yaml

@@ -0,0 +1,13 @@
+environment_variables: {}
+external_package_dirs: []
+model_metadata: {}
+model_name: Test Requirements File
+python_version: py39
+requirements_file: ./requirements.txt
+resources:
+  accelerator: null
+  cpu: '1'
+  memory: 2Gi
+  use_gpu: false
+secrets: {}
+system_packages: []

truss/{test_data → tests/test_data}/test_requirements_file_truss/model/model.py

@@ -11,6 +11,7 @@ The main methods to implement here are:
 
 See https://truss.baseten.co/quickstart for more.
 """
+
 import torch
 
 

truss/tests/test_data/test_streaming_async_generator_truss/config.yaml

@@ -0,0 +1,4 @@
+model_name: Test Streaming Async Generator
+python_version: py39
+environment_variables:
+  OTEL_TRACING_NDJSON_FILE: "/tmp/otel_traces.ndjson"

truss/tests/test_data/test_streaming_async_generator_truss/model/model.py

@@ -0,0 +1,7 @@
+from typing import Any, Dict, List
+
+
+class Model:
+    async def predict(self, model_input: Any) -> Dict[str, List]:
+        for i in range(5):
+            yield str(i)

truss/tests/test_data/test_streaming_truss/config.yaml

@@ -0,0 +1,4 @@
+model_name: Test Streaming
+python_version: py39
+environment_variables:
+  OTEL_TRACING_NDJSON_FILE: "/tmp/otel_traces.ndjson"

truss/{test_data → tests/test_data}/test_streaming_truss_with_error/model/model.py

@@ -1,23 +1,15 @@
 from typing import Any, Dict, List
 
+import helpers_1
 
-class Model:
-    def __init__(self, **kwargs) -> None:
-        self._data_dir = kwargs["data_dir"]
-        self._config = kwargs["config"]
-        self._secrets = kwargs["secrets"]
-        self._model = None
-
-    def load(self):
-        # Load model here and assign to self._model.
-        pass
 
+class Model:
     def predict(self, model_input: Any) -> Dict[str, List]:
         def inner():
             for i in range(5):
                 # Raise error partway through if throw_error is set
                 if i == 3 and model_input.get("throw_error"):
-                    raise Exception("error")
+                    helpers_1.foo(123)
                 yield str(i)
 
         return inner()

truss/tests/test_data/test_streaming_truss_with_error/packages/helpers_1.py

@@ -0,0 +1,5 @@
+import helpers_2
+
+
+def foo(x):
+    return helpers_2.bar(x)

truss/tests/test_data/test_streaming_truss_with_error/packages/helpers_2.py

@@ -0,0 +1,2 @@
+def bar(x):
+    raise Exception("Crashed in `bar`.")