inference-proxy 3.2.2__tar.gz → 3.3.0__tar.gz

{inference_proxy-3.2.2 → inference_proxy-3.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: inference-proxy
-Version: 3.2.2
+Version: 3.3.0
 Summary: Inference Proxy is an OpenAI-compatible http proxy server for inferencing various LLMs capable of working with Google, Anthropic, OpenAI APIs, local PyTorch inference, etc.
 License: MIT License
          
@@ -23,6 +23,7 @@ License: MIT License
          LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
          OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
          SOFTWARE.
+License-File: LICENSE
 Keywords: llm,large language models,ai,gpt,openai,proxy,http,proxy-server,llm gateway,openai,anthropic,google genai
 Author: Vitalii Stepanenko
 Author-email: mail@vitaliy.in
@@ -45,10 +46,10 @@ Provides-Extra: all
 Provides-Extra: anthropic
 Provides-Extra: google
 Provides-Extra: test
-Requires-Dist: ai-microcore (>=5.1.2,<7)
+Requires-Dist: ai-microcore (>=6.6.1,<7)
 Requires-Dist: anthropic (>=0.77,<1) ; extra == "all"
 Requires-Dist: anthropic (>=0.77,<1) ; extra == "anthropic"
-Requires-Dist: fastapi (>=0.121.3,<1)
+Requires-Dist: fastapi (>=0.136.3,<1)
 Requires-Dist: google-genai (>=1.62.0,<2) ; extra == "all"
 Requires-Dist: google-genai (>=1.62.0,<2) ; extra == "google"
 Requires-Dist: pydantic (>=2.12.5,<2.13.0)
@@ -71,7 +72,7 @@ Description-Content-Type: text/markdown
   <a href="https://pypi.org/project/lm-proxy/"><img src="https://img.shields.io/pypi/v/lm-proxy?color=blue" alt="PyPI"></a>
   <a href="https://github.com/Nayjest/lm-proxy/actions/workflows/tests.yml"><img src="https://github.com/Nayjest/lm-proxy/actions/workflows/tests.yml/badge.svg" alt="Tests"></a>
   <a href="https://github.com/Nayjest/lm-proxy/actions/workflows/code-style.yml"><img src="https://github.com/Nayjest/lm-proxy/actions/workflows/code-style.yml/badge.svg" alt="Code Style"></a>
-  <img src="https://raw.githubusercontent.com/Nayjest/lm-proxy/main/coverage.svg" alt="Code Coverage">
+  <img src="https://raw.githubusercontent.com/Nayjest/lm-proxy/coverage-badge/coverage.svg" alt="Code Coverage">
   <a href="https://www.bestpractices.dev/projects/11364"><img src="https://www.bestpractices.dev/projects/11364/badge"></a>
   <br>
   <a href="https://github.com/vshymanskyy/StandWithUkraine/blob/main/README.md"><img src="https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/refs/heads/main/badges/StandWithUkraine.svg" alt="Stand With Ukraine"></a>
@@ -91,6 +92,7 @@ It works as a drop-in replacement for OpenAI's API, allowing you to switch betwe
 - [Getting Started](#-getting-started)
   - [Installation](#installation)
   - [Quick Start](#quick-start)
+  - [Run with Docker](#run-with-docker)
 - [Configuration](#-configuration)
   - [Basic Structure](#basic-structure)
   - [Environment Variables](#environment-variables)
@@ -114,6 +116,8 @@ It works as a drop-in replacement for OpenAI's API, allowing you to switch betwe
 - [Guides & Reference](#-guides--reference)
 - [Known Limitations](#-known-limitations)
 - [Debugging](#-debugging)
+  - [Debugging Mode](#debugging-mode)
+  - [Printing LLM Requests and Responses](#printing-llm-requests-and-responses)
 - [Contributing](#-contributing)
 - [License](#-license)
 
@@ -210,6 +214,39 @@ completion = client.chat.completions.create(
 )
 ```
 
+### Run with Docker<a id="run-with-docker"></a>
+
+Official images are published to GitHub Container Registry on every release
+(`latest`, `3`, `3.2`, `3.2.2`, …; `edge` tracks the `main` branch).
+They include the Anthropic and Google connectors and YAML config support out of the box.
+
+Mount your `config.toml` and provide API keys via an `.env` file:
+
+```bash
+docker run -d --name inference-proxy \
+    -p 8000:8000 \
+    -v ./config.toml:/app/config.toml:ro \
+    --env-file .env \
+    lmproxy/lmproxy:edge
+```
+
+Any value referenced in the config as `env:<VAR_NAME>` (upstream provider keys,
+client API keys, etc.) can be defined in the `.env` file:
+
+```ini
+OPENAI_API_KEY=sk-...
+ANTHROPIC_API_KEY=sk-ant-...
+```
+
+See [examples/docker-compose.yml](examples/docker-compose.yml) for a Docker Compose setup.
+
+To bake additional Python packages into the image (e.g. for
+[database log storage](#database-connector)), build it with the `EXTRA_PIP_PACKAGES` argument:
+
+```bash
+docker build --build-arg EXTRA_PIP_PACKAGES="inference-proxy-db-connector sqlalchemy psycopg2-binary" -t inference-proxy .
+```
+
 
 ## 📝 Configuration<a id="-configuration"></a>
 
@@ -220,6 +257,7 @@ Inference Proxy is configured through a TOML/YAML/JSON/Python file that specifie
 host = "0.0.0.0"  # Interface to bind to
 port = 8000       # Port to listen on
 dev_autoreload = false  # Enable for development
+print_stream = false    # Print LLM requests / responses to stdout, see "Debugging"
 
 # API key validation function (optional)
 api_key_check = "lm_proxy.api_key_check.check_api_key_in_config"
@@ -258,7 +296,7 @@ api_keys = [
 [[loggers]]
 class = 'lm_proxy.loggers.BaseLogger'
 [loggers.log_writer]
-class = 'lm_proxy.loggers.log_writers.JsonLogWriter'
+class = 'lm_proxy.loggers.JsonLogWriter'
 file_name = 'storage/json.log'
 [loggers.entry_transformer]
 class = 'lm_proxy.loggers.LogEntryTransformer'
@@ -714,6 +752,7 @@ prefix = "SECURITY_AUDIT"
 For more detailed information, check out these articles:
 - [HTTP Header Management](https://github.com/Nayjest/lm-proxy/blob/main/doc/http_headers.md)
 - [Configuring fallbacks](https://github.com/Nayjest/lm-proxy/blob/main/doc/fallback.md)
+- [HTTPS / TLS Setup](https://github.com/Nayjest/lm-proxy/blob/main/doc/https.md)
 
 
 ## 🚧 Known Limitations<a id="-known-limitations"></a>
@@ -725,7 +764,9 @@ For more detailed information, check out these articles:
 
 ## 🔍 Debugging<a id="-debugging"></a>
 
-### Overview
+### Debugging Mode<a id="debugging-mode"></a>
+
+#### Overview
 When **debugging mode** is enabled,
 Inference Proxy provides detailed logging information to help diagnose issues:
 - Stack traces for exceptions are shown in the console
@@ -734,7 +775,7 @@ Inference Proxy provides detailed logging information to help diagnose issues:
 > **Warning** ⚠️  
 > Never enable debugging mode in production environments, as it may expose sensitive information to the application logs.
 
-### Enabling Debugging Mode
+#### Enabling Debugging Mode
 To enable debugging, set the `LM_PROXY_DEBUG` environment variable to a truthy value (e.g., "1", "true", "yes").
 > **Tip** 💡  
 > Environment variables can also be defined in a `.env` file.
@@ -746,6 +787,30 @@ Alternatively, you can enable or disable debugging via the command-line argument
 > **Note** ℹ️   
 > CLI arguments override environment variable settings.
 
+### Printing LLM Requests and Responses<a id="printing-llm-requests-and-responses"></a>
+
+The `print_stream` configuration option makes Inference Proxy print every LLM request
+and response to stdout, with the response appearing token by token as it is generated:
+
+```toml
+print_stream = true
+```
+
+```text
+Requesting LLM gpt-5:
+    [User]:
+        What is the capital of France?
+LLM Response:
+    The capital of France is Paris.
+```
+
+This works for both streaming and non-streaming client requests.
+
+> **Warning** ⚠️  
+> Intended for local development and troubleshooting only.
+> Prompts and generated content are written to the console in plain text,
+> and the output of requests processed in parallel is interleaved.
+
 
 ## 🤝 Contributing<a id="-contributing"></a>
 

{inference_proxy-3.2.2 → inference_proxy-3.3.0}/README.md

@@ -6,7 +6,7 @@
   <a href="https://pypi.org/project/lm-proxy/"><img src="https://img.shields.io/pypi/v/lm-proxy?color=blue" alt="PyPI"></a>
   <a href="https://github.com/Nayjest/lm-proxy/actions/workflows/tests.yml"><img src="https://github.com/Nayjest/lm-proxy/actions/workflows/tests.yml/badge.svg" alt="Tests"></a>
   <a href="https://github.com/Nayjest/lm-proxy/actions/workflows/code-style.yml"><img src="https://github.com/Nayjest/lm-proxy/actions/workflows/code-style.yml/badge.svg" alt="Code Style"></a>
-  <img src="https://raw.githubusercontent.com/Nayjest/lm-proxy/main/coverage.svg" alt="Code Coverage">
+  <img src="https://raw.githubusercontent.com/Nayjest/lm-proxy/coverage-badge/coverage.svg" alt="Code Coverage">
   <a href="https://www.bestpractices.dev/projects/11364"><img src="https://www.bestpractices.dev/projects/11364/badge"></a>
   <br>
   <a href="https://github.com/vshymanskyy/StandWithUkraine/blob/main/README.md"><img src="https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/refs/heads/main/badges/StandWithUkraine.svg" alt="Stand With Ukraine"></a>
@@ -26,6 +26,7 @@ It works as a drop-in replacement for OpenAI's API, allowing you to switch betwe
 - [Getting Started](#-getting-started)
   - [Installation](#installation)
   - [Quick Start](#quick-start)
+  - [Run with Docker](#run-with-docker)
 - [Configuration](#-configuration)
   - [Basic Structure](#basic-structure)
   - [Environment Variables](#environment-variables)
@@ -49,6 +50,8 @@ It works as a drop-in replacement for OpenAI's API, allowing you to switch betwe
 - [Guides & Reference](#-guides--reference)
 - [Known Limitations](#-known-limitations)
 - [Debugging](#-debugging)
+  - [Debugging Mode](#debugging-mode)
+  - [Printing LLM Requests and Responses](#printing-llm-requests-and-responses)
 - [Contributing](#-contributing)
 - [License](#-license)
 
@@ -145,6 +148,39 @@ completion = client.chat.completions.create(
 )
 ```
 
+### Run with Docker<a id="run-with-docker"></a>
+
+Official images are published to GitHub Container Registry on every release
+(`latest`, `3`, `3.2`, `3.2.2`, …; `edge` tracks the `main` branch).
+They include the Anthropic and Google connectors and YAML config support out of the box.
+
+Mount your `config.toml` and provide API keys via an `.env` file:
+
+```bash
+docker run -d --name inference-proxy \
+    -p 8000:8000 \
+    -v ./config.toml:/app/config.toml:ro \
+    --env-file .env \
+    lmproxy/lmproxy:edge
+```
+
+Any value referenced in the config as `env:<VAR_NAME>` (upstream provider keys,
+client API keys, etc.) can be defined in the `.env` file:
+
+```ini
+OPENAI_API_KEY=sk-...
+ANTHROPIC_API_KEY=sk-ant-...
+```
+
+See [examples/docker-compose.yml](examples/docker-compose.yml) for a Docker Compose setup.
+
+To bake additional Python packages into the image (e.g. for
+[database log storage](#database-connector)), build it with the `EXTRA_PIP_PACKAGES` argument:
+
+```bash
+docker build --build-arg EXTRA_PIP_PACKAGES="inference-proxy-db-connector sqlalchemy psycopg2-binary" -t inference-proxy .
+```
+
 
 ## 📝 Configuration<a id="-configuration"></a>
 
@@ -155,6 +191,7 @@ Inference Proxy is configured through a TOML/YAML/JSON/Python file that specifie
 host = "0.0.0.0"  # Interface to bind to
 port = 8000       # Port to listen on
 dev_autoreload = false  # Enable for development
+print_stream = false    # Print LLM requests / responses to stdout, see "Debugging"
 
 # API key validation function (optional)
 api_key_check = "lm_proxy.api_key_check.check_api_key_in_config"
@@ -193,7 +230,7 @@ api_keys = [
 [[loggers]]
 class = 'lm_proxy.loggers.BaseLogger'
 [loggers.log_writer]
-class = 'lm_proxy.loggers.log_writers.JsonLogWriter'
+class = 'lm_proxy.loggers.JsonLogWriter'
 file_name = 'storage/json.log'
 [loggers.entry_transformer]
 class = 'lm_proxy.loggers.LogEntryTransformer'
@@ -649,6 +686,7 @@ prefix = "SECURITY_AUDIT"
 For more detailed information, check out these articles:
 - [HTTP Header Management](https://github.com/Nayjest/lm-proxy/blob/main/doc/http_headers.md)
 - [Configuring fallbacks](https://github.com/Nayjest/lm-proxy/blob/main/doc/fallback.md)
+- [HTTPS / TLS Setup](https://github.com/Nayjest/lm-proxy/blob/main/doc/https.md)
 
 
 ## 🚧 Known Limitations<a id="-known-limitations"></a>
@@ -660,7 +698,9 @@ For more detailed information, check out these articles:
 
 ## 🔍 Debugging<a id="-debugging"></a>
 
-### Overview
+### Debugging Mode<a id="debugging-mode"></a>
+
+#### Overview
 When **debugging mode** is enabled,
 Inference Proxy provides detailed logging information to help diagnose issues:
 - Stack traces for exceptions are shown in the console
@@ -669,7 +709,7 @@ Inference Proxy provides detailed logging information to help diagnose issues:
 > **Warning** ⚠️  
 > Never enable debugging mode in production environments, as it may expose sensitive information to the application logs.
 
-### Enabling Debugging Mode
+#### Enabling Debugging Mode
 To enable debugging, set the `LM_PROXY_DEBUG` environment variable to a truthy value (e.g., "1", "true", "yes").
 > **Tip** 💡  
 > Environment variables can also be defined in a `.env` file.
@@ -681,6 +721,30 @@ Alternatively, you can enable or disable debugging via the command-line argument
 > **Note** ℹ️   
 > CLI arguments override environment variable settings.
 
+### Printing LLM Requests and Responses<a id="printing-llm-requests-and-responses"></a>
+
+The `print_stream` configuration option makes Inference Proxy print every LLM request
+and response to stdout, with the response appearing token by token as it is generated:
+
+```toml
+print_stream = true
+```
+
+```text
+Requesting LLM gpt-5:
+    [User]:
+        What is the capital of France?
+LLM Response:
+    The capital of France is Paris.
+```
+
+This works for both streaming and non-streaming client requests.
+
+> **Warning** ⚠️  
+> Intended for local development and troubleshooting only.
+> Prompts and generated content are written to the console in plain text,
+> and the output of requests processed in parallel is interleaved.
+
 
 ## 🤝 Contributing<a id="-contributing"></a>
 

{inference_proxy-3.2.2 → inference_proxy-3.3.0}/lm_proxy/__main__.py

@@ -2,6 +2,5 @@
 
 from .app import cli_app
 
-
 if __name__ == "__main__":
     cli_app()

{inference_proxy-3.2.2 → inference_proxy-3.3.0}/lm_proxy/app.py

@@ -13,7 +13,6 @@ from .core import chat_completions
 from .models_endpoint import models
 from .errors import OpenAIHTTPException
 
-
 cli_app = typer.Typer()
 
 

{inference_proxy-3.2.2 → inference_proxy-3.3.0}/lm_proxy/base_types.py

@@ -9,7 +9,6 @@ import microcore as mc
 from pydantic import BaseModel
 from starlette.requests import Request
 
-
 if TYPE_CHECKING:
     from .config import Group
 

{inference_proxy-3.2.2 → inference_proxy-3.3.0}/lm_proxy/bootstrap.py

@@ -84,7 +84,7 @@ class Env:
         # initialize connections
         env.connections = {}
         for conn_name, conn_config in env.config.connections.items():
-            logging.info("Initializing \"%s\" LLM proxy connection...", ui.green(conn_name))
+            logging.info("Initializing '%s' LLM proxy connection...", ui.green(conn_name))
             try:
                 fn_or_config = resolve_instance_or_callable(conn_config, allow_types=[dict])
                 if _is_async_callable(fn_or_config):
@@ -94,7 +94,10 @@ class Env:
                     env.connections[conn_name] = mc.env().llm_async_function
             except mc.LLMConfigError as e:
                 raise ValueError(f"Error in configuration for connection '{conn_name}': {e}") from e
-
+        if config.print_stream:
+            mc.use_logging(stream=True)
+            # Proxied prompts are printed in full, without "...(output was truncated)..."
+            mc.logging.LoggingConfig.STRIP_REQUEST_LINES = None
         logging.info("Done initializing %d connections.", len(env.connections))
 
 

{inference_proxy-3.2.2 → inference_proxy-3.3.0}/lm_proxy/config.py

@@ -90,6 +90,10 @@ class Config(BaseModel):
     )
     components: dict[str, Union[str, Callable, dict]] = Field(default_factory=dict)
     before: list[Union[str, Callable, dict]] = Field(default_factory=list)
+    print_stream: bool = Field(
+        default=False,
+        description="Prints LLM requests / responses to stdout"
+    )
 
     def __init__(self, **data):
         super().__init__(**data)

inference_proxy-3.3.0/lm_proxy/config_loaders/json.py

@@ -0,0 +1,11 @@
+"""JSON configuration loader."""
+
+import json
+
+
+def load_json_config(config_path: str) -> dict:
+    """Loads configuration from a JSON file."""
+    # utf-8-sig skips the byte order mark that Windows text editors may add,
+    # the JSON parser rejects it
+    with open(config_path, "r", encoding="utf-8-sig") as f:
+        return json.load(f)

inference_proxy-3.3.0/lm_proxy/config_loaders/toml.py

@@ -0,0 +1,11 @@
+"""TOML configuration loader."""
+
+import tomllib
+
+
+def load_toml_config(config_path: str) -> dict:
+    """Loads configuration from a TOML file."""
+    # utf-8-sig skips the byte order mark that Windows text editors may add,
+    # tomllib rejects it
+    with open(config_path, "r", encoding="utf-8-sig") as f:
+        return tomllib.loads(f.read())

{inference_proxy-3.2.2 → inference_proxy-3.3.0}/lm_proxy/core.py

@@ -11,6 +11,7 @@ import inspect
 from datetime import datetime
 from typing import Optional
 
+import microcore as mc
 from fastapi import HTTPException
 from openai.types.chat.chat_completion import Choice
 from starlette.requests import Request
@@ -65,6 +66,30 @@ def resolve_connection_and_model(config: Config, external_model: str) -> tuple[s
     )
 
 
+def print_llm_request(ctx: RequestContext):
+    """
+    Prints the LLM request to stdout when the `print_stream` option is enabled.
+
+    LM-Proxy queries the LLM API function directly instead of using microcore.allm(),
+    so microcore request / response handlers are triggered explicitly.
+    """
+    if env.config.print_stream:
+        for handler in mc.env().llm_before_handlers:
+            handler(ctx.request.messages, **ctx.llm_params)
+
+
+def finish_printing_llm_response(ctx: RequestContext):
+    """
+    Completes printing of the LLM response to stdout, in particular terminates
+    the line of the streamed output, when the `print_stream` option is enabled.
+
+    See print_llm_request().
+    """
+    if env.config.print_stream:
+        for handler in mc.env().llm_after_handlers:
+            handler(ctx.response)
+
+
 async def process_stream(
     async_llm_func,
     request: ChatCompletionRequest,
@@ -121,6 +146,7 @@ async def process_stream(
         except Exception as e:
             ctx.error = e
             yield make_chunk(error={"message": str(e), "type": type(e).__name__})
+        finish_printing_llm_response(ctx)
 
     yield make_chunk(finish_reason="error" if ctx.error else "stop")
     yield "data: [DONE]\n\n"
@@ -252,6 +278,7 @@ async def chat_completions(request: ChatCompletionRequest, raw_request: Request)
     async_llm_func = env.connections[connection]
 
     logging.info("Querying LLM... params: %s", ctx.llm_params)
+    print_llm_request(ctx)
     if request.stream:
         return StreamingResponse(
             process_stream(async_llm_func, request, ctx),
@@ -261,9 +288,11 @@ async def chat_completions(request: ChatCompletionRequest, raw_request: Request)
     try:
         out = await async_llm_func(request.messages, **ctx.llm_params)
         ctx.response = out
+        finish_printing_llm_response(ctx)
         logging.info("LLM response: %s", out)
     except Exception as e:
         ctx.error = e
+        finish_printing_llm_response(ctx)
         await log_non_blocking(ctx)
         error_details = f" [{type(e).__name__}]: {e}" if env.debug else ""
         logging.exception(e)

{inference_proxy-3.2.2 → inference_proxy-3.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "inference-proxy"
-version = "3.2.2"
+version = "3.3.0"
 description = "Inference Proxy is an OpenAI-compatible http proxy server for inferencing various LLMs capable of working with Google, Anthropic, OpenAI APIs, local PyTorch inference, etc."
 readme = "README.md"
 keywords = ["llm", "large language models", "ai", "gpt", "openai", "proxy", "http", "proxy-server", "llm gateway", "openai", "anthropic", "google genai"]
@@ -35,8 +35,8 @@ documentation = "https://github.com/Nayjest/lm-proxy#readme"
 license = { file = "LICENSE" }
 
 dependencies = [
-    "ai-microcore>=5.1.2,<7",
-    "fastapi>=0.121.3,<1",
+    "ai-microcore>=6.6.1,<7",
+    "fastapi>=0.136.3,<1",
     "uvicorn>=0.41.0",
     "typer>=0.24.0,<1",
     "requests>=2.32.5,<3",

inference_proxy-3.2.2/lm_proxy/config_loaders/json.py

@@ -1,9 +0,0 @@
-"""JSON configuration loader."""
-
-import json
-
-
-def load_json_config(config_path: str) -> dict:
-    """Loads configuration from a JSON file."""
-    with open(config_path, "r", encoding="utf-8") as f:
-        return json.load(f)

inference_proxy-3.2.2/lm_proxy/config_loaders/toml.py

@@ -1,9 +0,0 @@
-"""TOML configuration loader."""
-
-import tomllib
-
-
-def load_toml_config(config_path: str) -> dict:
-    """Loads configuration from a TOML file."""
-    with open(config_path, "rb") as f:
-        return tomllib.load(f)