inference-proxy 3.2.3__tar.gz → 3.3.1__tar.gz

{inference_proxy-3.2.3 → inference_proxy-3.3.1}/PKG-INFO

@@ -1,28 +1,8 @@
 Metadata-Version: 2.4
 Name: inference-proxy
-Version: 3.2.3
+Version: 3.3.1
 Summary: Inference Proxy is an OpenAI-compatible http proxy server for inferencing various LLMs capable of working with Google, Anthropic, OpenAI APIs, local PyTorch inference, etc.
-License: MIT License
-         
-         Copyright (c) 2025–2026 Vitalii Stepanenko
-         
-         Permission is hereby granted, free of charge, to any person obtaining a copy
-         of this software and associated documentation files (the "Software"), to deal
-         in the Software without restriction, including without limitation the rights
-         to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-         copies of the Software, and to permit persons to whom the Software is
-         furnished to do so, subject to the following conditions:
-         
-         The above copyright notice and this permission notice shall be included in all
-         copies or substantial portions of the Software.
-         
-         THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-         IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-         FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-         AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-         LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-         OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-         SOFTWARE.
+License-Expression: MIT
 License-File: LICENSE
 Keywords: llm,large language models,ai,gpt,openai,proxy,http,proxy-server,llm gateway,openai,anthropic,google genai
 Author: Vitalii Stepanenko
@@ -36,7 +16,6 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Intended Audience :: Developers
 Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
@@ -46,16 +25,16 @@ Provides-Extra: all
 Provides-Extra: anthropic
 Provides-Extra: google
 Provides-Extra: test
-Requires-Dist: ai-microcore (>=6.2.1,<7)
+Requires-Dist: ai-microcore (>=6.7.1,<7)
 Requires-Dist: anthropic (>=0.77,<1) ; extra == "all"
 Requires-Dist: anthropic (>=0.77,<1) ; extra == "anthropic"
 Requires-Dist: fastapi (>=0.136.3,<1)
 Requires-Dist: google-genai (>=1.62.0,<2) ; extra == "all"
 Requires-Dist: google-genai (>=1.62.0,<2) ; extra == "google"
 Requires-Dist: pydantic (>=2.12.5,<2.13.0)
-Requires-Dist: pytest (>=8.4.2,<8.5.0) ; extra == "test"
-Requires-Dist: pytest-asyncio (>=1.2.0,<1.3.0) ; extra == "test"
-Requires-Dist: pytest-cov (>=7.0.0,<7.1.0) ; extra == "test"
+Requires-Dist: pytest (>=9.1.1,<9.2.0) ; extra == "test"
+Requires-Dist: pytest-asyncio (>=1.4.0,<1.5.0) ; extra == "test"
+Requires-Dist: pytest-cov (>=7.1.0,<7.2.0) ; extra == "test"
 Requires-Dist: requests (>=2.32.5,<3)
 Requires-Dist: typer (>=0.24.0,<1)
 Requires-Dist: uvicorn (>=0.41.0)
@@ -116,6 +95,8 @@ It works as a drop-in replacement for OpenAI's API, allowing you to switch betwe
 - [Guides & Reference](#-guides--reference)
 - [Known Limitations](#-known-limitations)
 - [Debugging](#-debugging)
+  - [Debugging Mode](#debugging-mode)
+  - [Printing LLM Requests and Responses](#printing-llm-requests-and-responses)
 - [Contributing](#-contributing)
 - [License](#-license)
 
@@ -215,7 +196,7 @@ completion = client.chat.completions.create(
 ### Run with Docker<a id="run-with-docker"></a>
 
 Official images are published to GitHub Container Registry on every release
-(`latest`, `3`, `3.2`, `3.2.2`, …; `edge` tracks the `main` branch).
+(`latest`, `3`, `3.3`, `3.3.0`, …; `edge` tracks the `main` branch).
 They include the Anthropic and Google connectors and YAML config support out of the box.
 
 Mount your `config.toml` and provide API keys via an `.env` file:
@@ -255,6 +236,7 @@ Inference Proxy is configured through a TOML/YAML/JSON/Python file that specifie
 host = "0.0.0.0"  # Interface to bind to
 port = 8000       # Port to listen on
 dev_autoreload = false  # Enable for development
+print_stream = false    # Print LLM requests / responses to stdout, see "Debugging"
 
 # API key validation function (optional)
 api_key_check = "lm_proxy.api_key_check.check_api_key_in_config"
@@ -761,7 +743,9 @@ For more detailed information, check out these articles:
 
 ## 🔍 Debugging<a id="-debugging"></a>
 
-### Overview
+### Debugging Mode<a id="debugging-mode"></a>
+
+#### Overview
 When **debugging mode** is enabled,
 Inference Proxy provides detailed logging information to help diagnose issues:
 - Stack traces for exceptions are shown in the console
@@ -770,7 +754,7 @@ Inference Proxy provides detailed logging information to help diagnose issues:
 > **Warning** ⚠️  
 > Never enable debugging mode in production environments, as it may expose sensitive information to the application logs.
 
-### Enabling Debugging Mode
+#### Enabling Debugging Mode
 To enable debugging, set the `LM_PROXY_DEBUG` environment variable to a truthy value (e.g., "1", "true", "yes").
 > **Tip** 💡  
 > Environment variables can also be defined in a `.env` file.
@@ -782,6 +766,30 @@ Alternatively, you can enable or disable debugging via the command-line argument
 > **Note** ℹ️   
 > CLI arguments override environment variable settings.
 
+### Printing LLM Requests and Responses<a id="printing-llm-requests-and-responses"></a>
+
+The `print_stream` configuration option makes Inference Proxy print every LLM request
+and response to stdout, with the response appearing token by token as it is generated:
+
+```toml
+print_stream = true
+```
+
+```text
+Requesting LLM gpt-5:
+    [User]:
+        What is the capital of France?
+LLM Response:
+    The capital of France is Paris.
+```
+
+This works for both streaming and non-streaming client requests.
+
+> **Warning** ⚠️  
+> Intended for local development and troubleshooting only.
+> Prompts and generated content are written to the console in plain text,
+> and the output of requests processed in parallel is interleaved.
+
 
 ## 🤝 Contributing<a id="-contributing"></a>
 

{inference_proxy-3.2.3 → inference_proxy-3.3.1}/README.md

@@ -50,6 +50,8 @@ It works as a drop-in replacement for OpenAI's API, allowing you to switch betwe
 - [Guides & Reference](#-guides--reference)
 - [Known Limitations](#-known-limitations)
 - [Debugging](#-debugging)
+  - [Debugging Mode](#debugging-mode)
+  - [Printing LLM Requests and Responses](#printing-llm-requests-and-responses)
 - [Contributing](#-contributing)
 - [License](#-license)
 
@@ -149,7 +151,7 @@ completion = client.chat.completions.create(
 ### Run with Docker<a id="run-with-docker"></a>
 
 Official images are published to GitHub Container Registry on every release
-(`latest`, `3`, `3.2`, `3.2.2`, …; `edge` tracks the `main` branch).
+(`latest`, `3`, `3.3`, `3.3.0`, …; `edge` tracks the `main` branch).
 They include the Anthropic and Google connectors and YAML config support out of the box.
 
 Mount your `config.toml` and provide API keys via an `.env` file:
@@ -189,6 +191,7 @@ Inference Proxy is configured through a TOML/YAML/JSON/Python file that specifie
 host = "0.0.0.0"  # Interface to bind to
 port = 8000       # Port to listen on
 dev_autoreload = false  # Enable for development
+print_stream = false    # Print LLM requests / responses to stdout, see "Debugging"
 
 # API key validation function (optional)
 api_key_check = "lm_proxy.api_key_check.check_api_key_in_config"
@@ -695,7 +698,9 @@ For more detailed information, check out these articles:
 
 ## 🔍 Debugging<a id="-debugging"></a>
 
-### Overview
+### Debugging Mode<a id="debugging-mode"></a>
+
+#### Overview
 When **debugging mode** is enabled,
 Inference Proxy provides detailed logging information to help diagnose issues:
 - Stack traces for exceptions are shown in the console
@@ -704,7 +709,7 @@ Inference Proxy provides detailed logging information to help diagnose issues:
 > **Warning** ⚠️  
 > Never enable debugging mode in production environments, as it may expose sensitive information to the application logs.
 
-### Enabling Debugging Mode
+#### Enabling Debugging Mode
 To enable debugging, set the `LM_PROXY_DEBUG` environment variable to a truthy value (e.g., "1", "true", "yes").
 > **Tip** 💡  
 > Environment variables can also be defined in a `.env` file.
@@ -716,6 +721,30 @@ Alternatively, you can enable or disable debugging via the command-line argument
 > **Note** ℹ️   
 > CLI arguments override environment variable settings.
 
+### Printing LLM Requests and Responses<a id="printing-llm-requests-and-responses"></a>
+
+The `print_stream` configuration option makes Inference Proxy print every LLM request
+and response to stdout, with the response appearing token by token as it is generated:
+
+```toml
+print_stream = true
+```
+
+```text
+Requesting LLM gpt-5:
+    [User]:
+        What is the capital of France?
+LLM Response:
+    The capital of France is Paris.
+```
+
+This works for both streaming and non-streaming client requests.
+
+> **Warning** ⚠️  
+> Intended for local development and troubleshooting only.
+> Prompts and generated content are written to the console in plain text,
+> and the output of requests processed in parallel is interleaved.
+
 
 ## 🤝 Contributing<a id="-contributing"></a>
 

{inference_proxy-3.2.3 → inference_proxy-3.3.1}/lm_proxy/bootstrap.py

@@ -94,7 +94,10 @@ class Env:
                     env.connections[conn_name] = mc.env().llm_async_function
             except mc.LLMConfigError as e:
                 raise ValueError(f"Error in configuration for connection '{conn_name}': {e}") from e
-
+        if config.print_stream:
+            mc.use_logging(stream=True)
+            # Proxied prompts are printed in full, without "...(output was truncated)..."
+            mc.logging.LoggingConfig.STRIP_REQUEST_LINES = None
         logging.info("Done initializing %d connections.", len(env.connections))
 
 

{inference_proxy-3.2.3 → inference_proxy-3.3.1}/lm_proxy/config.py

@@ -90,6 +90,10 @@ class Config(BaseModel):
     )
     components: dict[str, Union[str, Callable, dict]] = Field(default_factory=dict)
     before: list[Union[str, Callable, dict]] = Field(default_factory=list)
+    print_stream: bool = Field(
+        default=False,
+        description="Prints LLM requests / responses to stdout"
+    )
 
     def __init__(self, **data):
         super().__init__(**data)

inference_proxy-3.3.1/lm_proxy/config_loaders/json.py

@@ -0,0 +1,11 @@
+"""JSON configuration loader."""
+
+import json
+
+
+def load_json_config(config_path: str) -> dict:
+    """Loads configuration from a JSON file."""
+    # utf-8-sig skips the byte order mark that Windows text editors may add,
+    # the JSON parser rejects it
+    with open(config_path, "r", encoding="utf-8-sig") as f:
+        return json.load(f)

inference_proxy-3.3.1/lm_proxy/config_loaders/toml.py

@@ -0,0 +1,11 @@
+"""TOML configuration loader."""
+
+import tomllib
+
+
+def load_toml_config(config_path: str) -> dict:
+    """Loads configuration from a TOML file."""
+    # utf-8-sig skips the byte order mark that Windows text editors may add,
+    # tomllib rejects it
+    with open(config_path, "r", encoding="utf-8-sig") as f:
+        return tomllib.loads(f.read())

{inference_proxy-3.2.3 → inference_proxy-3.3.1}/lm_proxy/core.py

@@ -11,6 +11,7 @@ import inspect
 from datetime import datetime
 from typing import Optional
 
+import microcore as mc
 from fastapi import HTTPException
 from openai.types.chat.chat_completion import Choice
 from starlette.requests import Request
@@ -65,6 +66,30 @@ def resolve_connection_and_model(config: Config, external_model: str) -> tuple[s
     )
 
 
+def print_llm_request(ctx: RequestContext):
+    """
+    Prints the LLM request to stdout when the `print_stream` option is enabled.
+
+    LM-Proxy queries the LLM API function directly instead of using microcore.allm(),
+    so microcore request / response handlers are triggered explicitly.
+    """
+    if env.config.print_stream:
+        for handler in mc.env().llm_before_handlers:
+            handler(ctx.request.messages, **ctx.llm_params)
+
+
+def finish_printing_llm_response(ctx: RequestContext):
+    """
+    Completes printing of the LLM response to stdout, in particular terminates
+    the line of the streamed output, when the `print_stream` option is enabled.
+
+    See print_llm_request().
+    """
+    if env.config.print_stream:
+        for handler in mc.env().llm_after_handlers:
+            handler(ctx.response)
+
+
 async def process_stream(
     async_llm_func,
     request: ChatCompletionRequest,
@@ -121,6 +146,7 @@ async def process_stream(
         except Exception as e:
             ctx.error = e
             yield make_chunk(error={"message": str(e), "type": type(e).__name__})
+        finish_printing_llm_response(ctx)
 
     yield make_chunk(finish_reason="error" if ctx.error else "stop")
     yield "data: [DONE]\n\n"
@@ -252,6 +278,7 @@ async def chat_completions(request: ChatCompletionRequest, raw_request: Request)
     async_llm_func = env.connections[connection]
 
     logging.info("Querying LLM... params: %s", ctx.llm_params)
+    print_llm_request(ctx)
     if request.stream:
         return StreamingResponse(
             process_stream(async_llm_func, request, ctx),
@@ -261,9 +288,11 @@ async def chat_completions(request: ChatCompletionRequest, raw_request: Request)
     try:
         out = await async_llm_func(request.messages, **ctx.llm_params)
         ctx.response = out
+        finish_printing_llm_response(ctx)
         logging.info("LLM response: %s", out)
     except Exception as e:
         ctx.error = e
+        finish_printing_llm_response(ctx)
         await log_non_blocking(ctx)
         error_details = f" [{type(e).__name__}]: {e}" if env.debug else ""
         logging.exception(e)

{inference_proxy-3.2.3 → inference_proxy-3.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "inference-proxy"
-version = "3.2.3"
+version = "3.3.1"
 description = "Inference Proxy is an OpenAI-compatible http proxy server for inferencing various LLMs capable of working with Google, Anthropic, OpenAI APIs, local PyTorch inference, etc."
 readme = "README.md"
 keywords = ["llm", "large language models", "ai", "gpt", "openai", "proxy", "http", "proxy-server", "llm gateway", "openai", "anthropic", "google genai"]
@@ -11,7 +11,6 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
-    "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
     "Intended Audience :: Developers",
     "Topic :: Software Development :: Libraries :: Application Frameworks",
@@ -32,10 +31,10 @@ maintainers = [
 homepage = "https://github.com/Nayjest/lm-proxy"
 repository = "https://github.com/Nayjest/lm-proxy"
 documentation = "https://github.com/Nayjest/lm-proxy#readme"
-license = { file = "LICENSE" }
+license = "MIT"
 
 dependencies = [
-    "ai-microcore>=6.2.1,<7",
+    "ai-microcore>=6.7.1,<7",
     "fastapi>=0.136.3,<1",
     "uvicorn>=0.41.0",
     "typer>=0.24.0,<1",
@@ -46,9 +45,9 @@ dependencies = [
 
 [project.optional-dependencies]
 test = [
-   "pytest~=8.4.2",
-   "pytest-asyncio~=1.2.0",
-   "pytest-cov~=7.0.0"
+   "pytest~=9.1.1",
+   "pytest-asyncio~=1.4.0",
+   "pytest-cov~=7.1.0"
 ]
 google = [
     "google-genai>=1.62.0,<2",

inference_proxy-3.2.3/lm_proxy/config_loaders/json.py

@@ -1,9 +0,0 @@
-"""JSON configuration loader."""
-
-import json
-
-
-def load_json_config(config_path: str) -> dict:
-    """Loads configuration from a JSON file."""
-    with open(config_path, "r", encoding="utf-8") as f:
-        return json.load(f)

inference_proxy-3.2.3/lm_proxy/config_loaders/toml.py

@@ -1,9 +0,0 @@
-"""TOML configuration loader."""
-
-import tomllib
-
-
-def load_toml_config(config_path: str) -> dict:
-    """Loads configuration from a TOML file."""
-    with open(config_path, "rb") as f:
-        return tomllib.load(f)