mlproxy-py 0.1.1__tar.gz

mlproxy_py-0.1.1/.github/workflows/ci.yml

@@ -0,0 +1,24 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install hatchling ruff
+      - run: ruff check .
+      - run: pip install .
+      - run: pip install pytest pytest-asyncio httpx
+      - run: pytest

mlproxy_py-0.1.1/.gitignore

@@ -0,0 +1,218 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+# Temporary file for partial code execution
+tempCodeRunnerFile.py
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml

mlproxy_py-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Felix
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

mlproxy_py-0.1.1/PKG-INFO

@@ -0,0 +1,131 @@
+Metadata-Version: 2.4
+Name: mlproxy-py
+Version: 0.1.1
+Summary: SLA/QoS-aware reverse proxy for ML inference workloads (batching, routing, latency metrics).
+Author: Kubenew
+License: MIT
+License-File: LICENSE
+Keywords: asyncio,batching,inference,llm,ml,qos,reverse-proxy
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: AsyncIO
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: fastapi>=0.111.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: prometheus-client>=0.20.0
+Requires-Dist: pydantic>=2.7.0
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: typer>=0.12.3
+Requires-Dist: uvicorn>=0.30.0
+Provides-Extra: dev
+Requires-Dist: httpx>=0.27.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.21; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.1; extra == 'dev'
+Provides-Extra: test
+Requires-Dist: httpx>=0.27.0; extra == 'test'
+Requires-Dist: pytest-asyncio>=0.21; extra == 'test'
+Requires-Dist: pytest>=7.0; extra == 'test'
+Description-Content-Type: text/markdown
+
+# mlproxy-py
+
+[![PyPI](https://img.shields.io/pypi/v/mlproxy-py)](https://pypi.org/project/mlproxy-py/)
+[![Python Versions](https://img.shields.io/pypi/pyversions/mlproxy-py)](https://pypi.org/project/mlproxy-py/)
+[![License](https://img.shields.io/pypi/l/mlproxy-py)](https://github.com/Kubenew/mlproxy-py/blob/main/LICENSE)
+[![GitHub stars](https://img.shields.io/github/stars/Kubenew/mlproxy-py?style=flat&logo=github)](https://github.com/Kubenew/mlproxy-py)
+[![Downloads](https://img.shields.io/pepy.tech/dt/mlproxy-py)](https://pepy.tech/project/mlproxy-py)
+
+**mlproxy-py** is a minimal ML inference reverse proxy with QoS-aware routing.
+
+Designed for LLM / ML inference workloads where routing decisions should be based on latency, SLA targets, backend health, queue depth, and batching potential.
+
+## Features
+
+- Reverse proxy for JSON inference requests
+- Backends grouped into model pools
+- SLA-aware routing (choose lowest latency backend)
+- Optional micro-batching (collect requests for N ms)
+- Concurrent health checks with connection pooling
+- Prometheus metrics (request count, latency, backend latency)
+
+## Quickstart
+
+### Install
+
+```bash
+pip install mlproxy-py
+```
+
+### Run proxy
+
+```bash
+mlproxy run -c examples/config.yml
+```
+
+### Send request
+
+```bash
+curl -X POST http://localhost:7000/infer/modelA \
+  -H "Content-Type: application/json" \
+  -d '{"text":"hello"}'
+```
+
+## Architecture
+
+```
+Client ──POST /infer/{model}──► FastAPI
+                                    │
+                          ┌─────────▼──────────┐
+                          │  ModelRouter       │
+                          │  choose_backend()  │
+                          │  (score = latency  │
+                          │   + active_req*5)  │
+                          └─────────┬──────────┘
+                                    │ backend URL
+                          ┌─────────▼──────────┐
+                          │  forward_json()    │
+                          │  (httpx conn pool) │
+                          └─────────┬──────────┘
+                                    ▼
+                            Backend ML server
+
+       ┌──────────────────┐    ┌──────────────────┐
+       │  BatchQueue      │    │  Healthcheck     │
+       │  (optional per   │    │  (concurrent,    │
+       │   model pool)    │    │   per-backend)   │
+       └──────────────────┘    └──────────────────┘
+```
+
+## Config
+
+See `examples/config.yml`.
+
+## Changelog
+
+### 0.1.1
+
+- **Lifespan pattern**: Migrated from deprecated `@app.on_event("startup")` to FastAPI `lifespan` context manager.
+- **Graceful shutdown**: Batch workers and healthcheck loop are properly cancelled on shutdown.
+- **Connection pooling**: Shared `httpx.AsyncClient` singletons for proxy and healthcheck (was creating a client per request/check).
+- **Concurrent health checks**: Backends checked in parallel via `asyncio.gather` (was sequential).
+- **Logging**: Added structured `logging` throughout; `--log-level` CLI option.
+- **Bare except fixes**: All `except Exception` blocks re-raise `asyncio.CancelledError`.
+- **Deprecated API fixes**: Replaced `asyncio.get_event_loop()` with `asyncio.get_running_loop()` in batching module.
+- **Build system**: Migrated from `setuptools` to `hatchling`. Added classifiers, keywords, optional dev/test deps, ruff/pytest config.
+- **Tests**: Expanded from 1 test to 15+ tests covering config, router, batching, proxy, healthcheck, and backends.
+
+### 0.1.0
+
+- Initial release: JSON inference proxy, model pools, SLA-aware routing, micro-batching, health checks, Prometheus metrics.
+
+## License
+
+MIT

mlproxy_py-0.1.1/README.md

@@ -0,0 +1,94 @@
+# mlproxy-py
+
+[![PyPI](https://img.shields.io/pypi/v/mlproxy-py)](https://pypi.org/project/mlproxy-py/)
+[![Python Versions](https://img.shields.io/pypi/pyversions/mlproxy-py)](https://pypi.org/project/mlproxy-py/)
+[![License](https://img.shields.io/pypi/l/mlproxy-py)](https://github.com/Kubenew/mlproxy-py/blob/main/LICENSE)
+[![GitHub stars](https://img.shields.io/github/stars/Kubenew/mlproxy-py?style=flat&logo=github)](https://github.com/Kubenew/mlproxy-py)
+[![Downloads](https://img.shields.io/pepy.tech/dt/mlproxy-py)](https://pepy.tech/project/mlproxy-py)
+
+**mlproxy-py** is a minimal ML inference reverse proxy with QoS-aware routing.
+
+Designed for LLM / ML inference workloads where routing decisions should be based on latency, SLA targets, backend health, queue depth, and batching potential.
+
+## Features
+
+- Reverse proxy for JSON inference requests
+- Backends grouped into model pools
+- SLA-aware routing (choose lowest latency backend)
+- Optional micro-batching (collect requests for N ms)
+- Concurrent health checks with connection pooling
+- Prometheus metrics (request count, latency, backend latency)
+
+## Quickstart
+
+### Install
+
+```bash
+pip install mlproxy-py
+```
+
+### Run proxy
+
+```bash
+mlproxy run -c examples/config.yml
+```
+
+### Send request
+
+```bash
+curl -X POST http://localhost:7000/infer/modelA \
+  -H "Content-Type: application/json" \
+  -d '{"text":"hello"}'
+```
+
+## Architecture
+
+```
+Client ──POST /infer/{model}──► FastAPI
+                                    │
+                          ┌─────────▼──────────┐
+                          │  ModelRouter       │
+                          │  choose_backend()  │
+                          │  (score = latency  │
+                          │   + active_req*5)  │
+                          └─────────┬──────────┘
+                                    │ backend URL
+                          ┌─────────▼──────────┐
+                          │  forward_json()    │
+                          │  (httpx conn pool) │
+                          └─────────┬──────────┘
+                                    ▼
+                            Backend ML server
+
+       ┌──────────────────┐    ┌──────────────────┐
+       │  BatchQueue      │    │  Healthcheck     │
+       │  (optional per   │    │  (concurrent,    │
+       │   model pool)    │    │   per-backend)   │
+       └──────────────────┘    └──────────────────┘
+```
+
+## Config
+
+See `examples/config.yml`.
+
+## Changelog
+
+### 0.1.1
+
+- **Lifespan pattern**: Migrated from deprecated `@app.on_event("startup")` to FastAPI `lifespan` context manager.
+- **Graceful shutdown**: Batch workers and healthcheck loop are properly cancelled on shutdown.
+- **Connection pooling**: Shared `httpx.AsyncClient` singletons for proxy and healthcheck (was creating a client per request/check).
+- **Concurrent health checks**: Backends checked in parallel via `asyncio.gather` (was sequential).
+- **Logging**: Added structured `logging` throughout; `--log-level` CLI option.
+- **Bare except fixes**: All `except Exception` blocks re-raise `asyncio.CancelledError`.
+- **Deprecated API fixes**: Replaced `asyncio.get_event_loop()` with `asyncio.get_running_loop()` in batching module.
+- **Build system**: Migrated from `setuptools` to `hatchling`. Added classifiers, keywords, optional dev/test deps, ruff/pytest config.
+- **Tests**: Expanded from 1 test to 15+ tests covering config, router, batching, proxy, healthcheck, and backends.
+
+### 0.1.0
+
+- Initial release: JSON inference proxy, model pools, SLA-aware routing, micro-batching, health checks, Prometheus metrics.
+
+## License
+
+MIT

mlproxy_py-0.1.1/examples/config.yml

@@ -0,0 +1,22 @@
+listen: "0.0.0.0:7000"
+
+models:
+  modelA:
+    sla_ms: 300
+    batching:
+      enabled: true
+      max_batch_size: 8
+      max_wait_ms: 20
+    backends:
+      - url: "http://localhost:8001"
+      - url: "http://localhost:8002"
+
+metrics:
+  enabled: true
+  path: "/metrics"
+
+healthcheck:
+  enabled: true
+  interval_seconds: 5
+  timeout_seconds: 2
+  path: "/health"

mlproxy_py-0.1.1/mlproxy_py/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.1"

mlproxy_py-0.1.1/mlproxy_py/app.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from contextlib import asynccontextmanager
+from typing import Dict, List
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.responses import JSONResponse
+
+from .batching import BatchQueue
+from .metrics import BACKEND_LATENCY, REQ_COUNT, REQ_LATENCY, metrics_response
+from .proxy import forward_json, close_client
+from .router import ModelRouter
+
+logger = logging.getLogger(__name__)
+
+
+def create_app(router: ModelRouter, model_cfg: dict, metrics_enabled: bool, metrics_path: str) -> FastAPI:
+    batch_queues: Dict[str, BatchQueue] = {}
+    worker_tasks: List[asyncio.Task] = []
+
+    for model, cfg in model_cfg.items():
+        batching_cfg = cfg.batching
+        if batching_cfg.enabled:
+            batch_queues[model] = BatchQueue(
+                max_batch_size=batching_cfg.max_batch_size,
+                max_wait_ms=batching_cfg.max_wait_ms,
+            )
+
+    async def batch_worker(model: str):
+        q = batch_queues[model]
+        while True:
+            try:
+                batch = await q.collect_batch()
+                if not batch:
+                    continue
+
+                backend = router.choose_backend(model)
+                if not backend:
+                    for item in batch:
+                        item.future.set_exception(RuntimeError("No healthy backend"))
+                    continue
+
+                payloads = [item.payload for item in batch]
+
+                try:
+                    result = await forward_json(backend, model, {"batch": payloads})
+                    results = result.get("results") or []
+                    for i, item in enumerate(batch):
+                        item.future.set_result(results[i] if i < len(results) else result)
+                except asyncio.CancelledError:
+                    for item in batch:
+                        item.future.set_exception(asyncio.CancelledError())
+                    raise
+                except Exception as e:
+                    for item in batch:
+                        item.future.set_exception(e)
+            except asyncio.CancelledError:
+                logger.info("Batch worker for %s stopped", model)
+                break
+            except Exception:
+                logger.exception("Batch worker for %s crashed", model)
+
+    @asynccontextmanager
+    async def lifespan(app: FastAPI):
+        for model in batch_queues:
+            task = asyncio.create_task(batch_worker(model))
+            worker_tasks.append(task)
+        yield
+        for task in worker_tasks:
+            task.cancel()
+        await asyncio.gather(*worker_tasks, return_exceptions=True)
+        worker_tasks.clear()
+        await close_client()
+
+    app = FastAPI(title="mlproxy-py", lifespan=lifespan)
+
+    @app.post("/infer/{model}")
+    async def infer(model: str, request: Request):
+        if model not in router.pools:
+            raise HTTPException(status_code=404, detail="Model not found")
+
+        payload = await request.json()
+
+        backend = router.choose_backend(model)
+        if not backend:
+            raise HTTPException(status_code=503, detail="No healthy backend")
+
+        start = time.perf_counter()
+        try:
+            if model in batch_queues and "batch" not in payload:
+                data = await batch_queues[model].add(payload)
+            else:
+                data = await forward_json(backend, model, payload)
+        except asyncio.CancelledError:
+            raise
+        except Exception as e:
+            raise HTTPException(status_code=502, detail=str(e))
+
+        latency = time.perf_counter() - start
+
+        REQ_COUNT.labels(model=model, backend=backend.url, status="200").inc()
+        REQ_LATENCY.labels(model=model, backend=backend.url).observe(latency)
+        BACKEND_LATENCY.labels(model=model, backend=backend.url).set(backend.last_latency_ms)
+
+        return JSONResponse(content=data)
+
+    if metrics_enabled:
+        @app.get(metrics_path)
+        async def metrics():
+            return metrics_response()
+
+    return app

mlproxy_py-0.1.1/mlproxy_py/backends.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import time
+
+
+@dataclass
+class Backend:
+    url: str
+    healthy: bool = True
+    last_latency_ms: float = 9999.0
+    active_requests: int = 0
+    last_seen: float = field(default_factory=lambda: time.time())
+
+    def score(self) -> float:
+        # Lower score is better
+        return self.last_latency_ms + (self.active_requests * 5.0)

mlproxy_py-0.1.1/mlproxy_py/batching.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from dataclasses import dataclass
+from typing import Any, Dict, List
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class BatchRequest:
+    payload: Dict[str, Any]
+    future: asyncio.Future
+
+
+class BatchQueue:
+    def __init__(self, max_batch_size: int, max_wait_ms: int):
+        self.max_batch_size = max_batch_size
+        self.max_wait_ms = max_wait_ms
+        self._queue: asyncio.Queue[BatchRequest] = asyncio.Queue()
+
+    async def add(self, payload: Dict[str, Any]) -> Any:
+        loop = asyncio.get_running_loop()
+        fut = loop.create_future()
+        await self._queue.put(BatchRequest(payload=payload, future=fut))
+        return await fut
+
+    async def collect_batch(self) -> List[BatchRequest]:
+        batch: List[BatchRequest] = []
+        try:
+            first = await asyncio.wait_for(self._queue.get(), timeout=self.max_wait_ms / 1000.0)
+            batch.append(first)
+        except asyncio.TimeoutError:
+            return batch
+
+        start = asyncio.get_running_loop().time()
+        while len(batch) < self.max_batch_size:
+            remaining = (self.max_wait_ms / 1000.0) - (asyncio.get_running_loop().time() - start)
+            if remaining <= 0:
+                break
+            try:
+                item = await asyncio.wait_for(self._queue.get(), timeout=remaining)
+                batch.append(item)
+            except asyncio.TimeoutError:
+                break
+
+        logger.debug("Collected batch of %d items", len(batch))
+        return batch

mlproxy_py-0.1.1/mlproxy_py/cli.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import signal
+
+import typer
+import uvicorn
+
+from .app import create_app
+from .config import load_config
+from .healthcheck import close_healthcheck_client, loop as healthcheck_loop
+from .router import ModelRouter
+
+logger = logging.getLogger("mlproxy_py")
+
+app = typer.Typer(help="mlproxy-py - QoS-aware ML inference reverse proxy")
+
+_log_levels = {
+    "DEBUG": logging.DEBUG,
+    "INFO": logging.INFO,
+    "WARNING": logging.WARNING,
+    "ERROR": logging.ERROR,
+}
+
+
+@app.command()
+def run(
+    config: str = typer.Option(..., "--config", "-c", help="Path to YAML config file"),
+    log_level: str = typer.Option("INFO", "--log-level", help="Log level: DEBUG, INFO, WARNING, ERROR"),
+):
+    logging.basicConfig(level=_log_levels.get(log_level.upper(), logging.INFO), format="%(levelname)s %(name)s: %(message)s")
+
+    cfg = load_config(config)
+
+    router = ModelRouter()
+    for model, pool_cfg in cfg.models.items():
+        router.register_model(model, [b.url for b in pool_cfg.backends])
+
+    host, port = "0.0.0.0", 7000
+    if ":" in cfg.listen:
+        host, port_str = cfg.listen.split(":", 1)
+        port = int(port_str)
+
+    app_instance = create_app(
+        router=router,
+        model_cfg=cfg.models,
+        metrics_enabled=cfg.metrics.enabled,
+        metrics_path=cfg.metrics.path,
+    )
+
+    loop = asyncio.get_event_loop()
+    hc_task: asyncio.Task | None = None
+    if cfg.healthcheck.enabled:
+        hc_task = loop.create_task(
+            healthcheck_loop(
+                router=router,
+                interval_seconds=cfg.healthcheck.interval_seconds,
+                timeout_seconds=cfg.healthcheck.timeout_seconds,
+                path=cfg.healthcheck.path,
+            )
+        )
+
+    stop_event = asyncio.Event()
+
+    def _signal_handler():
+        logger.info("Shutdown signal received...")
+        stop_event.set()
+
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        try:
+            loop.add_signal_handler(sig, _signal_handler)
+        except NotImplementedError:
+            pass
+
+    config = uvicorn.Config(app_instance, host=host, port=port)
+    server = uvicorn.Server(config)
+
+    async def wait_for_shutdown():
+        await stop_event.wait()
+        if hc_task:
+            hc_task.cancel()
+            try:
+                await hc_task
+            except asyncio.CancelledError:
+                pass
+        await close_healthcheck_client()
+        server.should_exit = True
+
+    loop.create_task(wait_for_shutdown())
+
+    server.run()
+
+
+if __name__ == "__main__":
+    app()

mlproxy_py-0.1.1/mlproxy_py/config.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from typing import Dict, List
+
+import yaml
+from pydantic import BaseModel, Field
+
+
+class BackendConfig(BaseModel):
+    url: str
+
+
+class BatchingConfig(BaseModel):
+    enabled: bool = False
+    max_batch_size: int = 8
+    max_wait_ms: int = 20
+
+
+class ModelPoolConfig(BaseModel):
+    sla_ms: int = 300
+    batching: BatchingConfig = Field(default_factory=BatchingConfig)
+    backends: List[BackendConfig] = Field(default_factory=list)
+
+
+class MetricsConfig(BaseModel):
+    enabled: bool = True
+    path: str = "/metrics"
+
+
+class HealthcheckConfig(BaseModel):
+    enabled: bool = True
+    interval_seconds: int = 5
+    timeout_seconds: int = 2
+    path: str = "/health"
+
+
+class AppConfig(BaseModel):
+    listen: str = "0.0.0.0:7000"
+    models: Dict[str, ModelPoolConfig] = Field(default_factory=dict)
+    metrics: MetricsConfig = Field(default_factory=MetricsConfig)
+    healthcheck: HealthcheckConfig = Field(default_factory=HealthcheckConfig)
+
+
+def load_config(path: str) -> AppConfig:
+    with open(path, "r", encoding="utf-8") as f:
+        raw = yaml.safe_load(f) or {}
+    return AppConfig.model_validate(raw)

mlproxy_py-0.1.1/mlproxy_py/healthcheck.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+
+import httpx
+
+from .router import ModelRouter
+
+logger = logging.getLogger(__name__)
+
+_client: httpx.AsyncClient | None = None
+
+
+def _get_client() -> httpx.AsyncClient:
+    global _client
+    if _client is None:
+        _client = httpx.AsyncClient()
+    return _client
+
+
+async def close_healthcheck_client():
+    global _client
+    if _client is not None:
+        await _client.aclose()
+        _client = None
+
+
+async def _check(url: str, path: str, timeout_seconds: int, client: httpx.AsyncClient) -> bool:
+    try:
+        r = await client.get(url.rstrip("/") + path, timeout=timeout_seconds)
+        return 200 <= r.status_code < 400
+    except asyncio.CancelledError:
+        raise
+    except Exception:
+        return False
+
+
+async def loop(router: ModelRouter, interval_seconds: int, timeout_seconds: int, path: str):
+    client = _get_client()
+    while True:
+        try:
+            backends = router.all_backends()
+            checks = [_check(b.url, path, timeout_seconds, client) for b in backends]
+            results = await asyncio.gather(*checks, return_exceptions=True)
+            for backend, ok in zip(backends, results):
+                backend.healthy = bool(ok)
+            healthy_count = sum(1 for b in backends if b.healthy)
+            logger.info("Healthcheck: %d/%d backends healthy", healthy_count, len(backends))
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            logger.exception("Healthcheck loop error")
+
+        await asyncio.sleep(interval_seconds)

mlproxy_py-0.1.1/mlproxy_py/metrics.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from prometheus_client import Counter, Histogram, Gauge, generate_latest, CONTENT_TYPE_LATEST
+from starlette.responses import Response
+
+
+REQ_COUNT = Counter(
+    "mlproxy_requests_total",
+    "Total inference requests",
+    ["model", "backend", "status"],
+)
+
+REQ_LATENCY = Histogram(
+    "mlproxy_request_latency_seconds",
+    "Inference request latency seconds",
+    ["model", "backend"],
+)
+
+BACKEND_LATENCY = Gauge(
+    "mlproxy_backend_latency_ms",
+    "Observed backend latency (ms)",
+    ["model", "backend"],
+)
+
+
+def metrics_response() -> Response:
+    return Response(content=generate_latest(), media_type=CONTENT_TYPE_LATEST)

mlproxy_py-0.1.1/mlproxy_py/proxy.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from typing import Any, Dict
+
+import httpx
+
+from .backends import Backend
+
+logger = logging.getLogger(__name__)
+
+_client: httpx.AsyncClient | None = None
+
+
+def _get_client() -> httpx.AsyncClient:
+    global _client
+    if _client is None:
+        _client = httpx.AsyncClient(timeout=60)
+    return _client
+
+
+async def close_client():
+    global _client
+    if _client is not None:
+        await _client.aclose()
+        _client = None
+
+
+async def forward_json(backend: Backend, model: str, payload: Dict[str, Any]) -> Dict[str, Any]:
+    url = backend.url.rstrip("/") + f"/infer/{model}"
+    client = _get_client()
+
+    backend.active_requests += 1
+    start = time.perf_counter()
+    try:
+        r = await client.post(url, json=payload)
+        r.raise_for_status()
+        data = r.json()
+    except asyncio.CancelledError:
+        raise
+    except Exception:
+        logger.exception("Failed to forward request to %s for model %s", backend.url, model)
+        raise
+    finally:
+        backend.active_requests -= 1
+
+    backend.last_latency_ms = (time.perf_counter() - start) * 1000.0
+    return data

mlproxy_py-0.1.1/mlproxy_py/router.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+from .backends import Backend
+
+
+class ModelRouter:
+    def __init__(self):
+        self.pools: Dict[str, List[Backend]] = {}
+
+    def register_model(self, model: str, backend_urls: List[str]):
+        self.pools[model] = [Backend(url=u) for u in backend_urls]
+
+    def choose_backend(self, model: str) -> Optional[Backend]:
+        pool = self.pools.get(model) or []
+        healthy = [b for b in pool if b.healthy]
+        if not healthy:
+            return None
+        return min(healthy, key=lambda b: b.score())
+
+    def all_backends(self) -> List[Backend]:
+        out = []
+        for pool in self.pools.values():
+            out.extend(pool)
+        return out

mlproxy_py-0.1.1/pyproject.toml

@@ -0,0 +1,56 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "mlproxy-py"
+version = "0.1.1"
+description = "SLA/QoS-aware reverse proxy for ML inference workloads (batching, routing, latency metrics)."
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "MIT"}
+authors = [{name = "Kubenew"}]
+keywords = ["ml", "inference", "reverse-proxy", "qos", "batching", "asyncio", "llm"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Framework :: AsyncIO",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Internet :: WWW/HTTP :: HTTP Servers",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "fastapi>=0.111.0",
+    "uvicorn>=0.30.0",
+    "httpx>=0.27.0",
+    "pydantic>=2.7.0",
+    "pyyaml>=6.0.1",
+    "typer>=0.12.3",
+    "prometheus-client>=0.20.0",
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest>=7.0",
+    "pytest-asyncio>=0.21",
+    "httpx>=0.27.0",
+]
+dev = ["mlproxy-py[test]", "ruff>=0.1"]
+
+[project.scripts]
+mlproxy = "mlproxy_py.cli:app"
+
+[tool.hatch.build.targets.wheel]
+packages = ["mlproxy_py"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"

mlproxy_py-0.1.1/tests/test_backends.py

@@ -0,0 +1,27 @@
+from mlproxy_py.backends import Backend
+
+
+def test_score_default():
+    b = Backend(url="http://localhost:8000")
+    assert b.score() > 0
+
+
+def test_score_latency():
+    b = Backend(url="http://localhost:8000", last_latency_ms=100)
+    assert b.score() == 100
+
+
+def test_score_active_requests():
+    b = Backend(url="http://localhost:8000", last_latency_ms=100, active_requests=2)
+    assert b.score() == 100 + 2 * 5
+
+
+def test_healthy_default():
+    b = Backend(url="http://localhost:8000")
+    assert b.healthy is True
+
+
+def test_last_seen_default():
+    import time
+    b = Backend(url="http://localhost:8000")
+    assert abs(b.last_seen - time.time()) < 1

mlproxy_py-0.1.1/tests/test_batching.py

@@ -0,0 +1,45 @@
+import asyncio
+
+import pytest
+
+from mlproxy_py.batching import BatchQueue, BatchRequest
+
+
+@pytest.mark.asyncio
+async def test_add_and_collect():
+    q = BatchQueue(max_batch_size=8, max_wait_ms=5000)
+
+    async def producer():
+        return await q.add({"text": "hello"})
+
+    async def consumer():
+        batch = await q.collect_batch()
+        assert len(batch) == 1
+        batch[0].future.set_result({"reply": "ok"})
+        return batch
+
+    result, _ = await asyncio.gather(producer(), consumer())
+    assert result == {"reply": "ok"}
+
+
+@pytest.mark.asyncio
+async def test_collect_multiple():
+    q = BatchQueue(max_batch_size=3, max_wait_ms=5000)
+    loop = asyncio.get_running_loop()
+
+    for i in range(3):
+        fut = loop.create_future()
+        q._queue.put_nowait(BatchRequest(payload={"n": i}, future=fut))
+
+    batch = await q.collect_batch()
+    assert len(batch) == 3
+    for item in batch:
+        item.future.set_result({"n": item.payload["n"]})
+        assert await item.future == {"n": item.payload["n"]}
+
+
+@pytest.mark.asyncio
+async def test_collect_timeout():
+    q = BatchQueue(max_batch_size=8, max_wait_ms=50)
+    batch = await q.collect_batch()
+    assert len(batch) == 0

mlproxy_py-0.1.1/tests/test_config.py

@@ -0,0 +1,57 @@
+
+from mlproxy_py.config import load_config, BatchingConfig
+
+
+def test_load_config(tmp_path):
+    cfg_file = tmp_path / "config.yml"
+    cfg_file.write_text("""
+listen: "0.0.0.0:8000"
+models:
+  modelA:
+    sla_ms: 500
+    backends:
+      - url: "http://localhost:8001"
+metrics:
+  enabled: true
+  path: "/metrics"
+healthcheck:
+  enabled: true
+  interval_seconds: 10
+""")
+    cfg = load_config(str(cfg_file))
+    assert cfg.listen == "0.0.0.0:8000"
+    assert "modelA" in cfg.models
+    assert len(cfg.models["modelA"].backends) == 1
+    assert cfg.models["modelA"].backends[0].url == "http://localhost:8001"
+    assert cfg.models["modelA"].sla_ms == 500
+    assert cfg.healthcheck.interval_seconds == 10
+
+
+def test_load_config_defaults(tmp_path):
+    cfg_file = tmp_path / "config.yml"
+    cfg_file.write_text("models:\n  m:\n    backends:\n      - url: http://localhost:8001\n")
+    cfg = load_config(str(cfg_file))
+    assert cfg.listen == "0.0.0.0:7000"
+    assert cfg.metrics.enabled is True
+    assert cfg.healthcheck.interval_seconds == 5
+
+
+def test_load_config_empty(tmp_path):
+    cfg_file = tmp_path / "config.yml"
+    cfg_file.write_text("")
+    cfg = load_config(str(cfg_file))
+    assert cfg.listen == "0.0.0.0:7000"
+    assert len(cfg.models) == 0
+
+
+def test_load_config_missing_file():
+    import pytest
+    with pytest.raises(FileNotFoundError):
+        load_config("/nonexistent/config.yml")
+
+
+def test_batching_config_defaults():
+    b = BatchingConfig()
+    assert b.enabled is False
+    assert b.max_batch_size == 8
+    assert b.max_wait_ms == 20

mlproxy_py-0.1.1/tests/test_healthcheck.py

@@ -0,0 +1,29 @@
+import pytest
+
+from mlproxy_py.healthcheck import _check, close_healthcheck_client
+
+
+@pytest.mark.asyncio
+async def test_check_healthy(httpserver):
+    httpserver.serve_content(content="ok", code=200)
+    client = __import__("mlproxy_py.healthcheck", fromlist=["_get_client"])._get_client()
+    result = await _check(httpserver.url, "/health", 2, client)
+    assert result is True
+    await close_healthcheck_client()
+
+
+@pytest.mark.asyncio
+async def test_check_unhealthy(httpserver):
+    httpserver.serve_content(content="error", code=500)
+    client = __import__("mlproxy_py.healthcheck", fromlist=["_get_client"])._get_client()
+    result = await _check(httpserver.url, "/health", 2, client)
+    assert result is False
+    await close_healthcheck_client()
+
+
+@pytest.mark.asyncio
+async def test_check_connection_refused():
+    client = __import__("mlproxy_py.healthcheck", fromlist=["_get_client"])._get_client()
+    result = await _check("http://localhost:1", "/health", 1, client)
+    assert result is False
+    await close_healthcheck_client()

mlproxy_py-0.1.1/tests/test_router.py

@@ -0,0 +1,48 @@
+from mlproxy_py.router import ModelRouter
+
+
+def test_choose_backend():
+    r = ModelRouter()
+    r.register_model("m", ["http://a", "http://b"])
+    b = r.choose_backend("m")
+    assert b is not None
+    assert b.url in ("http://a", "http://b")
+
+
+def test_choose_backend_nonexistent_model():
+    r = ModelRouter()
+    assert r.choose_backend("nonexistent") is None
+
+
+def test_choose_backend_all_unhealthy():
+    r = ModelRouter()
+    r.register_model("m", ["http://a"])
+    for b in r.pools["m"]:
+        b.healthy = False
+    assert r.choose_backend("m") is None
+
+
+def test_choose_backend_picks_lowest_score():
+    r = ModelRouter()
+    r.register_model("m", ["http://slow", "http://fast"])
+    pool = r.pools["m"]
+    pool[0].last_latency_ms = 500
+    pool[1].last_latency_ms = 50
+    chosen = r.choose_backend("m")
+    assert chosen.url == "http://fast"
+
+
+def test_all_backends():
+    r = ModelRouter()
+    r.register_model("m1", ["http://a", "http://b"])
+    r.register_model("m2", ["http://c"])
+    all_b = r.all_backends()
+    assert len(all_b) == 3
+
+
+def test_register_overwrites_model():
+    r = ModelRouter()
+    r.register_model("m", ["http://a"])
+    r.register_model("m", ["http://b"])
+    assert len(r.pools["m"]) == 1
+    assert r.pools["m"][0].url == "http://b"