tb-router-embed 1.0.1__tar.gz

tb_router_embed-1.0.1/.env.example

@@ -0,0 +1,12 @@
+# Comma-separated vLLM backend URLs (host:port or full URL)
+# Examples: localhost:8001 (same machine) or 192.168.86.173:8001,192.168.86.176:8001
+EMBEDDING_BACKENDS=192.168.86.173:8001,192.168.86.176:8001
+
+# Routing strategy: failover (default) or round_robin
+ROUTER_STRATEGY=failover
+
+# Port for the router (do not use 8001; reserved for vLLM backends)
+ROUTER_PORT=8011
+
+# Max concurrent requests to backends; excess requests wait in queue
+ROUTER_MAX_CONCURRENT=20

tb_router_embed-1.0.1/.github/workflows/build.yml

@@ -0,0 +1,27 @@
+name: Build
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install
+        run: pip install -e ".[dev]"
+
+      - name: Test
+        run: pytest tests/ -v
+
+      - name: Build package
+        run: pip install build && python -m build

tb_router_embed-1.0.1/.github/workflows/publish.yml

@@ -0,0 +1,30 @@
+name: Publish to PyPI
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build twine
+
+      - name: Build distribution
+        run: python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

tb_router_embed-1.0.1/.gitignore

@@ -0,0 +1,23 @@
+# Build
+build/
+dist/
+*.egg-info/
+*.egg
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+.pytest_cache/
+.venv/
+venv/
+env/
+
+# Env (keep .env.example)
+.env
+.env.local
+
+# IDE
+.idea/
+.vscode/
+*.swp

tb_router_embed-1.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Layers
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tb_router_embed-1.0.1/PKG-INFO

@@ -0,0 +1,115 @@
+Metadata-Version: 2.4
+Name: tb-router-embed
+Version: 1.0.1
+Summary: HTTP proxy router for vLLM embedding API with round-robin and failover
+License-Expression: MIT
+License-File: LICENSE
+Keywords: embedding,load-balancer,rag,router,vllm
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.11
+Requires-Dist: fastapi>=0.100.0
+Requires-Dist: httpx>=0.20.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: uvicorn[standard]>=0.20.0
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# router-embed
+
+HTTP proxy router for vLLM embedding API. Routes `/v1/embeddings` and `/v1/models` requests to multiple vLLM backends with configurable round-robin or failover strategies.
+
+## Setup
+
+```bash
+python3.11 -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+pip install -e .
+```
+
+## Run
+
+```bash
+cp .env.example .env   # edit EMBEDDING_BACKENDS for your vLLM servers
+source venv/bin/activate
+python -m router_embed.main
+```
+
+Or `router-embed` (after activating venv). If `router-embed` isn't found, use `./venv/bin/python -m router_embed.main` with no activation.
+
+Configure clients (ingest/retrieve layers): set `EMBEDDING_URL=http://<router-host>:8011` instead of pointing directly at a vLLM server.
+
+## Deploy (simple)
+
+```bash
+# From the project directory
+source venv/bin/activate
+cp .env.example .env   # edit EMBEDDING_BACKENDS
+nohup python -m router_embed.main &
+```
+
+Or run in foreground (logs to terminal):
+```bash
+source venv/bin/activate && python -m router_embed.main
+```
+
+## Configuration
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `EMBEDDING_BACKENDS` | `192.168.86.173:8001,192.168.86.176:8001` | Comma-separated backend URLs |
+| `ROUTER_STRATEGY` | `failover` | `failover` or `round_robin` |
+| `ROUTER_PORT` | `8011` | Port for the router (do not use 8001) |
+| `ROUTER_MAX_CONCURRENT` | `20` | Max concurrent requests; excess wait in queue |
+
+## API
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `/v1/embeddings` | POST | Forward to backend (OpenAI-compatible) |
+| `/v1/models` | GET | Forward to backend |
+| `/health` | GET | Health check |
+
+Examples:
+```bash
+# Embeddings
+curl -X POST http://localhost:8011/v1/embeddings \
+  -H "Content-Type: application/json" \
+  -d '{"model": "BAAI/bge-m3", "input": "hello world"}'
+
+# Models
+curl http://localhost:8011/v1/models
+
+# Health
+curl http://localhost:8011/health
+```
+
+## Troubleshooting
+
+**"address already in use" (port 8011)** — Another process is using the port. Stop it: `lsof -ti:8011 | xargs kill` (macOS/Linux). Or set `ROUTER_PORT=8012` in `.env` to use a different port.
+
+**"All backends unavailable: All connection attempts failed"** — The router could not reach any vLLM backend. Ensure:
+1. vLLM is running on your backend hosts (default: 192.168.86.173:8001, 192.168.86.176:8001)
+2. Your machine can reach those IPs (same network, no firewall blocking)
+3. Or set `EMBEDDING_BACKENDS` in `.env` to your vLLM URLs, e.g. `localhost:8001` if vLLM runs on the same machine
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest tests/ -v
+# or: make test
+```
+
+## Publish to PyPI
+
+One-shot via GitHub Actions (see [pypi-hello-world](https://github.com/taixingbi/pypi-hello-world)):
+
+1. Add `PYPI_API_TOKEN` as a repo secret (Settings → Secrets → Actions). Get token at [pypi.org/manage/account/token/](https://pypi.org/manage/account/token/).
+2. Push to `main` or run **Actions → Publish to PyPI → Run workflow** manually.
+3. The workflow builds and uploads to PyPI.

tb_router_embed-1.0.1/README.md

@@ -0,0 +1,94 @@
+# router-embed
+
+HTTP proxy router for vLLM embedding API. Routes `/v1/embeddings` and `/v1/models` requests to multiple vLLM backends with configurable round-robin or failover strategies.
+
+## Setup
+
+```bash
+python3.11 -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+pip install -e .
+```
+
+## Run
+
+```bash
+cp .env.example .env   # edit EMBEDDING_BACKENDS for your vLLM servers
+source venv/bin/activate
+python -m router_embed.main
+```
+
+Or `router-embed` (after activating venv). If `router-embed` isn't found, use `./venv/bin/python -m router_embed.main` with no activation.
+
+Configure clients (ingest/retrieve layers): set `EMBEDDING_URL=http://<router-host>:8011` instead of pointing directly at a vLLM server.
+
+## Deploy (simple)
+
+```bash
+# From the project directory
+source venv/bin/activate
+cp .env.example .env   # edit EMBEDDING_BACKENDS
+nohup python -m router_embed.main &
+```
+
+Or run in foreground (logs to terminal):
+```bash
+source venv/bin/activate && python -m router_embed.main
+```
+
+## Configuration
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `EMBEDDING_BACKENDS` | `192.168.86.173:8001,192.168.86.176:8001` | Comma-separated backend URLs |
+| `ROUTER_STRATEGY` | `failover` | `failover` or `round_robin` |
+| `ROUTER_PORT` | `8011` | Port for the router (do not use 8001) |
+| `ROUTER_MAX_CONCURRENT` | `20` | Max concurrent requests; excess wait in queue |
+
+## API
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `/v1/embeddings` | POST | Forward to backend (OpenAI-compatible) |
+| `/v1/models` | GET | Forward to backend |
+| `/health` | GET | Health check |
+
+Examples:
+```bash
+# Embeddings
+curl -X POST http://localhost:8011/v1/embeddings \
+  -H "Content-Type: application/json" \
+  -d '{"model": "BAAI/bge-m3", "input": "hello world"}'
+
+# Models
+curl http://localhost:8011/v1/models
+
+# Health
+curl http://localhost:8011/health
+```
+
+## Troubleshooting
+
+**"address already in use" (port 8011)** — Another process is using the port. Stop it: `lsof -ti:8011 | xargs kill` (macOS/Linux). Or set `ROUTER_PORT=8012` in `.env` to use a different port.
+
+**"All backends unavailable: All connection attempts failed"** — The router could not reach any vLLM backend. Ensure:
+1. vLLM is running on your backend hosts (default: 192.168.86.173:8001, 192.168.86.176:8001)
+2. Your machine can reach those IPs (same network, no firewall blocking)
+3. Or set `EMBEDDING_BACKENDS` in `.env` to your vLLM URLs, e.g. `localhost:8001` if vLLM runs on the same machine
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest tests/ -v
+# or: make test
+```
+
+## Publish to PyPI
+
+One-shot via GitHub Actions (see [pypi-hello-world](https://github.com/taixingbi/pypi-hello-world)):
+
+1. Add `PYPI_API_TOKEN` as a repo secret (Settings → Secrets → Actions). Get token at [pypi.org/manage/account/token/](https://pypi.org/manage/account/token/).
+2. Push to `main` or run **Actions → Publish to PyPI → Run workflow** manually.
+3. The workflow builds and uploads to PyPI.

tb_router_embed-1.0.1/pyproject.toml

@@ -0,0 +1,36 @@
+[build-system]
+requires = ["hatchling>=1.0"]
+build-backend = "hatchling.build"
+
+[project]
+name = "tb-router-embed"
+version = "1.0.1"
+description = "HTTP proxy router for vLLM embedding API with round-robin and failover"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.11"
+keywords = ["rag", "embedding", "vllm", "router", "load-balancer"]
+classifiers = [
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+dependencies = [
+    "httpx>=0.20.0",
+    "uvicorn[standard]>=0.20.0",
+    "fastapi>=0.100.0",
+    "python-dotenv>=1.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "pytest-asyncio>=0.21.0",
+]
+
+[project.scripts]
+router-embed = "router_embed.main:run"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/router_embed"]

tb_router_embed-1.0.1/requirements-publish.txt

@@ -0,0 +1,3 @@
+# For PyPI publish: pip install -r requirements-publish.txt
+build>=1.0.0
+twine>=4.0.0

tb_router_embed-1.0.1/requirements.txt

@@ -0,0 +1,5 @@
+# Runtime dependencies (sync with pyproject.toml)
+httpx>=0.20.0
+uvicorn[standard]>=0.20.0
+fastapi>=0.100.0
+python-dotenv>=1.0.0

tb_router_embed-1.0.1/src/router_embed/__init__.py

@@ -0,0 +1,3 @@
+"""vLLM embedding router: HTTP proxy with round-robin and failover."""
+
+__version__ = "1.0.0"

tb_router_embed-1.0.1/src/router_embed/config.py

@@ -0,0 +1,71 @@
+"""
+Configuration for vLLM embedding router.
+Loads .env if present; values can be overridden by environment variables or configure().
+"""
+import os
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+_overrides: dict = {}
+
+DEFAULT_BACKENDS = "192.168.86.173:8001,192.168.86.176:8001"
+DEFAULT_STRATEGY = "failover"
+DEFAULT_PORT = 8011
+DEFAULT_MAX_CONCURRENT = 20
+
+
+def configure(
+    backends: str | None = None,
+    strategy: str | None = None,
+    port: int | None = None,
+    max_concurrent: int | None = None,
+    **kwargs,
+) -> None:
+    """Set configuration overrides (used before starting the server)."""
+    global _overrides
+    if backends is not None:
+        _overrides["backends"] = backends
+    if strategy is not None:
+        _overrides["strategy"] = strategy
+    if port is not None:
+        _overrides["port"] = port
+    if max_concurrent is not None:
+        _overrides["max_concurrent"] = max_concurrent
+    for k, v in kwargs.items():
+        if v is not None:
+            _overrides[k] = v
+
+
+def get_backends() -> list[str]:
+    """Backend URLs: override > env > default. Returns list of base URLs."""
+    raw = _overrides.get("backends") or os.getenv("EMBEDDING_BACKENDS", DEFAULT_BACKENDS)
+    urls = []
+    for b in raw.split(","):
+        b = b.strip()
+        if not b:
+            continue
+        if not b.startswith(("http://", "https://")):
+            b = f"http://{b}"
+        urls.append(b.rstrip("/"))
+    return urls
+
+
+def get_strategy() -> str:
+    """Routing strategy: override > env > default. One of round_robin, failover."""
+    return _overrides.get("strategy") or os.getenv("ROUTER_STRATEGY", DEFAULT_STRATEGY)
+
+
+def get_port() -> int:
+    """Router port: override > env > default. Do not use 8001 (reserved for vLLM backends)."""
+    raw = _overrides.get("port") or os.getenv("ROUTER_PORT", str(DEFAULT_PORT))
+    return int(raw)
+
+
+def get_max_concurrent() -> int:
+    """Max concurrent requests to backends; excess wait in queue."""
+    raw = _overrides.get("max_concurrent") or os.getenv(
+        "ROUTER_MAX_CONCURRENT", str(DEFAULT_MAX_CONCURRENT)
+    )
+    return int(raw)

tb_router_embed-1.0.1/src/router_embed/main.py

@@ -0,0 +1,91 @@
+"""FastAPI app: proxy /v1/embeddings and /v1/models to vLLM backends."""
+import sys
+from pathlib import Path
+
+if __name__ == "__main__":
+    _src = Path(__file__).resolve().parent.parent
+    if str(_src) not in sys.path:
+        sys.path.insert(0, str(_src))
+
+import asyncio
+from contextlib import asynccontextmanager
+
+import httpx
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse, Response
+
+from router_embed.config import get_backends, get_max_concurrent, get_port
+from router_embed.router import proxy_request
+
+_http_client: httpx.AsyncClient | None = None
+_queue: asyncio.Semaphore | None = None
+
+_SKIP_HEADERS = frozenset({"host", "content-length"})
+
+
+def _err(code: int, msg: str) -> JSONResponse:
+    return JSONResponse(status_code=code, content={"error": msg})
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    global _http_client, _queue
+    _http_client = httpx.AsyncClient(timeout=60.0)
+    _queue = asyncio.Semaphore(get_max_concurrent())
+    yield
+    await _http_client.aclose()
+    _http_client = None
+    _queue = None
+
+
+app = FastAPI(title="vLLM Embedding Router", lifespan=lifespan)
+
+
+@app.get("/health")
+async def health():
+    """Health check: router is up."""
+    return {"status": "ok"}
+
+
+@app.api_route("/v1/embeddings", methods=["POST"])
+@app.api_route("/v1/models", methods=["GET"])
+async def proxy(request: Request):
+    """Forward /v1/embeddings and /v1/models to backend(s)."""
+    if _http_client is None or _queue is None:
+        return _err(503, "Router not initialized")
+
+    backends = get_backends()
+    if not backends:
+        return _err(503, "No backends configured. Set EMBEDDING_BACKENDS.")
+
+    headers = {k: v for k, v in request.headers.items() if k.lower() not in _SKIP_HEADERS}
+
+    try:
+        async with _queue:
+            resp, _ = await proxy_request(
+                _http_client,
+                request.method,
+                request.url.path,
+                content=await request.body(),
+                headers=headers,
+            )
+        return Response(content=resp.content, status_code=resp.status_code, headers=dict(resp.headers))
+    except (httpx.TimeoutException, httpx.ConnectError) as e:
+        return _err(503, f"All backends unavailable: {e!s}")
+
+
+def run():
+    """CLI entry point: run uvicorn server."""
+    import uvicorn
+
+    port = get_port()
+    uvicorn.run(
+        "router_embed.main:app",
+        host="0.0.0.0",
+        port=port,
+        reload=False,
+    )
+
+
+if __name__ == "__main__":
+    run()

tb_router_embed-1.0.1/src/router_embed/router.py

@@ -0,0 +1,81 @@
+"""Backend selection and proxy logic for vLLM embedding router."""
+import itertools
+
+import httpx
+
+from router_embed.config import get_backends, get_strategy
+
+REQUEST_TIMEOUT = 60.0
+_ROUND_ROBIN_INDEX: itertools.cycle | None = None
+
+
+def _get_round_robin_index() -> itertools.cycle:
+    global _ROUND_ROBIN_INDEX
+    if _ROUND_ROBIN_INDEX is None:
+        backends = get_backends()
+        if not backends:
+            raise ValueError("No backends configured. Set EMBEDDING_BACKENDS.")
+        _ROUND_ROBIN_INDEX = itertools.cycle(range(len(backends)))
+    return _ROUND_ROBIN_INDEX
+
+
+def _select_backend() -> str:
+    """Select backend URL based on strategy."""
+    backends = get_backends()
+    if not backends:
+        raise ValueError("No backends configured. Set EMBEDDING_BACKENDS.")
+
+    strategy = get_strategy()
+    if strategy == "round_robin":
+        idx = next(_get_round_robin_index())
+        return backends[idx]
+    if strategy == "failover":
+        return backends[0]
+    raise ValueError(f"Unknown ROUTER_STRATEGY: {strategy}. Use round_robin or failover.")
+
+
+def _is_retryable(status_code: int) -> bool:
+    return status_code == 408 or status_code >= 500
+
+
+async def proxy_request(
+    client: httpx.AsyncClient,
+    method: str,
+    path: str,
+    content: bytes | None = None,
+    headers: dict | None = None,
+) -> tuple[httpx.Response, str]:
+    """
+    Forward request to backend(s). Returns (response, backend_url_used).
+    On failover: tries next backend on 5xx or timeout.
+    Raises httpx.HTTPStatusError if all backends fail.
+    """
+    backends = get_backends()
+    if not backends:
+        raise ValueError("No backends configured. Set EMBEDDING_BACKENDS.")
+
+    strategy = get_strategy()
+    candidates = backends if strategy == "failover" else [_select_backend()]
+
+    last_error: Exception | None = None
+    last_response: httpx.Response | None = None
+
+    for base_url in candidates:
+        url = f"{base_url}{path}"
+        try:
+            resp = await client.request(
+                method, url, content=content, headers=headers, timeout=REQUEST_TIMEOUT
+            )
+            if strategy == "failover" and _is_retryable(resp.status_code):
+                last_response = resp
+                continue
+            return resp, base_url
+        except (httpx.TimeoutException, httpx.ConnectError) as e:
+            last_error = e
+            continue
+
+    if last_response is not None:
+        return last_response, candidates[-1]
+    if last_error is not None:
+        raise last_error
+    raise RuntimeError("No backends available")

tb_router_embed-1.0.1/tests/test_router.py

@@ -0,0 +1,44 @@
+"""Tests for router_embed."""
+import pytest
+
+from router_embed.config import configure, get_backends, get_max_concurrent, get_port, get_strategy
+
+
+@pytest.fixture(autouse=True)
+def reset_config():
+    import router_embed.config as cfg
+    cfg._overrides.clear()
+    yield
+
+
+def test_get_backends_default():
+    backends = get_backends()
+    assert len(backends) == 2
+    assert "192.168.86.173" in backends[0]
+    assert "192.168.86.176" in backends[1]
+
+
+def test_get_backends_override():
+    configure(backends="http://a:8001,http://b:8002")
+    assert get_backends() == ["http://a:8001", "http://b:8002"]
+
+
+def test_get_strategy_default():
+    assert get_strategy() == "failover"
+
+
+def test_get_port_default():
+    assert get_port() == 8011
+
+
+def test_get_max_concurrent_default():
+    assert get_max_concurrent() == 20
+
+
+def test_health_endpoint():
+    from fastapi.testclient import TestClient
+    from router_embed.main import app
+    client = TestClient(app)
+    r = client.get("/health")
+    assert r.status_code == 200
+    assert r.json() == {"status": "ok"}