mcpfleet-obs 0.1.0__tar.gz

mcpfleet_obs-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    name: pytest (fastmcp ${{ matrix.fastmcp }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        fastmcp: ["3.2.3", "3.2.4", "3.3.0", "3.4.5"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.12"
+      - run: uv sync --group dev
+      - run: uv pip install "fastmcp==${{ matrix.fastmcp }}"
+      - run: uv run --no-sync pytest tests/ -q
+
+  test-fastmcp-4-beta:
+    name: pytest (fastmcp 4.0.0b1, allowed failure)
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.12"
+      - run: uv sync --group dev
+      - run: uv pip install --prerelease=allow "fastmcp==4.0.0b1"
+      - run: uv run --no-sync pytest tests/ -q

mcpfleet_obs-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,20 @@
+name: Release
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - run: uv build
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          skip_existing: true

mcpfleet_obs-0.1.0/.gitignore

@@ -0,0 +1,4 @@
+.venv/
+dist/
+__pycache__/
+.pytest_cache/

mcpfleet_obs-0.1.0/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: mcpfleet-obs
+Version: 0.1.0
+Summary: Shared error taxonomy, metrics, and client tracking for the mcpfleet MCP servers
+Project-URL: Homepage, https://github.com/uk-agents/mcpfleet-obs
+Project-URL: Repository, https://github.com/uk-agents/mcpfleet-obs
+Author: Paul Boucher
+License: MIT
+Requires-Python: >=3.11
+Requires-Dist: fastmcp<4,>=3.2.3
+Requires-Dist: httpx>=0.27
+Requires-Dist: prometheus-client>=0.20
+Requires-Dist: pydantic>=2.7
+Requires-Dist: starlette>=1.0.1
+Description-Content-Type: text/markdown
+
+# mcpfleet-obs
+
+Shared error taxonomy, metrics middleware, and client tracking for the mcpfleet MCP servers. This package provides standardized error handling and Prometheus instrumentation for FastMCP v3+ applications in the mcpfleet ecosystem. See the error shapes probe report at `probes/fastmcp-error-shapes/PROBE-REPORT.md` in the mcpfleet workspace for detailed FastMCP error taxonomy research.

mcpfleet_obs-0.1.0/README.md

@@ -0,0 +1,3 @@
+# mcpfleet-obs
+
+Shared error taxonomy, metrics middleware, and client tracking for the mcpfleet MCP servers. This package provides standardized error handling and Prometheus instrumentation for FastMCP v3+ applications in the mcpfleet ecosystem. See the error shapes probe report at `probes/fastmcp-error-shapes/PROBE-REPORT.md` in the mcpfleet workspace for detailed FastMCP error taxonomy research.

mcpfleet_obs-0.1.0/pyproject.toml

@@ -0,0 +1,34 @@
+[project]
+name = "mcpfleet-obs"
+version = "0.1.0"
+description = "Shared error taxonomy, metrics, and client tracking for the mcpfleet MCP servers"
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "MIT" }
+authors = [
+    { name = "Paul Boucher" },
+]
+dependencies = [
+    "fastmcp>=3.2.3,<4",
+    "prometheus-client>=0.20",
+    "pydantic>=2.7",
+    "httpx>=0.27",
+    "starlette>=1.0.1",
+]
+
+[project.urls]
+Homepage = "https://github.com/uk-agents/mcpfleet-obs"
+Repository = "https://github.com/uk-agents/mcpfleet-obs"
+
+[dependency-groups]
+dev = ["pytest>=8", "pytest-asyncio>=0.24"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/mcpfleet_obs"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"

mcpfleet_obs-0.1.0/src/mcpfleet_obs/__init__.py

@@ -0,0 +1,17 @@
+"""Fleet-wide observability for MCP servers: error taxonomy, metrics middleware, client tracking."""
+
+from .clients import ClientTrackingMiddleware
+from .errors import FleetErrorPayload, parse_error_payload, raise_http_tool_error, raise_tool_error
+from .install import install
+from .metrics import FleetMetricsMiddleware, make_metrics
+
+__all__ = [
+    "install",
+    "make_metrics",
+    "FleetMetricsMiddleware",
+    "ClientTrackingMiddleware",
+    "raise_tool_error",
+    "raise_http_tool_error",
+    "parse_error_payload",
+    "FleetErrorPayload",
+]

mcpfleet_obs-0.1.0/src/mcpfleet_obs/clients.py

@@ -0,0 +1,29 @@
+"""Client tracking middleware."""
+import logging
+
+from fastmcp.server.middleware import Middleware, MiddlewareContext
+
+from .metrics import REGION, TRANSPORT, FleetMetrics
+
+
+class ClientTrackingMiddleware(Middleware):
+    """Track MCP client connections and increment counter on initialize.
+
+    Logs client identity (name, version) and transport context on every initialize handshake.
+    """
+
+    def __init__(self, metrics: FleetMetrics, *, logger_name: str):
+        self.metrics = metrics
+        self._log = logging.getLogger(logger_name)
+
+    async def on_request(self, context: MiddlewareContext, call_next):
+        result = await call_next(context)
+        if context.method == "initialize":
+            params = context.message.params
+            info = getattr(params, "clientInfo", None)
+            client_name = getattr(info, "name", "unknown") or "unknown"
+            client_version = getattr(info, "version", "unknown") or "unknown"
+            self._log.info("client_connected client=%s version=%s transport=%s region=%s",
+                          client_name, client_version, TRANSPORT, REGION)
+            self.metrics.client_connections_total.labels(client_name, client_version, TRANSPORT, REGION).inc()
+        return result

mcpfleet_obs-0.1.0/src/mcpfleet_obs/errors.py

@@ -0,0 +1,62 @@
+"""Fleet error taxonomy. Payload contract verified against fastmcp 3.2.3-4.0.0b1
+(probes/fastmcp-error-shapes/PROBE-REPORT.md): ToolError messages survive
+mask_error_details=True and create_proxy() hops verbatim."""
+from typing import Literal, NoReturn
+
+import httpx
+from fastmcp.exceptions import ToolError
+from pydantic import BaseModel
+
+ErrorCategory = Literal[
+    "transient", "validation", "not_found", "auth_required", "configuration", "unknown"
+]
+
+
+class FleetErrorPayload(BaseModel):
+    error_category: ErrorCategory
+    is_retryable: bool
+    attempted: str
+    description: str
+
+
+def raise_tool_error(category: ErrorCategory, *, is_retryable: bool,
+                     attempted: str, description: str) -> NoReturn:
+    raise ToolError(FleetErrorPayload(
+        error_category=category, is_retryable=is_retryable,
+        attempted=attempted, description=description,
+    ).model_dump_json())
+
+
+def raise_http_tool_error(exc: Exception, *, attempted: str) -> NoReturn:
+    if isinstance(exc, httpx.HTTPStatusError):
+        status = exc.response.status_code
+        url = exc.request.url
+        if status == 404:
+            raise_tool_error("not_found", is_retryable=False, attempted=attempted,
+                             description=f"Resource not found (404). Check the identifier. URL: {url}")
+        if status in (401, 403):
+            raise_tool_error("auth_required", is_retryable=False, attempted=attempted,
+                             description=f"Access denied ({status}). URL: {url}")
+        if status in (429, 503):
+            raise_tool_error("transient", is_retryable=True, attempted=attempted,
+                             description=f"Upstream returned {status} — retry after a short delay. URL: {url}")
+        if 400 <= status < 500:
+            raise_tool_error("validation", is_retryable=False, attempted=attempted,
+                             description=f"Upstream rejected the request ({status}) — fix the input and retry. URL: {url}")
+        raise_tool_error("unknown", is_retryable=False, attempted=attempted,
+                         description=f"Upstream returned {status}. URL: {url}")
+    if isinstance(exc, httpx.TimeoutException):
+        raise_tool_error("transient", is_retryable=True, attempted=attempted,
+                         description="Request timed out — upstream may be slow, retry is safe.")
+    if isinstance(exc, httpx.ConnectError):
+        raise_tool_error("transient", is_retryable=True, attempted=attempted,
+                         description="Could not connect to upstream API — network error, retry is safe.")
+    raise_tool_error("unknown", is_retryable=False, attempted=attempted,
+                     description=f"Unexpected error: {type(exc).__name__}: {exc}")
+
+
+def parse_error_payload(message: str) -> FleetErrorPayload | None:
+    try:
+        return FleetErrorPayload.model_validate_json(message)
+    except Exception:
+        return None

mcpfleet_obs-0.1.0/src/mcpfleet_obs/install.py

@@ -0,0 +1,32 @@
+"""One-call installation of observability middleware stack."""
+from fastmcp import FastMCP
+from prometheus_client import CONTENT_TYPE_LATEST, REGISTRY, CollectorRegistry, generate_latest
+from starlette.requests import Request
+from starlette.responses import Response
+
+from .clients import ClientTrackingMiddleware
+from .metrics import FleetMetrics, FleetMetricsMiddleware, make_metrics
+
+
+def install(mcp: FastMCP, *, prefix: str,
+            registry: CollectorRegistry | None = None) -> FleetMetrics:
+    """Install the fleet observability stack: client tracking, metrics, and /metrics endpoint.
+
+    Args:
+        mcp: FastMCP instance to instrument
+        prefix: Prometheus metric name prefix (e.g. "uk_legal")
+        registry: Optional Prometheus registry (uses default if None)
+
+    Returns:
+        FleetMetrics instance (for further configuration if needed)
+    """
+    reg = registry if registry is not None else REGISTRY
+    metrics = make_metrics(prefix, registry=reg)
+    mcp.add_middleware(ClientTrackingMiddleware(metrics, logger_name=f"fastmcp.{prefix}.clients"))
+    mcp.add_middleware(FleetMetricsMiddleware(metrics))
+
+    @mcp.custom_route("/metrics", methods=["GET"])
+    async def metrics_endpoint(request: Request) -> Response:
+        return Response(generate_latest(reg), media_type=CONTENT_TYPE_LATEST)
+
+    return metrics

mcpfleet_obs-0.1.0/src/mcpfleet_obs/metrics.py

@@ -0,0 +1,82 @@
+"""Fleet-standard Prometheus metrics.
+
+Dual-path error counting is REQUIRED, not defensive: on fastmcp 3.4.x a
+proxied tool failure surfaces as a result with is_error=True and no
+exception (probes/fastmcp-error-shapes/PROBE-REPORT.md, P3). Exception-only
+middleware silently counts those as ok.
+"""
+import os
+import time
+from dataclasses import dataclass
+
+from fastmcp.exceptions import NotFoundError, ToolError
+from fastmcp.server.middleware import Middleware, MiddlewareContext
+from prometheus_client import REGISTRY, CollectorRegistry, Counter, Histogram
+
+from .errors import parse_error_payload
+
+BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0)
+TRANSPORT = os.getenv("FASTMCP_TRANSPORT", "http")
+REGION = os.getenv("FLY_REGION", "local")
+
+
+@dataclass
+class FleetMetrics:
+    tool_calls_total: Counter
+    tool_duration_seconds: Histogram
+    client_connections_total: Counter
+
+
+def make_metrics(prefix: str, *, registry: CollectorRegistry | None = None) -> FleetMetrics:
+    registry = registry if registry is not None else REGISTRY
+    return FleetMetrics(
+        tool_calls_total=Counter(
+            f"{prefix}_tool_calls_total", "Count of MCP tool invocations.",
+            labelnames=["tool", "transport", "region", "status", "error_category"],
+            registry=registry),
+        tool_duration_seconds=Histogram(
+            f"{prefix}_tool_duration_seconds", "Tool invocation latency in seconds.",
+            labelnames=["tool", "transport", "region"], buckets=BUCKETS,
+            registry=registry),
+        client_connections_total=Counter(
+            f"{prefix}_client_connections_total", "Count of MCP client initialize handshakes.",
+            labelnames=["client_name", "client_version", "transport", "region"],
+            registry=registry),
+    )
+
+
+class FleetMetricsMiddleware(Middleware):
+    def __init__(self, metrics: FleetMetrics):
+        self.metrics = metrics
+
+    async def on_call_tool(self, context: MiddlewareContext, call_next):
+        tool = context.message.name
+        t0 = time.perf_counter()
+        try:
+            result = await call_next(context)
+        except NotFoundError:
+            # Scanners probing bogus tool names must not mint metric series.
+            self.metrics.tool_calls_total.labels(
+                "_unknown_", TRANSPORT, REGION, "error", "not_found").inc()
+            raise
+        except ToolError as exc:
+            payload = parse_error_payload(str(exc))
+            category = payload.error_category if payload else "unknown"
+            self.metrics.tool_calls_total.labels(tool, TRANSPORT, REGION, "error", category).inc()
+            raise
+        except BaseException:
+            self.metrics.tool_calls_total.labels(tool, TRANSPORT, REGION, "error", "unknown").inc()
+            raise
+        finally:
+            self.metrics.tool_duration_seconds.labels(tool, TRANSPORT, REGION).observe(
+                time.perf_counter() - t0)
+        if getattr(result, "is_error", False):
+            payload = None
+            content = getattr(result, "content", None)
+            if content and getattr(content[0], "text", None):
+                payload = parse_error_payload(content[0].text)
+            category = payload.error_category if payload else "unknown"
+            self.metrics.tool_calls_total.labels(tool, TRANSPORT, REGION, "error", category).inc()
+        else:
+            self.metrics.tool_calls_total.labels(tool, TRANSPORT, REGION, "ok", "").inc()
+        return result

mcpfleet_obs-0.1.0/tests/fixtures/error-shapes-3.2.3.json

@@ -0,0 +1,214 @@
+{
+  "fastmcp_version": "3.2.3",
+  "python": "3.12.11",
+  "cases": [
+    {
+      "tool": "ok_tool",
+      "mask_error_details": true,
+      "wire_result": {
+        "meta": {
+          "fastmcp": {
+            "wrap_result": true
+          }
+        },
+        "content": [
+          {
+            "type": "text",
+            "text": "fine",
+            "annotations": null,
+            "meta": null
+          }
+        ],
+        "structuredContent": {
+          "result": "fine"
+        },
+        "isError": false
+      },
+      "middleware_observation": {
+        "tool": "ok_tool",
+        "middleware_saw": "result",
+        "result_is_error": "<no attr>",
+        "result_type": "ToolResult"
+      }
+    },
+    {
+      "tool": "empty_tool",
+      "mask_error_details": true,
+      "wire_result": {
+        "meta": {
+          "fastmcp": {
+            "wrap_result": true
+          }
+        },
+        "content": [
+          {
+            "type": "text",
+            "text": "{\"status\": \"empty\", \"result_count\": 0, \"detail\": \"Query executed successfully, no matches.\"}",
+            "annotations": null,
+            "meta": null
+          }
+        ],
+        "structuredContent": {
+          "result": "{\"status\": \"empty\", \"result_count\": 0, \"detail\": \"Query executed successfully, no matches.\"}"
+        },
+        "isError": false
+      },
+      "middleware_observation": {
+        "tool": "empty_tool",
+        "middleware_saw": "result",
+        "result_is_error": "<no attr>",
+        "result_type": "ToolResult"
+      }
+    },
+    {
+      "tool": "structured_tool_error",
+      "mask_error_details": true,
+      "wire_result": {
+        "meta": null,
+        "content": [
+          {
+            "type": "text",
+            "text": "{\"error_category\": \"transient\", \"is_retryable\": true, \"attempted\": \"structured_tool_error()\", \"description\": \"Upstream returned 503 \\u2014 retry after a short delay.\"}",
+            "annotations": null,
+            "meta": null
+          }
+        ],
+        "structuredContent": null,
+        "isError": true
+      },
+      "middleware_observation": {
+        "tool": "structured_tool_error",
+        "middleware_saw": "exception",
+        "exc_type": "ToolError",
+        "exc_str": "{\"error_category\": \"transient\", \"is_retryable\": true, \"attempted\": \"structured_tool_error()\", \"description\": \"Upstream returned 503 \\u2014 retry after a short delay.\"}"
+      }
+    },
+    {
+      "tool": "plain_exception",
+      "mask_error_details": true,
+      "wire_result": {
+        "meta": null,
+        "content": [
+          {
+            "type": "text",
+            "text": "Error calling tool 'plain_exception'",
+            "annotations": null,
+            "meta": null
+          }
+        ],
+        "structuredContent": null,
+        "isError": true
+      },
+      "middleware_observation": {
+        "tool": "plain_exception",
+        "middleware_saw": "exception",
+        "exc_type": "ToolError",
+        "exc_str": "Error calling tool 'plain_exception'"
+      }
+    },
+    {
+      "tool": "ok_tool",
+      "mask_error_details": false,
+      "wire_result": {
+        "meta": {
+          "fastmcp": {
+            "wrap_result": true
+          }
+        },
+        "content": [
+          {
+            "type": "text",
+            "text": "fine",
+            "annotations": null,
+            "meta": null
+          }
+        ],
+        "structuredContent": {
+          "result": "fine"
+        },
+        "isError": false
+      },
+      "middleware_observation": {
+        "tool": "ok_tool",
+        "middleware_saw": "result",
+        "result_is_error": "<no attr>",
+        "result_type": "ToolResult"
+      }
+    },
+    {
+      "tool": "empty_tool",
+      "mask_error_details": false,
+      "wire_result": {
+        "meta": {
+          "fastmcp": {
+            "wrap_result": true
+          }
+        },
+        "content": [
+          {
+            "type": "text",
+            "text": "{\"status\": \"empty\", \"result_count\": 0, \"detail\": \"Query executed successfully, no matches.\"}",
+            "annotations": null,
+            "meta": null
+          }
+        ],
+        "structuredContent": {
+          "result": "{\"status\": \"empty\", \"result_count\": 0, \"detail\": \"Query executed successfully, no matches.\"}"
+        },
+        "isError": false
+      },
+      "middleware_observation": {
+        "tool": "empty_tool",
+        "middleware_saw": "result",
+        "result_is_error": "<no attr>",
+        "result_type": "ToolResult"
+      }
+    },
+    {
+      "tool": "structured_tool_error",
+      "mask_error_details": false,
+      "wire_result": {
+        "meta": null,
+        "content": [
+          {
+            "type": "text",
+            "text": "{\"error_category\": \"transient\", \"is_retryable\": true, \"attempted\": \"structured_tool_error()\", \"description\": \"Upstream returned 503 \\u2014 retry after a short delay.\"}",
+            "annotations": null,
+            "meta": null
+          }
+        ],
+        "structuredContent": null,
+        "isError": true
+      },
+      "middleware_observation": {
+        "tool": "structured_tool_error",
+        "middleware_saw": "exception",
+        "exc_type": "ToolError",
+        "exc_str": "{\"error_category\": \"transient\", \"is_retryable\": true, \"attempted\": \"structured_tool_error()\", \"description\": \"Upstream returned 503 \\u2014 retry after a short delay.\"}"
+      }
+    },
+    {
+      "tool": "plain_exception",
+      "mask_error_details": false,
+      "wire_result": {
+        "meta": null,
+        "content": [
+          {
+            "type": "text",
+            "text": "Error calling tool 'plain_exception': INTERNAL-SECRET-DETAIL db password wrong",
+            "annotations": null,
+            "meta": null
+          }
+        ],
+        "structuredContent": null,
+        "isError": true
+      },
+      "middleware_observation": {
+        "tool": "plain_exception",
+        "middleware_saw": "exception",
+        "exc_type": "ToolError",
+        "exc_str": "Error calling tool 'plain_exception': INTERNAL-SECRET-DETAIL db password wrong"
+      }
+    }
+  ]
+}