toolrate 0.3.2__tar.gz

toolrate-0.3.2/.gitignore

@@ -0,0 +1,36 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.venv/
+venv/
+
+# Environment
+.env
+
+# Secrets — never commit
+api_key.py
+*.token
+npm-token.txt
+npmj-codes.txt
+*-codes.txt
+.pypirc
+
+# IDE
+.vscode/
+.idea/
+
+# Docker
+docker-compose.override.yml
+
+# Test databases
+*.db
+
+# Build artifacts
+sdks/python/dist/
+
+# OS
+.DS_Store
+nemo

toolrate-0.3.2/PKG-INFO

@@ -0,0 +1,149 @@
+Metadata-Version: 2.4
+Name: toolrate
+Version: 0.3.2
+Summary: Reliability oracle for AI agents — pick the right tool from the start
+Project-URL: Homepage, https://toolrate.ai
+Project-URL: Documentation, https://api.toolrate.ai/docs
+Project-URL: Repository, https://github.com/netvistamedia/toolrate
+Author-email: ToolRate <bleep@toolrate.ai>
+License-Expression: MIT
+Keywords: agents,ai,api,llm,reliability,tools
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.9
+Requires-Dist: httpx>=0.24.0
+Description-Content-Type: text/markdown
+
+# ToolRate Python SDK
+
+Python client for the [ToolRate API](https://api.toolrate.ai) — the reliability oracle for AI agents.
+
+## Installation
+
+```bash
+pip install toolrate
+```
+
+## Quick start — one line of code
+
+The `guard` function wraps any tool call with automatic reliability checking:
+
+```python
+from toolrate import ToolRate, guard
+
+client = ToolRate(api_key="nf_live_...")
+
+# Wrap any tool call — assesses before, reports after, automatically
+result = guard(client, "https://api.openai.com/v1/chat/completions",
+               lambda: openai.chat.completions.create(model="gpt-4", messages=[...]))
+```
+
+That's it. ToolRate will:
+1. Check the tool's reliability score before calling
+2. Execute the tool call
+3. Report success/failure back (building the data moat)
+4. Classify errors automatically
+
+## Auto-fallback
+
+When a tool fails, automatically try alternatives:
+
+```python
+result = guard(
+    client,
+    "https://api.openai.com/v1/chat/completions",
+    lambda: openai.chat.completions.create(model="gpt-4", messages=msgs),
+    fallbacks=[
+        ("https://api.anthropic.com/v1/messages",
+         lambda: anthropic.messages.create(model="claude-sonnet-4-20250514", messages=msgs)),
+        ("https://api.groq.com/openai/v1/chat/completions",
+         lambda: groq.chat.completions.create(model="llama-3.3-70b", messages=msgs)),
+    ],
+    min_score=50,  # Skip tools scoring below 50
+)
+```
+
+## Decorator
+
+```python
+from toolrate import ToolRate, toolrate_guard
+
+client = ToolRate(api_key="nf_live_...")
+
+@toolrate_guard(client, "https://api.stripe.com/v1/charges")
+def charge_customer(amount, currency):
+    return stripe.Charge.create(amount=amount, currency=currency)
+
+# Every call is now automatically assessed + reported
+charge_customer(1000, "usd")
+```
+
+## Journey tracking
+
+Report fallback patterns to power hidden gem discovery:
+
+```python
+# First attempt fails
+client.report("https://api.sendgrid.com/v3/mail/send",
+    success=False, error_category="rate_limit",
+    session_id="session-123", attempt_number=1)
+
+# Fallback succeeds
+client.report("https://api.resend.com/emails",
+    success=True, latency_ms=180,
+    session_id="session-123", attempt_number=2,
+    previous_tool="https://api.sendgrid.com/v3/mail/send")
+```
+
+## Discovery
+
+Find hidden gems and fallback chains based on real agent behavior:
+
+```python
+# Tools that shine as fallbacks
+gems = client.discover_hidden_gems(category="email")
+
+# What to try when SendGrid fails
+chain = client.discover_fallback_chain("https://api.sendgrid.com/v3/mail/send")
+```
+
+## Direct API usage
+
+```python
+from toolrate import ToolRate
+
+client = ToolRate(api_key="nf_live_...")
+
+# Assess
+result = client.assess("https://api.openai.com/v1/chat/completions",
+                        context="customer support chatbot")
+print(result["reliability_score"])      # 89.0
+print(result["predicted_failure_risk"]) # "low"
+print(result["common_pitfalls"])        # ["timeout (8% of failures)"]
+print(result["top_alternatives"])       # [{"tool": "...", "score": 90}]
+
+# Report
+client.report("https://api.openai.com/v1/chat/completions",
+              success=True, latency_ms=2500)
+
+client.close()
+```
+
+## Async support
+
+`AsyncToolRate` has the same interface — all methods are `async`.
+
+```python
+from toolrate import AsyncToolRate
+
+async with AsyncToolRate(api_key="nf_live_...") as client:
+    result = await client.assess("https://api.openai.com/v1/chat/completions")
+```

toolrate-0.3.2/README.md

@@ -0,0 +1,125 @@
+# ToolRate Python SDK
+
+Python client for the [ToolRate API](https://api.toolrate.ai) — the reliability oracle for AI agents.
+
+## Installation
+
+```bash
+pip install toolrate
+```
+
+## Quick start — one line of code
+
+The `guard` function wraps any tool call with automatic reliability checking:
+
+```python
+from toolrate import ToolRate, guard
+
+client = ToolRate(api_key="nf_live_...")
+
+# Wrap any tool call — assesses before, reports after, automatically
+result = guard(client, "https://api.openai.com/v1/chat/completions",
+               lambda: openai.chat.completions.create(model="gpt-4", messages=[...]))
+```
+
+That's it. ToolRate will:
+1. Check the tool's reliability score before calling
+2. Execute the tool call
+3. Report success/failure back (building the data moat)
+4. Classify errors automatically
+
+## Auto-fallback
+
+When a tool fails, automatically try alternatives:
+
+```python
+result = guard(
+    client,
+    "https://api.openai.com/v1/chat/completions",
+    lambda: openai.chat.completions.create(model="gpt-4", messages=msgs),
+    fallbacks=[
+        ("https://api.anthropic.com/v1/messages",
+         lambda: anthropic.messages.create(model="claude-sonnet-4-20250514", messages=msgs)),
+        ("https://api.groq.com/openai/v1/chat/completions",
+         lambda: groq.chat.completions.create(model="llama-3.3-70b", messages=msgs)),
+    ],
+    min_score=50,  # Skip tools scoring below 50
+)
+```
+
+## Decorator
+
+```python
+from toolrate import ToolRate, toolrate_guard
+
+client = ToolRate(api_key="nf_live_...")
+
+@toolrate_guard(client, "https://api.stripe.com/v1/charges")
+def charge_customer(amount, currency):
+    return stripe.Charge.create(amount=amount, currency=currency)
+
+# Every call is now automatically assessed + reported
+charge_customer(1000, "usd")
+```
+
+## Journey tracking
+
+Report fallback patterns to power hidden gem discovery:
+
+```python
+# First attempt fails
+client.report("https://api.sendgrid.com/v3/mail/send",
+    success=False, error_category="rate_limit",
+    session_id="session-123", attempt_number=1)
+
+# Fallback succeeds
+client.report("https://api.resend.com/emails",
+    success=True, latency_ms=180,
+    session_id="session-123", attempt_number=2,
+    previous_tool="https://api.sendgrid.com/v3/mail/send")
+```
+
+## Discovery
+
+Find hidden gems and fallback chains based on real agent behavior:
+
+```python
+# Tools that shine as fallbacks
+gems = client.discover_hidden_gems(category="email")
+
+# What to try when SendGrid fails
+chain = client.discover_fallback_chain("https://api.sendgrid.com/v3/mail/send")
+```
+
+## Direct API usage
+
+```python
+from toolrate import ToolRate
+
+client = ToolRate(api_key="nf_live_...")
+
+# Assess
+result = client.assess("https://api.openai.com/v1/chat/completions",
+                        context="customer support chatbot")
+print(result["reliability_score"])      # 89.0
+print(result["predicted_failure_risk"]) # "low"
+print(result["common_pitfalls"])        # ["timeout (8% of failures)"]
+print(result["top_alternatives"])       # [{"tool": "...", "score": 90}]
+
+# Report
+client.report("https://api.openai.com/v1/chat/completions",
+              success=True, latency_ms=2500)
+
+client.close()
+```
+
+## Async support
+
+`AsyncToolRate` has the same interface — all methods are `async`.
+
+```python
+from toolrate import AsyncToolRate
+
+async with AsyncToolRate(api_key="nf_live_...") as client:
+    result = await client.assess("https://api.openai.com/v1/chat/completions")
+```

toolrate-0.3.2/pyproject.toml

@@ -0,0 +1,35 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "toolrate"
+version = "0.3.2"
+description = "Reliability oracle for AI agents — pick the right tool from the start"
+readme = "README.md"
+requires-python = ">=3.9"
+license = "MIT"
+authors = [
+    {name = "ToolRate", email = "bleep@toolrate.ai"},
+]
+keywords = ["ai", "agents", "reliability", "tools", "llm", "api"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Libraries",
+]
+dependencies = [
+    "httpx>=0.24.0",
+]
+
+[project.urls]
+Homepage = "https://toolrate.ai"
+Documentation = "https://api.toolrate.ai/docs"
+Repository = "https://github.com/netvistamedia/toolrate"

toolrate-0.3.2/toolrate/__init__.py

@@ -0,0 +1,28 @@
+"""ToolRate Python SDK — reliability oracle for AI agents.
+
+Before your agent calls an external tool, check ToolRate for the
+reliability score, common pitfalls, and smart alternatives.
+"""
+from .client import (
+    ToolRate,
+    AsyncToolRate,
+    # Backwards-compatible aliases (the package used to be called `nemoflow`)
+    NemoFlowClient,
+    AsyncNemoFlowClient,
+)
+from .guard import guard, toolrate_guard
+
+# Legacy alias for the decorator that used to be called nemoflow_guard
+nemoflow_guard = toolrate_guard
+
+__all__ = [
+    "ToolRate",
+    "AsyncToolRate",
+    "guard",
+    "toolrate_guard",
+    # Backwards-compat exports
+    "NemoFlowClient",
+    "AsyncNemoFlowClient",
+    "nemoflow_guard",
+]
+__version__ = "0.3.2"

toolrate-0.3.2/toolrate/client.py

@@ -0,0 +1,430 @@
+from __future__ import annotations
+
+from typing import Any, Optional
+
+import httpx
+
+_DEFAULT_BASE_URL = "https://api.toolrate.ai"
+_DEFAULT_TIMEOUT = 30.0
+
+
+class ToolRate:
+    """Synchronous client for the ToolRate API."""
+
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str = _DEFAULT_BASE_URL,
+        timeout: float = _DEFAULT_TIMEOUT,
+    ) -> None:
+        self._api_key = api_key
+        self._base_url = base_url.rstrip("/")
+        self._client = httpx.Client(
+            base_url=self._base_url,
+            headers={"X-Api-Key": self._api_key},
+            timeout=timeout,
+        )
+
+    # -- Assessment ------------------------------------------------------------
+
+    def assess(
+        self,
+        tool_identifier: str,
+        context: str = "",
+        sample_payload: Optional[dict[str, Any]] = None,
+    ) -> dict[str, Any]:
+        """Assess a tool's reliability and get recommendations."""
+        body: dict[str, Any] = {
+            "tool_identifier": tool_identifier,
+            "context": context,
+        }
+        if sample_payload is not None:
+            body["sample_payload"] = sample_payload
+
+        resp = self._client.post("/v1/assess", json=body)
+        resp.raise_for_status()
+        return resp.json()
+
+    def assess_batch(
+        self,
+        tools: list[dict[str, str]],
+    ) -> dict[str, Any]:
+        """Assess up to 20 tools in a single request.
+
+        Args:
+            tools: List of dicts with 'tool_identifier' and optional 'context'.
+                   Example: [{"tool_identifier": "https://api.stripe.com/v1/charges", "context": "payment"}]
+        """
+        resp = self._client.post("/v1/assess/batch", json={"tools": tools})
+        resp.raise_for_status()
+        return resp.json()
+
+    # -- Reporting -------------------------------------------------------------
+
+    def report(
+        self,
+        tool_identifier: str,
+        success: bool,
+        error_category: Optional[str] = None,
+        latency_ms: Optional[int] = None,
+        context: str = "",
+        session_id: Optional[str] = None,
+        attempt_number: Optional[int] = None,
+        previous_tool: Optional[str] = None,
+    ) -> dict[str, Any]:
+        """Report a tool execution outcome.
+
+        For journey tracking, include session_id, attempt_number, and
+        previous_tool when retrying after a failure. This data powers
+        the hidden gems and fallback chain features.
+        """
+        body: dict[str, Any] = {
+            "tool_identifier": tool_identifier,
+            "success": success,
+            "context": context,
+        }
+        if error_category is not None:
+            body["error_category"] = error_category
+        if latency_ms is not None:
+            body["latency_ms"] = latency_ms
+        if session_id is not None:
+            body["session_id"] = session_id
+        if attempt_number is not None:
+            body["attempt_number"] = attempt_number
+        if previous_tool is not None:
+            body["previous_tool"] = previous_tool
+
+        resp = self._client.post("/v1/report", json=body)
+        resp.raise_for_status()
+        return resp.json()
+
+    # -- Discovery -------------------------------------------------------------
+
+    def discover_hidden_gems(
+        self, category: Optional[str] = None, limit: int = 10
+    ) -> dict[str, Any]:
+        """Find hidden gem tools that shine as fallbacks."""
+        params: dict[str, Any] = {"limit": limit}
+        if category:
+            params["category"] = category
+        resp = self._client.get("/v1/discover/hidden-gems", params=params)
+        resp.raise_for_status()
+        return resp.json()
+
+    def discover_fallback_chain(
+        self, tool_identifier: str, limit: int = 5
+    ) -> dict[str, Any]:
+        """Get the best fallback tools when this tool fails."""
+        resp = self._client.get(
+            "/v1/discover/fallback-chain",
+            params={"tool_identifier": tool_identifier, "limit": limit},
+        )
+        resp.raise_for_status()
+        return resp.json()
+
+    # -- Tools -----------------------------------------------------------------
+
+    def search_tools(
+        self,
+        q: Optional[str] = None,
+        category: Optional[str] = None,
+        offset: int = 0,
+        limit: int = 50,
+    ) -> dict[str, Any]:
+        """Search and browse all rated tools."""
+        params: dict[str, Any] = {"offset": offset, "limit": limit}
+        if q:
+            params["q"] = q
+        if category:
+            params["category"] = category
+        resp = self._client.get("/v1/tools", params=params)
+        resp.raise_for_status()
+        return resp.json()
+
+    def list_categories(self) -> dict[str, Any]:
+        """List all tool categories with counts."""
+        resp = self._client.get("/v1/tools/categories")
+        resp.raise_for_status()
+        return resp.json()
+
+    # -- Stats -----------------------------------------------------------------
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get platform-wide statistics."""
+        resp = self._client.get("/v1/stats")
+        resp.raise_for_status()
+        return resp.json()
+
+    def get_my_stats(self) -> dict[str, Any]:
+        """Get personal usage statistics (tier, limits, usage)."""
+        resp = self._client.get("/v1/stats/me")
+        resp.raise_for_status()
+        return resp.json()
+
+    # -- Webhooks --------------------------------------------------------------
+
+    def create_webhook(
+        self,
+        url: str,
+        threshold: int = 5,
+        tool_identifier: Optional[str] = None,
+        event: str = "score.change",
+    ) -> dict[str, Any]:
+        """Register a webhook for score change alerts.
+
+        Returns the webhook details including the HMAC signing secret
+        (only shown once — store it securely).
+        """
+        body: dict[str, Any] = {"url": url, "event": event, "threshold": threshold}
+        if tool_identifier is not None:
+            body["tool_identifier"] = tool_identifier
+        resp = self._client.post("/v1/webhooks", json=body)
+        resp.raise_for_status()
+        return resp.json()
+
+    def list_webhooks(self) -> dict[str, Any]:
+        """List all your registered webhooks."""
+        resp = self._client.get("/v1/webhooks")
+        resp.raise_for_status()
+        return resp.json()
+
+    def delete_webhook(self, webhook_id: str) -> dict[str, Any]:
+        """Delete a webhook by ID."""
+        resp = self._client.delete(f"/v1/webhooks/{webhook_id}")
+        resp.raise_for_status()
+        return resp.json()
+
+    # -- Account ---------------------------------------------------------------
+
+    def rotate_key(self) -> dict[str, Any]:
+        """Rotate your API key. Returns a new key; the current key is deactivated.
+
+        Important: Update your client with the new key after calling this.
+        """
+        resp = self._client.post("/v1/auth/rotate-key")
+        resp.raise_for_status()
+        return resp.json()
+
+    def delete_account(self) -> dict[str, Any]:
+        """Permanently delete your account and all associated data.
+
+        This action cannot be undone. Your API key will be deactivated
+        and all webhooks removed.
+        """
+        resp = self._client.delete("/v1/account")
+        resp.raise_for_status()
+        return resp.json()
+
+    # -- Lifecycle -------------------------------------------------------------
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> ToolRate:
+        return self
+
+    def __exit__(self, *exc: Any) -> None:
+        self.close()
+
+
+class AsyncToolRate:
+    """Asynchronous client for the ToolRate API."""
+
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str = _DEFAULT_BASE_URL,
+        timeout: float = _DEFAULT_TIMEOUT,
+    ) -> None:
+        self._api_key = api_key
+        self._base_url = base_url.rstrip("/")
+        self._client = httpx.AsyncClient(
+            base_url=self._base_url,
+            headers={"X-Api-Key": self._api_key},
+            timeout=timeout,
+        )
+
+    # -- Assessment ------------------------------------------------------------
+
+    async def assess(
+        self,
+        tool_identifier: str,
+        context: str = "",
+        sample_payload: Optional[dict[str, Any]] = None,
+    ) -> dict[str, Any]:
+        """Assess a tool's reliability and get recommendations."""
+        body: dict[str, Any] = {
+            "tool_identifier": tool_identifier,
+            "context": context,
+        }
+        if sample_payload is not None:
+            body["sample_payload"] = sample_payload
+
+        resp = await self._client.post("/v1/assess", json=body)
+        resp.raise_for_status()
+        return resp.json()
+
+    async def assess_batch(
+        self,
+        tools: list[dict[str, str]],
+    ) -> dict[str, Any]:
+        """Assess up to 20 tools in a single request."""
+        resp = await self._client.post("/v1/assess/batch", json={"tools": tools})
+        resp.raise_for_status()
+        return resp.json()
+
+    # -- Reporting -------------------------------------------------------------
+
+    async def report(
+        self,
+        tool_identifier: str,
+        success: bool,
+        error_category: Optional[str] = None,
+        latency_ms: Optional[int] = None,
+        context: str = "",
+        session_id: Optional[str] = None,
+        attempt_number: Optional[int] = None,
+        previous_tool: Optional[str] = None,
+    ) -> dict[str, Any]:
+        """Report a tool execution outcome."""
+        body: dict[str, Any] = {
+            "tool_identifier": tool_identifier,
+            "success": success,
+            "context": context,
+        }
+        if error_category is not None:
+            body["error_category"] = error_category
+        if latency_ms is not None:
+            body["latency_ms"] = latency_ms
+        if session_id is not None:
+            body["session_id"] = session_id
+        if attempt_number is not None:
+            body["attempt_number"] = attempt_number
+        if previous_tool is not None:
+            body["previous_tool"] = previous_tool
+
+        resp = await self._client.post("/v1/report", json=body)
+        resp.raise_for_status()
+        return resp.json()
+
+    # -- Discovery -------------------------------------------------------------
+
+    async def discover_hidden_gems(
+        self, category: Optional[str] = None, limit: int = 10
+    ) -> dict[str, Any]:
+        """Find hidden gem tools that shine as fallbacks."""
+        params: dict[str, Any] = {"limit": limit}
+        if category:
+            params["category"] = category
+        resp = await self._client.get("/v1/discover/hidden-gems", params=params)
+        resp.raise_for_status()
+        return resp.json()
+
+    async def discover_fallback_chain(
+        self, tool_identifier: str, limit: int = 5
+    ) -> dict[str, Any]:
+        """Get the best fallback tools when this tool fails."""
+        resp = await self._client.get(
+            "/v1/discover/fallback-chain",
+            params={"tool_identifier": tool_identifier, "limit": limit},
+        )
+        resp.raise_for_status()
+        return resp.json()
+
+    # -- Tools -----------------------------------------------------------------
+
+    async def search_tools(
+        self,
+        q: Optional[str] = None,
+        category: Optional[str] = None,
+        offset: int = 0,
+        limit: int = 50,
+    ) -> dict[str, Any]:
+        """Search and browse all rated tools."""
+        params: dict[str, Any] = {"offset": offset, "limit": limit}
+        if q:
+            params["q"] = q
+        if category:
+            params["category"] = category
+        resp = await self._client.get("/v1/tools", params=params)
+        resp.raise_for_status()
+        return resp.json()
+
+    async def list_categories(self) -> dict[str, Any]:
+        """List all tool categories with counts."""
+        resp = await self._client.get("/v1/tools/categories")
+        resp.raise_for_status()
+        return resp.json()
+
+    # -- Stats -----------------------------------------------------------------
+
+    async def get_stats(self) -> dict[str, Any]:
+        """Get platform-wide statistics."""
+        resp = await self._client.get("/v1/stats")
+        resp.raise_for_status()
+        return resp.json()
+
+    async def get_my_stats(self) -> dict[str, Any]:
+        """Get personal usage statistics (tier, limits, usage)."""
+        resp = await self._client.get("/v1/stats/me")
+        resp.raise_for_status()
+        return resp.json()
+
+    # -- Webhooks --------------------------------------------------------------
+
+    async def create_webhook(
+        self,
+        url: str,
+        threshold: int = 5,
+        tool_identifier: Optional[str] = None,
+        event: str = "score.change",
+    ) -> dict[str, Any]:
+        """Register a webhook for score change alerts."""
+        body: dict[str, Any] = {"url": url, "event": event, "threshold": threshold}
+        if tool_identifier is not None:
+            body["tool_identifier"] = tool_identifier
+        resp = await self._client.post("/v1/webhooks", json=body)
+        resp.raise_for_status()
+        return resp.json()
+
+    async def list_webhooks(self) -> dict[str, Any]:
+        """List all your registered webhooks."""
+        resp = await self._client.get("/v1/webhooks")
+        resp.raise_for_status()
+        return resp.json()
+
+    async def delete_webhook(self, webhook_id: str) -> dict[str, Any]:
+        """Delete a webhook by ID."""
+        resp = await self._client.delete(f"/v1/webhooks/{webhook_id}")
+        resp.raise_for_status()
+        return resp.json()
+
+    # -- Account ---------------------------------------------------------------
+
+    async def rotate_key(self) -> dict[str, Any]:
+        """Rotate your API key. Returns a new key; the current key is deactivated."""
+        resp = await self._client.post("/v1/auth/rotate-key")
+        resp.raise_for_status()
+        return resp.json()
+
+    async def delete_account(self) -> dict[str, Any]:
+        """Permanently delete your account and all associated data."""
+        resp = await self._client.delete("/v1/account")
+        resp.raise_for_status()
+        return resp.json()
+
+    # -- Lifecycle -------------------------------------------------------------
+
+    async def close(self) -> None:
+        await self._client.aclose()
+
+    async def __aenter__(self) -> AsyncToolRate:
+        return self
+
+    async def __aexit__(self, *exc: Any) -> None:
+        await self.close()
+
+
+# Backwards-compatible aliases (deprecated names kept for existing imports)
+NemoFlowClient = ToolRate
+AsyncNemoFlowClient = AsyncToolRate

toolrate-0.3.2/toolrate/guard.py

@@ -0,0 +1,268 @@
+"""ToolRate Guard — one-line reliability wrapper for tool calls.
+
+Usage:
+    from toolrate import ToolRate, guard
+
+    client = ToolRate("nf_live_...")
+
+    # Wrap any tool call — assess before, report after
+    result = guard(client, "https://api.openai.com/v1/chat/completions",
+                   lambda: openai.chat.completions.create(...))
+
+    # Explicit fallbacks — tries each in order on failure
+    result = guard(client, "https://api.openai.com/v1/chat/completions",
+                   lambda: openai.chat.completions.create(...),
+                   fallbacks=[
+                       ("https://api.anthropic.com/v1/messages",
+                        lambda: anthropic.messages.create(...)),
+                   ])
+
+    # Dynamic (auto) fallbacks — ToolRate picks from real agent journey data
+    result = guard(client, "https://api.openai.com/v1/chat/completions",
+                   lambda: openai.chat.completions.create(...),
+                   fallbacks="auto",
+                   resolvers={
+                       "https://api.anthropic.com/v1/messages":
+                           lambda: anthropic.messages.create(...),
+                       "https://api.groq.com/openai/v1/chat/completions":
+                           lambda: groq_client.chat.completions.create(...),
+                   })
+
+    # As a decorator
+    @toolrate_guard(client, "https://api.stripe.com/v1/charges")
+    def charge_customer(amount, currency):
+        return stripe.Charge.create(amount=amount, currency=currency)
+"""
+
+from __future__ import annotations
+
+import time
+import uuid
+from functools import wraps
+from typing import Any, Callable, Literal, TypeVar, Union
+
+from toolrate.client import ToolRate
+
+T = TypeVar("T")
+
+Fallbacks = Union[list[tuple[str, Callable[[], T]]], Literal["auto"], None]
+
+
+def guard(
+    client: ToolRate,
+    tool_identifier: str,
+    fn: Callable[[], T],
+    *,
+    context: str = "",
+    min_score: float = 0.0,
+    fallbacks: Fallbacks = None,
+    resolvers: dict[str, Callable[[], T]] | None = None,
+    max_fallbacks: int = 3,
+) -> T:
+    """Execute a tool call with ToolRate reliability guard.
+
+    1. Assesses the tool's reliability score
+    2. If score < min_score and fallbacks exist, tries the best-scoring fallback
+    3. Executes the tool call
+    4. Reports success/failure back to ToolRate
+    5. On failure with fallbacks, automatically tries the next option
+
+    Args:
+        client: ToolRate instance
+        tool_identifier: The tool's API identifier
+        fn: The actual tool call to execute (as a callable)
+        context: Workflow context for context-bucketed scoring
+        min_score: Minimum reliability score to proceed (0-100). Default 0 = always try.
+        fallbacks: Either a list of (tool_identifier, callable) pairs, or the string
+            "auto" to have ToolRate pick fallbacks dynamically from the primary tool's
+            top alternatives and real fallback-chain data. "auto" requires `resolvers`.
+        resolvers: Mapping of tool identifier → callable. When `fallbacks="auto"`,
+            ToolRate matches candidate alternatives against these keys and only tries
+            tools the caller has pre-registered a runner for.
+        max_fallbacks: Max number of auto fallbacks to include (default 3).
+
+    Returns:
+        The result of the successful tool call
+
+    Raises:
+        The exception from the last failed tool call if all options are exhausted
+    """
+    session_id = uuid.uuid4().hex[:16]
+
+    if fallbacks == "auto":
+        explicit_fallbacks: list[tuple[str, Callable[[], T]]] = []
+        auto_mode = True
+    else:
+        explicit_fallbacks = list(fallbacks or [])
+        auto_mode = False
+
+    all_tools: list[tuple[str, Callable[[], T]]] = [(tool_identifier, fn)] + explicit_fallbacks
+
+    last_error: Exception | None = None
+    i = 0
+    while i < len(all_tools):
+        attempt = i + 1
+        ident, call = all_tools[i]
+
+        # Assess
+        assessment: dict[str, Any] | None = None
+        try:
+            assessment = client.assess(ident, context=context)
+            score = assessment.get("reliability_score", 100)
+        except Exception:
+            score = 100  # If assess fails, don't block the tool call
+
+        # Resolve auto fallbacks once, using the primary tool's assessment (no extra API call if it has alternatives)
+        if auto_mode and i == 0:
+            auto_tools = _resolve_auto_fallbacks(
+                client, ident, assessment, resolvers or {}, max_fallbacks
+            )
+            all_tools.extend(auto_tools)
+            auto_mode = False
+
+        # Skip if score too low and we have more options
+        if score < min_score and attempt < len(all_tools):
+            _safe_report(
+                client, ident, success=False, error_category="skipped_low_score",
+                context=context, session_id=session_id, attempt_number=attempt,
+                previous_tool=all_tools[i - 1][0] if i > 0 else None,
+            )
+            i += 1
+            continue
+
+        # Execute
+        start = time.perf_counter()
+        try:
+            result = call()
+            latency_ms = int((time.perf_counter() - start) * 1000)
+
+            _safe_report(
+                client, ident, success=True, latency_ms=latency_ms,
+                context=context, session_id=session_id, attempt_number=attempt,
+                previous_tool=all_tools[i - 1][0] if i > 0 else None,
+            )
+            return result
+
+        except Exception as e:
+            latency_ms = int((time.perf_counter() - start) * 1000)
+            last_error = e
+
+            _safe_report(
+                client, ident, success=False, error_category=_classify_error(e),
+                latency_ms=latency_ms, context=context, session_id=session_id,
+                attempt_number=attempt,
+                previous_tool=all_tools[i - 1][0] if i > 0 else None,
+            )
+
+            if attempt >= len(all_tools):
+                raise
+            i += 1
+
+    raise last_error  # type: ignore[misc]
+
+
+def toolrate_guard(
+    client: ToolRate,
+    tool_identifier: str,
+    *,
+    context: str = "",
+    min_score: float = 0.0,
+    fallbacks: Fallbacks = None,
+    resolvers: dict[str, Callable[[], Any]] | None = None,
+    max_fallbacks: int = 3,
+):
+    """Decorator version of guard.
+
+    Usage:
+        @toolrate_guard(client, "https://api.stripe.com/v1/charges")
+        def charge(amount, currency):
+            return stripe.Charge.create(amount=amount, currency=currency)
+    """
+    def decorator(fn: Callable[..., T]) -> Callable[..., T]:
+        @wraps(fn)
+        def wrapper(*args: Any, **kwargs: Any) -> T:
+            return guard(
+                client, tool_identifier,
+                lambda: fn(*args, **kwargs),
+                context=context, min_score=min_score,
+                fallbacks=fallbacks, resolvers=resolvers,
+                max_fallbacks=max_fallbacks,
+            )
+        return wrapper
+    return decorator
+
+
+def _resolve_auto_fallbacks(
+    client: ToolRate,
+    primary_identifier: str,
+    primary_assessment: dict[str, Any] | None,
+    resolvers: dict[str, Callable[[], Any]],
+    max_n: int,
+) -> list[tuple[str, Callable[[], Any]]]:
+    """Pick fallback callables by matching ToolRate's alternatives against user resolvers."""
+    if not resolvers or max_n <= 0:
+        return []
+
+    candidates: list[str] = []
+
+    # 1. Reuse top_alternatives from the assessment we already fetched (no extra API call)
+    if primary_assessment:
+        for alt in primary_assessment.get("top_alternatives") or []:
+            if isinstance(alt, dict) and alt.get("tool"):
+                candidates.append(alt["tool"])
+
+    # 2. If no alternatives in assess response, query fallback-chain endpoint
+    if not candidates:
+        try:
+            chain_resp = client.discover_fallback_chain(primary_identifier)
+            for item in chain_resp.get("fallback_chain") or []:
+                if isinstance(item, dict) and item.get("fallback_tool"):
+                    candidates.append(item["fallback_tool"])
+        except Exception:
+            pass
+
+    out: list[tuple[str, Callable[[], Any]]] = []
+    seen: set[str] = {primary_identifier}
+    for ident in candidates:
+        if ident in seen:
+            continue
+        runner = resolvers.get(ident)
+        if runner is None:
+            continue
+        out.append((ident, runner))
+        seen.add(ident)
+        if len(out) >= max_n:
+            break
+
+    return out
+
+
+def _safe_report(client: ToolRate, tool_identifier: str, **kwargs: Any) -> None:
+    """Fire-and-forget reporting. Never fail the user's tool call because reporting failed."""
+    try:
+        client.report(tool_identifier, **kwargs)
+    except Exception:
+        pass
+
+
+def _classify_error(error: Exception) -> str:
+    """Best-effort classification of an exception into ToolRate error categories."""
+    name = type(error).__name__.lower()
+    message = str(error).lower()
+
+    if "timeout" in name or "timeout" in message or "timed out" in message:
+        return "timeout"
+    if "ratelimit" in name or "rate" in message and "limit" in message or "429" in message or "too many" in message:
+        return "rate_limit"
+    if "auth" in name or "unauthorized" in message or "403" in message or "401" in message:
+        return "auth_failure"
+    if "validation" in name or "invalid" in message or "422" in message:
+        return "validation_error"
+    if "notfound" in name or "not found" in message or "404" in message:
+        return "not_found"
+    if "permission" in name or "forbidden" in message:
+        return "permission_denied"
+    if "connect" in name or "connection" in message or "dns" in message:
+        return "connection_error"
+
+    return "server_error"