agent-handler-sdk 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

agent_handler_sdk/auth.py

@@ -11,9 +11,14 @@ class AuthContext:
     Each tool execution should receive its own instance.
     """
 
-    def __init__(self, secrets: Optional[dict[str, str]] = None):
+    def __init__(self, secrets: Optional[dict[str, str]] = None, oauth2_token: Optional[str] = None):
         self._secrets = secrets or {}
+        self._oauth2_token = oauth2_token
 
     def get(self, key: str, default: Optional[str] = None) -> Optional[str]:
         """Get a secret value by key"""
         return self._secrets.get(key, default)
+
+    def get_oauth2_token(self) -> Optional[str]:
+        """Get the OAuth token from the secrets"""
+        return self._oauth2_token

agent_handler_sdk/cli.py

@@ -56,9 +56,10 @@ def scaffold_connector() -> int:
     pkg_dir = base / f"{slug}_connector"
     tools_dir = pkg_dir / "tools"
     tests_dir = base / "tests"
+    evals_dir = base / "evals"
 
     # Create directories
-    for d in (base, pkg_dir, tools_dir, tests_dir):
+    for d in (base, pkg_dir, tools_dir, tests_dir, evals_dir):
         d.mkdir(parents=True, exist_ok=True)
 
     # Map template → output path
@@ -68,6 +69,7 @@ def scaffold_connector() -> int:
         "init.py.tpl": pkg_dir / "__init__.py",
         "handlers.py.tpl": tools_dir / "handlers.py",
         "test_handlers.py.tpl": tests_dir / "test_handlers.py",
+        "evals.json.tpl": evals_dir / "evals.json",
         "README.md.tpl": base / "README.md",
     }
 

agent_handler_sdk/eval_types.py

@@ -0,0 +1,113 @@
+from typing import List, Dict, Any, Optional, Literal, Union
+from pydantic import BaseModel, Extra
+from datetime import datetime
+
+
+class JsonSchema(BaseModel):
+    type: Optional[str] = None
+    properties: Optional[Dict[str, "JsonSchema"]] = None
+    items: Optional[Union["JsonSchema", List["JsonSchema"]]] = None
+    required: Optional[List[str]] = None
+    enum: Optional[List[Any]] = None
+    description: Optional[str] = None
+    additional_properties: Optional[Union[bool, "JsonSchema"]] = None
+    model: Optional[str] = None
+
+    class Config:
+        arbitrary_types_allowed = True
+        extra = "allow"
+
+
+JsonSchema.model_rebuild()
+
+
+class DataSourceConfig(BaseModel):
+    input_schema: JsonSchema
+
+
+class MessageContent(BaseModel):
+    type: str
+    text: str
+
+
+class MessageInput(BaseModel):
+    type: str
+    role: str
+    content: MessageContent
+
+
+class BaseEvaluator(BaseModel):
+    name: str
+    id: str
+    type: str  # Discriminator for future extension
+
+
+class ReferenceToolCallsMatchEvaluator(BaseEvaluator):
+    type: Literal["reference_tool_calls_match"]
+    enforce_ordering: bool
+    fail_on_args_mismatch: bool
+
+
+class LabelModelEvaluator(BaseEvaluator):
+    type: Literal["label_model"]
+    passing_labels: Optional[List[str]]
+    labels: Optional[List[str]]
+    model: Optional[str]
+    input: List[MessageInput]
+
+
+Evaluator = Union[ReferenceToolCallsMatchEvaluator, LabelModelEvaluator, BaseEvaluator]
+
+
+class EvalMetadata(BaseModel):
+    description: Optional[str]
+
+
+class EvalItemInput(BaseModel, extra=Extra.allow):
+    input: str
+
+
+class EvalItem(BaseModel, extra=Extra.allow):
+    """
+    Schema for individual eval items.
+    Supports both runtime evaluation (with id and tool_calls) and connector eval files (flexible input).
+    """
+
+    input: Union[str, EvalItemInput]  # Can be either a string or EvalItemInput object
+    id: Optional[str] = None  # Optional for connector eval files
+
+
+class ConnectorEvalBundle(BaseModel):
+    """
+    Schema for eval bundles stored in connector /evals/ folders.
+    This matches the JSON structure that contains config, items, and prompts together.
+    """
+
+    data_source_config: DataSourceConfig
+    items: List[EvalItem]
+    prompts: List[MessageInput]
+    name: str
+    metadata: Optional[EvalMetadata] = None
+
+    def to_eval_config(self) -> "EvalConfig":
+        """
+        Convert this bundle to an EvalConfig for use with the eval runner.
+        Note: This creates a minimal EvalConfig without testing_evaluators.
+        """
+        return EvalConfig(
+            id=None,
+            created_at=None,
+            updated_at=None,
+            data_source_config=self.data_source_config,
+            testing_evaluators=[],  # Empty list since connector evals don't define evaluators
+            metadata=self.metadata,
+        )
+
+
+class EvalConfig(BaseModel):
+    id: Optional[str]
+    created_at: Optional[datetime]
+    updated_at: Optional[datetime]
+    data_source_config: DataSourceConfig
+    testing_evaluators: Optional[List[Evaluator]] = []
+    metadata: Optional[EvalMetadata]

agent_handler_sdk/templates/connector/evals.json.tpl

@@ -0,0 +1,55 @@
+[
+    {
+      "data_source_config": {
+        "input_schema": {
+          "type": "object",
+          "properties": {
+            "input": { "type": "string" },
+            "reference_value": { "type": "string" },
+            "reference_tools": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "name": { "type": "string" },
+                  "args": { "type": "object" }
+                },
+                "required": ["name"]
+              }
+            }
+          },
+          "required": ["input"]
+        }
+      },
+      "items": [
+        {
+          "input": "Tell me about George Washington",
+          "reference_value": null,
+          "reference_tools": [
+            {
+              "name": "wikipedia__search",
+              "args": {
+                "query": "George Washington"
+              }
+            }
+          ]
+        },
+        {
+          "input": "Where does London's name come from? Don't use any tools to solve this.",
+          "reference_value": "London's name is believed to originate from the Latin word \"Londinium,\" which was the name used during the Roman period when the city was established as a settlement. The exact origin of \"Londinium\" is uncertain, but it may derive from a pre-Roman or Celtic word. Over time, the name evolved through various forms, such as \"Londinium\" in Latin and \"Lunden\" in Old English, eventually becoming \"London\" as we know it today.",
+          "reference_tools": []
+          ]
+        }
+      ],
+      "prompts": [
+        {
+          "type": "text",
+          "role": "user",
+          "content": {
+            "type": "text",
+            "text": "{{input}}"
+          }
+        }
+      ]
+    }
+]

agent_handler_sdk/templates/evals/models.py

@@ -0,0 +1,83 @@
+from typing import List, Dict, Any, Optional, Literal, Union
+from pydantic import BaseModel, Extra
+from datetime import datetime
+
+
+class JsonSchema(BaseModel):
+    type: Optional[str] = None
+    properties: Optional[Dict[str, "JsonSchema"]] = None
+    items: Optional[Union["JsonSchema", List["JsonSchema"]]] = None
+    required: Optional[List[str]] = None
+    enum: Optional[List[Any]] = None
+    description: Optional[str] = None
+    additional_properties: Optional[Union[bool, "JsonSchema"]] = None
+    model: Optional[str] = None
+
+    class Config:
+        arbitrary_types_allowed = True
+        extra = "allow"
+
+
+JsonSchema.model_rebuild()
+
+
+class DataSourceConfig(BaseModel):
+    input_schema: JsonSchema
+
+
+class MessageContent(BaseModel):
+    type: str
+    text: str
+
+
+class MessageInput(BaseModel):
+    type: str
+    role: str
+    content: MessageContent
+
+
+class BaseEvaluator(BaseModel):
+    name: str
+    id: str
+    type: str
+
+
+class ToolCallModelEvaluator(BaseEvaluator):
+    type: Literal["tool_call"]
+    tool_name: str
+    params: dict | None
+
+
+class LabelModelEvaluator(BaseEvaluator):
+    type: Literal["label_model"]
+    passing_labels: Optional[List[str]]
+    labels: Optional[List[str]]
+    model: Optional[str]
+    input: List[MessageInput]
+
+
+Evaluator = Union[ToolCallModelEvaluator, LabelModelEvaluator, BaseEvaluator]
+
+
+class EvalMetadata(BaseModel):
+    description: Optional[str]
+
+
+class EvalItemInput(BaseModel, extra=Extra.allow):
+    input: str
+
+
+class EvalItem(BaseModel):
+    id: str
+    input: EvalItemInput
+    tool_calls: Optional[List[Dict[str, Any]]] = None
+
+
+class EvalConfig(BaseModel):
+    id: str
+    created_at: datetime
+    updated_at: datetime
+    data_source_config: DataSourceConfig
+    testing_evaluators: List[Evaluator]
+    name: str
+    metadata: EvalMetadata

{agent_handler_sdk-0.1.3.dist-info → agent_handler_sdk-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: agent-handler-sdk
-Version: 0.1.3
+Version: 0.1.5
 Summary: Agent Handler SDK for defining and invoking LLM tools
 Author: David Dalmaso
 Author-email: david.dalmaso@merge.dev

{agent_handler_sdk-0.1.3.dist-info → agent_handler_sdk-0.1.5.dist-info}/RECORD

@@ -1,19 +1,22 @@
 agent_handler_sdk/__init__.py,sha256=QAAV_YdhBAP5P3DVB0fZfjV6jJnLnRwKBYvGixzrIRI,56
-agent_handler_sdk/auth.py,sha256=OC24JhY0qiQbx8rM7sRPwz4gpsZGd3c-zxZ6uLoEwp0,579
-agent_handler_sdk/cli.py,sha256=gQWq5kmtaOWiO4v8suu-5k-mA5pQOL1zIBEkmMj94Ho,2496
+agent_handler_sdk/auth.py,sha256=PThZSKS1N5W2wofUKvsQr-e4SpFmjDZK0o46f85qyBs,792
+agent_handler_sdk/cli.py,sha256=Yd5d7KdCqzXlGv5LIjDBVru7OPgsr2gq-cn7h-ZF3sE,2590
 agent_handler_sdk/connector.py,sha256=IhCfxZ9-ah1fgQsHHYD2Ip9CzOG6Sok6C6dbKA-iOWU,3395
+agent_handler_sdk/eval_types.py,sha256=uF4K08vXtEB7rrUBA_Wr4wVsMAC4Vzm4aNL2Xm2i2P0,3073
 agent_handler_sdk/exceptions.py,sha256=78LAytptKzrK2Vo1QsucnDlXFIH7pKDOz-K1Jwi5kdQ,98
 agent_handler_sdk/invocation.py,sha256=0mn_arvbMncrWqrXtKcH2wjfWlXP49VtcJhAekFGxkw,2054
 agent_handler_sdk/registry.py,sha256=NecI41OSK37YAlFPeKf2bONqyoKrUq7GJ5eMuQ8kb1w,1699
 agent_handler_sdk/templates/connector/README.md.tpl,sha256=KrCFKwrYSVxHJu6PK-03USpDellO-FYXsfUdOZS7gB4,646
+agent_handler_sdk/templates/connector/evals.json.tpl,sha256=j01AwtMYq8w7lprdYVKmgj_IlVRXmGe9awigmj_nIJ8,1772
 agent_handler_sdk/templates/connector/handlers.py.tpl,sha256=bijFfajA3o0M4x33UAEOnf9958lszTM6OuykolST3fQ,132
 agent_handler_sdk/templates/connector/init.py.tpl,sha256=oTrEFARFfcsMOmaJ_A5dKfierKSSzwDLg9PdrlgXXi0,361
 agent_handler_sdk/templates/connector/metadata.yaml.tpl,sha256=imIElqfomfMHFp-dFEPwbuWixKO9HIkkwSmIRYva46c,225
 agent_handler_sdk/templates/connector/pyproject.toml.tpl,sha256=QddVTsMbKcj6iV0svcWxb_hk3bJ74F2sXKYeQpi0l9I,652
 agent_handler_sdk/templates/connector/test_handlers.py.tpl,sha256=ptV4Ci-5C1Edr7e3FHqM4yj6d2H-Qi4cBI3MXQZIimk,696
+agent_handler_sdk/templates/evals/models.py,sha256=MtC4Cr2o-ENz0GfOe6Armtogo374d8Ly_TBal2uRveY,1809
 agent_handler_sdk/tool.py,sha256=s7xMRoB_HwtSkeeuGqkbZzp6Rt9kI4KVUxpULeIrLn8,4292
 agent_handler_sdk/utils.py,sha256=oFSfocf0PqbeOz9LH0kgBfegDFRSNeh9LPS19esNOXY,7219
-agent_handler_sdk-0.1.3.dist-info/METADATA,sha256=sqTnLCiGhmgRD0k5DkvLl1MCVVEVumkPs9_1kWWAiJY,536
-agent_handler_sdk-0.1.3.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-agent_handler_sdk-0.1.3.dist-info/entry_points.txt,sha256=QGDWaLUjKDHhNb4ZpvSj1MrrlQWozz1QuqLecdWAEYA,73
-agent_handler_sdk-0.1.3.dist-info/RECORD,,
+agent_handler_sdk-0.1.5.dist-info/METADATA,sha256=1XzlGEwwYVDG856-JJQaD_2F8O_aIhjNhU2j-rvokbY,536
+agent_handler_sdk-0.1.5.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+agent_handler_sdk-0.1.5.dist-info/entry_points.txt,sha256=QGDWaLUjKDHhNb4ZpvSj1MrrlQWozz1QuqLecdWAEYA,73
+agent_handler_sdk-0.1.5.dist-info/RECORD,,