secondshift 0.1.0b12__py3-none-any.whl

secondshift/connections/http_api/spec.py

@@ -0,0 +1,382 @@
+"""Read an OpenAPI 3 spec into operations a job can call.
+
+The spec only describes operations. Where requests go is the base URL the owner
+confirmed, which is saved with the connection and never read from the spec at
+call time.
+"""
+
+import json
+import re
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import urljoin, urlsplit
+
+import anyio
+import yaml
+from asgiref.sync import async_to_sync
+from django.conf import settings
+from jsonschema import Draft202012Validator
+from jsonschema.exceptions import SchemaError
+
+from secondshift.connections.errors import (
+    ExternalRequestError,
+    ExternalToolError,
+    ToolArgumentError,
+)
+from secondshift.connections.outbound import (
+    OutboundUrlError,
+    outbound_client,
+    validate_outbound_url,
+)
+
+SPEC_MAX_BYTES = 5 * 1024 * 1024
+MAX_OPERATIONS = 500
+SCHEMA_MAX_DEPTH = 20
+SCHEMA_MAX_BYTES = 64 * 1024
+DESCRIPTION_MAX_CHARS = 500
+METHODS = ("get", "post", "put", "patch", "delete")
+OPERATION_NAME_PATTERN = r"[A-Za-z0-9_.-]{1,64}"
+# Path and query only. Headers and cookies stay under SecondShift's control.
+PARAMETER_LOCATIONS = {"path", "query"}
+BODY = "body"
+
+
+class APIError(ExternalToolError):
+    pass
+
+
+class APIRequestError(APIError, ExternalRequestError):
+    """A remote failure the agent may report as unavailable evidence."""
+
+
+class APIArgumentError(APIError, ToolArgumentError):
+    """Arguments the operation cannot take. The model may fix them."""
+
+
+@dataclass(frozen=True)
+class Parameter:
+    name: str
+    location: str
+    required: bool
+
+
+@dataclass(frozen=True)
+class Operation:
+    name: str
+    method: str
+    path: str
+    description: str
+    parameters: tuple[Parameter, ...]
+    has_body: bool
+    input_schema: dict[str, Any]
+
+    @property
+    def read_only(self) -> bool:
+        return self.method == "get"
+
+    def definition(self) -> dict[str, Any]:
+        """What is stored and frozen into run plans. No credentials."""
+        return {
+            "method": self.method,
+            "path": self.path,
+            "description": self.description,
+            "parameters": [
+                {"name": item.name, "in": item.location, "required": item.required}
+                for item in self.parameters
+            ],
+            "body": self.has_body,
+            "inputSchema": self.input_schema,
+            "readOnly": self.read_only,
+        }
+
+
+@dataclass(frozen=True)
+class Spec:
+    title: str
+    servers: tuple[str, ...]
+    operations: dict[str, Operation] = field(default_factory=dict)
+
+
+class NoAliasLoader(yaml.SafeLoader):
+    """YAML without anchors and aliases, so a small file cannot expand to a huge one."""
+
+    def compose_node(self, parent: Any, index: Any) -> Any:
+        if self.check_event(yaml.AliasEvent):
+            raise APIError("Specs with YAML aliases are not supported.")
+        return super().compose_node(parent, index)
+
+
+def parse_document(text: str) -> dict[str, Any]:
+    if len(text.encode()) > SPEC_MAX_BYTES:
+        raise APIError("The spec is larger than 5 MB.")
+    try:
+        document = json.loads(text)
+    except ValueError:
+        try:
+            document = yaml.load(text, Loader=NoAliasLoader)
+        except yaml.YAMLError:
+            raise APIError("The spec is not valid JSON or YAML.") from None
+    if not isinstance(document, dict):
+        raise APIError("The spec is not an OpenAPI document.")
+    if not str(document.get("openapi", "")).startswith("3."):
+        raise APIError("Only OpenAPI 3 specs are supported.")
+    return document
+
+
+async def fetch_text(url: str) -> str:
+    try:
+        with anyio.fail_after(settings.OUTBOUND_TIMEOUT_SECONDS):
+            async with outbound_client(
+                {"Accept": "application/json, application/yaml, text/yaml, */*"}
+            ) as client:
+                response = await client.get(url)
+                body = await response.aread()
+    except Exception:
+        raise APIRequestError("The spec could not be downloaded.") from None
+    if 300 <= response.status_code < 400:
+        raise APIError("The spec URL redirects. Use the final address.")
+    if response.status_code != 200:
+        raise APIError("The spec could not be downloaded.")
+    return body.decode("utf-8", errors="replace")
+
+
+def checked_url(value: str, noun: str) -> str:
+    try:
+        return validate_outbound_url(value, noun, resolve=False)
+    except OutboundUrlError as error:
+        raise APIError(str(error)) from None
+
+
+def load_spec(spec_url: str = "", spec_text: str = "") -> Spec:
+    """From a URL, or from pasted text when the spec is not published."""
+    if spec_url:
+        spec_url = checked_url(spec_url, "spec URL")
+        spec_text = async_to_sync(fetch_text)(spec_url)
+    if not spec_text.strip():
+        raise APIError("Enter a spec URL or paste a spec.")
+    return read_spec(parse_document(spec_text), spec_url)
+
+
+def read_spec(document: dict[str, Any], spec_url: str = "") -> Spec:
+    info = document.get("info") if isinstance(document.get("info"), dict) else {}
+    operations: dict[str, Operation] = {}
+    paths = document.get("paths") if isinstance(document.get("paths"), dict) else {}
+    for path, item in paths.items():
+        if (
+            not isinstance(path, str)
+            or not path.startswith("/")
+            or not isinstance(item, dict)
+        ):
+            continue
+        shared = item.get("parameters") or []
+        for method in METHODS:
+            raw = item.get(method)
+            if not isinstance(raw, dict):
+                continue
+            operation = read_operation(document, path, method, raw, shared)
+            if operation is None or operation.name in operations:
+                continue
+            operations[operation.name] = operation
+            if len(operations) > MAX_OPERATIONS:
+                raise APIError(f"The spec has more than {MAX_OPERATIONS} operations.")
+    if not operations:
+        raise APIError("The spec lists no operations SecondShift can call.")
+    return Spec(
+        title=str(info.get("title", ""))[:200],
+        servers=servers(document, spec_url),
+        operations=operations,
+    )
+
+
+def servers(document: dict[str, Any], spec_url: str) -> tuple[str, ...]:
+    """Absolute server URLs. Relative ones resolve against the spec URL.
+
+    Variables use their defaults. A server that still has a variable, or that
+    fails the outbound check, is left out for the owner to type instead.
+    """
+    found: list[str] = []
+    for item in document.get("servers") or []:
+        if not isinstance(item, dict) or not isinstance(item.get("url"), str):
+            continue
+        url = item["url"]
+        for name, variable in (item.get("variables") or {}).items():
+            if isinstance(variable, dict) and "default" in variable:
+                url = url.replace("{" + str(name) + "}", str(variable["default"]))
+        if "{" in url:
+            continue
+        if not urlsplit(url).scheme:
+            if not spec_url:
+                continue
+            url = urljoin(spec_url, url)
+        try:
+            found.append(checked_url(url.rstrip("/"), "server URL"))
+        except APIError:
+            continue
+    return tuple(dict.fromkeys(found))
+
+
+def operation_name(raw: dict[str, Any], method: str, path: str) -> str:
+    name = str(raw.get("operationId") or "")
+    if re.fullmatch(OPERATION_NAME_PATTERN, name):
+        return name
+    slug = re.sub(r"[^A-Za-z0-9]+", "_", path).strip("_")
+    return f"{method}_{slug}"[:64] or method
+
+
+def read_operation(
+    document: dict[str, Any],
+    path: str,
+    method: str,
+    raw: dict[str, Any],
+    shared: list[Any],
+) -> Operation | None:
+    """An operation SecondShift can call, or None when it is too unclear."""
+    declared = declared_parameters(document, raw, shared)
+    inputs = parameter_inputs(document, declared)
+    if inputs is None:
+        return None
+    properties, required, parameters = inputs
+    # Every {name} in the path must be a declared path parameter.
+    template_names = set(re.findall(r"{([^}/]+)}", path))
+    if template_names != {item.name for item in parameters if item.location == "path"}:
+        return None
+    body = json_body(document, method, raw)
+    if body is not None:
+        properties[BODY], body_required = body
+        if body_required:
+            required.append(BODY)
+    input_schema = checked_input_schema(properties, required)
+    if input_schema is None:
+        return None
+    description = str(raw.get("summary") or raw.get("description") or "")
+    return Operation(
+        name=operation_name(raw, method, path),
+        method=method,
+        path=path,
+        description=f"{method.upper()} {path}. {description}"[:DESCRIPTION_MAX_CHARS],
+        parameters=tuple(parameters),
+        has_body=body is not None,
+        input_schema=input_schema,
+    )
+
+
+def declared_parameters(
+    document: dict[str, Any], raw: dict[str, Any], shared: list[Any]
+) -> dict[tuple[str, str], dict[str, Any]]:
+    """Parameters by location and name. The operation's override the path's."""
+    declared: dict[tuple[str, str], dict[str, Any]] = {}
+    for item in [*shared, *(raw.get("parameters") or [])]:
+        parameter = resolve(document, item)
+        if not isinstance(parameter, dict):
+            continue
+        location = parameter.get("in")
+        name = parameter.get("name")
+        if location in PARAMETER_LOCATIONS and isinstance(name, str) and name != BODY:
+            declared[(location, name)] = parameter
+    return declared
+
+
+def parameter_inputs(
+    document: dict[str, Any], declared: dict[tuple[str, str], dict[str, Any]]
+) -> tuple[dict[str, Any], list[str], list[Parameter]] | None:
+    """Schema properties, required names and parameters. None when ambiguous."""
+    properties: dict[str, Any] = {}
+    required: list[str] = []
+    parameters: list[Parameter] = []
+    for (location, name), parameter in declared.items():
+        if name in properties:
+            # A path and query parameter share a name. Too ambiguous to call.
+            return None
+        is_required = location == "path" or bool(parameter.get("required"))
+        schema = inline(document, parameter.get("schema") or {"type": "string"})
+        if parameter.get("description"):
+            schema = {**schema, "description": str(parameter["description"])[:300]}
+        properties[name] = schema
+        parameters.append(Parameter(name, location, is_required))
+        if is_required:
+            required.append(name)
+    return properties, required, parameters
+
+
+def json_body(
+    document: dict[str, Any], method: str, raw: dict[str, Any]
+) -> tuple[dict[str, Any], bool] | None:
+    """The JSON body schema and whether it is required. None without one."""
+    body = resolve(document, raw.get("requestBody"))
+    if method == "get" or not isinstance(body, dict):
+        return None
+    content = body.get("content") if isinstance(body.get("content"), dict) else {}
+    media = content.get("application/json")
+    if not isinstance(media, dict):
+        return None
+    return inline(document, media.get("schema") or {}), bool(body.get("required"))
+
+
+def checked_input_schema(
+    properties: dict[str, Any], required: list[str]
+) -> dict[str, Any] | None:
+    """The tool's input schema, or None when too large or not valid."""
+    input_schema: dict[str, Any] = {
+        "type": "object",
+        "properties": properties,
+        "additionalProperties": False,
+    }
+    if required:
+        input_schema["required"] = sorted(required)
+    if len(json.dumps(input_schema).encode()) > SCHEMA_MAX_BYTES:
+        return None
+    try:
+        Draft202012Validator.check_schema(input_schema)
+    except SchemaError:
+        return None
+    return input_schema
+
+
+def resolve(document: dict[str, Any], value: Any, depth: int = 0) -> Any:
+    """Follow a local $ref such as #/components/parameters/Limit."""
+    while isinstance(value, dict) and "$ref" in value:
+        if depth > SCHEMA_MAX_DEPTH:
+            return {}
+        reference = value["$ref"]
+        if not isinstance(reference, str) or not reference.startswith("#/"):
+            return {}
+        target: Any = document
+        for part in reference[2:].split("/"):
+            part = part.replace("~1", "/").replace("~0", "~")
+            target = target.get(part) if isinstance(target, dict) else None
+        value = target
+        depth += 1
+    return value
+
+
+def inline(document: dict[str, Any], schema: Any, depth: int = 0) -> dict[str, Any]:
+    """A schema with every local $ref replaced. Deep or cyclic parts become {}."""
+    if depth > SCHEMA_MAX_DEPTH:
+        return {}
+    schema = resolve(document, schema)
+    if not isinstance(schema, dict):
+        return {}
+    result: dict[str, Any] = {}
+    for key, value in schema.items():
+        if key in {"$ref", "$dynamicRef", "$recursiveRef", "$id", "$schema"}:
+            continue
+        # OpenAPI 3.0 writes these as booleans. JSON Schema wants numbers.
+        if key in {"exclusiveMinimum", "exclusiveMaximum"} and isinstance(value, bool):
+            continue
+        if isinstance(value, dict):
+            result[key] = (
+                {
+                    name: inline(document, item, depth + 1)
+                    for name, item in value.items()
+                }
+                if key in {"properties", "patternProperties", "$defs", "definitions"}
+                else inline(document, value, depth + 1)
+            )
+        elif isinstance(value, list):
+            result[key] = [
+                inline(document, item, depth + 1) if isinstance(item, dict) else item
+                for item in value
+            ]
+        else:
+            result[key] = value
+    return result

secondshift/connections/mcp/__init__.py

@@ -0,0 +1 @@
+"""MCP connections: a shared client plus one module per supported provider."""

secondshift/connections/mcp/base.py

@@ -0,0 +1,139 @@
+import re
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+from secondshift.connections.access import AccessReason
+from secondshift.connections.errors import (
+    ExternalRequestError,
+    ExternalToolError,
+    ToolArgumentError,
+)
+from secondshift.connections.repository_ref import RepositoryRef
+from secondshift.workspaces.models import Workspace
+
+MCPSettings = dict[str, str]
+# The connection borrows each repository's own GitHub source instead of a token.
+SOURCE_AUTH = "source"
+
+
+class MCPError(ExternalToolError):
+    pass
+
+
+class MCPRequestError(MCPError, ExternalRequestError):
+    """A remote failure the agent may report as unavailable evidence."""
+
+
+class CustomMCPToolError(MCPRequestError):
+    """A custom tool failure with a redacted reason for the model."""
+
+    def __init__(self, message: str, reason: str) -> None:
+        super().__init__(message)
+        self.reason = reason
+
+
+class MCPArgumentError(MCPError, ToolArgumentError):
+    """Arguments outside the approved scope. The model may fix them."""
+
+
+@dataclass(frozen=True)
+class MCPSetting:
+    """A value the owner enters when connecting, such as a Sentry organization."""
+
+    key: str
+    label: str
+    # Matched in full. Keep it to simple character classes so browsers agree.
+    pattern: str
+    required: bool = True
+    help: str = ""
+    placeholder: str = ""
+    max_length: int = 64
+    # Extra check after the pattern, such as an outbound URL check.
+    validate: Callable[[str], str] | None = None
+
+
+@dataclass(frozen=True)
+class MCPProvider:
+    """Everything SecondShift trusts about one built-in MCP server.
+
+    The endpoint, tools and scoping rules are server-owned. Repository config
+    and model output can never change them.
+    """
+
+    key: str
+    label: str
+    endpoint_url: str
+    # Only these tools are discovered, approved, and called.
+    tools: tuple[str, ...]
+    # What a job may touch, such as "owner/repo". None means it cannot run.
+    scope: Callable[[MCPSettings, RepositoryRef], str | None]
+    # Forces model arguments into the approved scope, or raises MCPError.
+    scope_arguments: Callable[[str, str, dict[str, Any]], dict[str, Any]]
+    # Completes "Allow these tools to read ...".
+    describe_scope: Callable[[str], str]
+    access_verb: str = "read"
+    scope_label: str = "scope"
+    missing_scope_reason: AccessReason = AccessReason.MCP_SCOPE_UNKNOWN
+    missing_scope_message: str = "This job has no scope for this provider."
+    settings: tuple[MCPSetting, ...] = ()
+    # Appended to endpoint_url, built only from validated settings.
+    endpoint_path: Callable[[MCPSettings], str] = lambda settings: ""
+    auth_scheme: str = "Bearer"
+    # Fixed request headers, sent alongside the token.
+    headers: dict[str, str] = field(default_factory=dict)
+    # Rewrites a discovered tool's input schema so the model sees only the
+    # arguments scope_arguments accepts. Applied before the schema is stored.
+    narrow_schema: Callable[[str, dict[str, Any]], dict[str, Any]] = (
+        lambda tool, schema: schema
+    )
+    # A safe message for a failed tool call, chosen by matching the remote
+    # error text, or None to use the client's fallback.
+    tool_error: Callable[[str, str], str | None] = lambda tool, text: None
+    # Drop tools the server does not mark read-only.
+    require_read_only_hint: bool = False
+    # Custom servers: keep every tool and let the owner choose. Read-only
+    # tools start selected.
+    owner_selects_tools: bool = False
+    # "token" for a pasted bearer token, "oauth" for a browser sign-in,
+    # SOURCE_AUTH for the token of the repository's source connection.
+    auth_methods: tuple[str, ...] = ("token",)
+    # SOURCE_AUTH only. A token for one repository, minted when a tool is called.
+    source_token: Callable[[RepositoryRef], str] | None = None
+    # SOURCE_AUTH only. A token for listing tools when the connection is saved.
+    discovery_token: Callable[[Workspace], str] | None = None
+    # SOURCE_AUTH only. Whether a repository has a source this can borrow.
+    source_reachable: Callable[[RepositoryRef], bool] = lambda repository: False
+    missing_source_message: str = "This repository has no source connection."
+    token_pattern: str = r"\S+"
+    token_hint: str = "Enter a token without whitespace."
+    # Shown when an owner sets up a connection.
+    description: str = ""
+    default_binding: str = ""
+    token_label: str = "Access token"
+    setup_url: str = ""
+    setup_help: str = ""
+
+    def endpoint(self, settings: MCPSettings) -> str:
+        return self.endpoint_url + self.endpoint_path(settings)
+
+    def clean_settings(self, values: dict[str, str]) -> MCPSettings:
+        """Keep only declared settings, each matching its pattern."""
+        known = {setting.key for setting in self.settings}
+        if set(values) - known:
+            raise MCPError(f"{self.label} does not accept these settings.")
+        cleaned: MCPSettings = {}
+        for setting in self.settings:
+            value = values.get(setting.key, "").strip()
+            if not value:
+                if setting.required:
+                    raise MCPError(f"Enter the {setting.label.lower()}.")
+                continue
+            if len(value) > setting.max_length or not re.fullmatch(
+                setting.pattern, value
+            ):
+                raise MCPError(f"The {setting.label.lower()} is not valid.")
+            if setting.validate:
+                value = setting.validate(value)
+            cleaned[setting.key] = value
+        return cleaned