secondshift 0.1.0b12__py3-none-any.whl

secondshift/connections/outbound.py

@@ -0,0 +1,317 @@
+import ipaddress
+import socket
+from collections.abc import AsyncIterator, Iterable
+from urllib.parse import urlsplit
+
+import anyio
+import httpcore
+import httpx
+from django.conf import settings
+
+
+class OutboundUrlError(ValueError):
+    pass
+
+
+def parse_address(address: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address:
+    parsed = ipaddress.ip_address(address.split("%", 1)[0])
+    if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped:
+        return parsed.ipv4_mapped
+    return parsed
+
+
+def public_address(address: str) -> bool:
+    return parse_address(address).is_global
+
+
+def link_local_address(address: str) -> bool:
+    """Link-local hosts include cloud metadata services such as 169.254.169.254.
+
+    These stay blocked even when local addresses are allowed, because they can
+    hand out the host machine's own credentials.
+    """
+    return parse_address(address).is_link_local
+
+
+def literal_address(host: str) -> str | None:
+    """The IP address a host names directly, including the short numeric
+    forms that resolvers accept, such as 2852039166 or 0xa9fea9fe."""
+    host = host.strip("[]").split("%", 1)[0]
+    try:
+        return str(ipaddress.ip_address(host))
+    except ValueError:
+        pass
+    if not host or not all(char in "0123456789abcdefx." for char in host.lower()):
+        return None
+    try:
+        return socket.inet_ntoa(socket.inet_aton(host))
+    except OSError:
+        return None
+
+
+def resolves_to_link_local(host: str, port: int) -> bool:
+    try:
+        infos = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)
+    except (socket.gaierror, UnicodeError):
+        return False
+    return any(link_local_address(str(info[4][0])) for info in infos)
+
+
+def validate_outbound_url(url: str, noun: str = "URL", resolve: bool = True) -> str:
+    """Reject URLs the server must not call. Messages never echo the URL."""
+    url = url.strip()
+    # "Base URLs", not the "Base urls" that str.capitalize would give.
+    plural = f"{noun[:1].upper()}{noun[1:]}s"
+    try:
+        parts = urlsplit(url)
+        port = parts.port
+    except ValueError as error:
+        raise OutboundUrlError(f"Enter a valid {noun}.") from error
+    allow_local = settings.OUTBOUND_ALLOW_LOCAL
+    allowed_schemes = {"https", "http"} if allow_local else {"https"}
+    if parts.scheme not in allowed_schemes:
+        raise OutboundUrlError(
+            f"{plural} must use HTTPS."
+            if not allow_local
+            else f"{plural} must use HTTP or HTTPS."
+        )
+    if not parts.hostname:
+        raise OutboundUrlError(f"Enter a valid {noun}.")
+    if parts.username or parts.password:
+        raise OutboundUrlError(f"{plural} cannot contain a username or password.")
+    literal = literal_address(parts.hostname)
+    link_local = literal is not None and link_local_address(literal)
+    if not link_local and allow_local and resolve:
+        # Local mode skips the public-address check below, but names that
+        # resolve to a metadata service are still refused.
+        link_local = resolves_to_link_local(parts.hostname, port or 443)
+    if link_local:
+        raise OutboundUrlError(
+            f"{plural} cannot point to a link-local address, such as a cloud "
+            "metadata service."
+        )
+    if allow_local or not resolve:
+        return url
+    try:
+        addresses = {
+            info[4][0]
+            for info in socket.getaddrinfo(
+                parts.hostname, port or 443, proto=socket.IPPROTO_TCP
+            )
+        }
+    except (socket.gaierror, UnicodeError) as error:
+        raise OutboundUrlError(f"The {noun} host could not be resolved.") from error
+    if not addresses or not all(public_address(item) for item in addresses):
+        raise OutboundUrlError(f"{plural} must point to a public address.")
+    return url
+
+
+def host_label(url: str) -> str:
+    """A redacted label safe to show: the host only, never the path or query."""
+    parts = urlsplit(url)
+    host = parts.hostname or "unknown host"
+    return f"{host}:{parts.port}" if parts.port else host
+
+
+def vetted_addresses(infos: Iterable[tuple[object, ...]]) -> list[str]:
+    """The addresses from one DNS answer, if every one of them may be called."""
+    addresses = list(dict.fromkeys(str(info[4][0]) for info in infos))  # type: ignore[index]
+    if not addresses:
+        raise httpcore.ConnectError("The host could not be resolved.")
+    if any(link_local_address(address) for address in addresses):
+        raise httpcore.ConnectError("The host resolves to a link-local address.")
+    if not settings.OUTBOUND_ALLOW_LOCAL and not all(
+        public_address(address) for address in addresses
+    ):
+        raise httpcore.ConnectError("The host does not resolve to a public address.")
+    return addresses
+
+
+class PinnedNetworkBackend(httpcore.AsyncNetworkBackend):
+    """Connects only to addresses that pass the outbound check.
+
+    The check and the connection use the same DNS answer, so a host cannot
+    pass validation and then rebind to a private address. TLS still verifies
+    the original hostname, which httpcore sends as SNI.
+    """
+
+    def __init__(self) -> None:
+        self.inner = httpcore.AnyIOBackend()
+
+    async def public_addresses(self, host: str, port: int) -> list[str]:
+        try:
+            infos = await anyio.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)
+        except (OSError, UnicodeError) as error:
+            raise httpcore.ConnectError("The host could not be resolved.") from error
+        return vetted_addresses(infos)
+
+    async def connect_tcp(
+        self,
+        host: str,
+        port: int,
+        timeout: float | None = None,
+        local_address: str | None = None,
+        socket_options: Iterable[httpcore.SOCKET_OPTION] | None = None,
+    ) -> httpcore.AsyncNetworkStream:
+        last_error: Exception | None = None
+        for address in await self.public_addresses(host, port):
+            try:
+                return await self.inner.connect_tcp(
+                    address,
+                    port,
+                    timeout=timeout,
+                    local_address=local_address,
+                    socket_options=socket_options,
+                )
+            except (httpcore.ConnectError, httpcore.ConnectTimeout) as error:
+                last_error = error
+        assert last_error is not None
+        raise last_error
+
+    async def connect_unix_socket(
+        self,
+        path: str,
+        timeout: float | None = None,
+        socket_options: Iterable[httpcore.SOCKET_OPTION] | None = None,
+    ) -> httpcore.AsyncNetworkStream:
+        raise httpcore.ConnectError("Unix sockets are not allowed.")
+
+    async def sleep(self, seconds: float) -> None:
+        await self.inner.sleep(seconds)
+
+
+class PinnedSyncNetworkBackend(httpcore.NetworkBackend):
+    """The blocking twin of PinnedNetworkBackend, for code that is not async."""
+
+    def __init__(self) -> None:
+        self.inner = httpcore.SyncBackend()
+
+    def public_addresses(self, host: str, port: int) -> list[str]:
+        try:
+            infos = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)
+        except (OSError, UnicodeError) as error:
+            raise httpcore.ConnectError("The host could not be resolved.") from error
+        return vetted_addresses(infos)
+
+    def connect_tcp(
+        self,
+        host: str,
+        port: int,
+        timeout: float | None = None,
+        local_address: str | None = None,
+        socket_options: Iterable[httpcore.SOCKET_OPTION] | None = None,
+    ) -> httpcore.NetworkStream:
+        last_error: Exception | None = None
+        for address in self.public_addresses(host, port):
+            try:
+                return self.inner.connect_tcp(
+                    address,
+                    port,
+                    timeout=timeout,
+                    local_address=local_address,
+                    socket_options=socket_options,
+                )
+            except (httpcore.ConnectError, httpcore.ConnectTimeout) as error:
+                last_error = error
+        assert last_error is not None
+        raise last_error
+
+    def connect_unix_socket(
+        self,
+        path: str,
+        timeout: float | None = None,
+        socket_options: Iterable[httpcore.SOCKET_OPTION] | None = None,
+    ) -> httpcore.NetworkStream:
+        raise httpcore.ConnectError("Unix sockets are not allowed.")
+
+    def sleep(self, seconds: float) -> None:
+        self.inner.sleep(seconds)
+
+
+class PinnedTransport(httpx.AsyncHTTPTransport):
+    """An httpx transport that ignores proxy settings and pins vetted addresses."""
+
+    def __init__(self, limits: httpx.Limits = httpx.Limits()) -> None:
+        super().__init__(trust_env=False, limits=limits)
+        # httpx has no public hook for the network backend. Swap the pool it
+        # built for one that uses ours, with the same TLS and pool settings.
+        self._pool = httpcore.AsyncConnectionPool(
+            ssl_context=httpx.create_ssl_context(trust_env=False),
+            max_connections=limits.max_connections,
+            max_keepalive_connections=limits.max_keepalive_connections,
+            keepalive_expiry=limits.keepalive_expiry,
+            network_backend=PinnedNetworkBackend(),
+        )
+
+
+class PinnedSyncTransport(httpx.HTTPTransport):
+    """The blocking twin of PinnedTransport."""
+
+    def __init__(self, limits: httpx.Limits = httpx.Limits()) -> None:
+        super().__init__(trust_env=False, limits=limits)
+        self._pool = httpcore.ConnectionPool(
+            ssl_context=httpx.create_ssl_context(trust_env=False),
+            max_connections=limits.max_connections,
+            max_keepalive_connections=limits.max_keepalive_connections,
+            keepalive_expiry=limits.keepalive_expiry,
+            network_backend=PinnedSyncNetworkBackend(),
+        )
+
+
+# The most a response from a configured service may send.
+RESPONSE_MAX_BYTES = 2 * 1024 * 1024
+
+
+class OutboundResponseError(Exception):
+    """A response SecondShift will not read."""
+
+
+class BoundedStream(httpx.AsyncByteStream):
+    def __init__(self, inner: httpx.AsyncByteStream) -> None:
+        self.inner = inner
+
+    async def __aiter__(self) -> AsyncIterator[bytes]:
+        size = 0
+        async for chunk in self.inner:
+            size += len(chunk)
+            if size > RESPONSE_MAX_BYTES:
+                raise OutboundResponseError("The response exceeded the size limit.")
+            yield chunk
+
+    async def aclose(self) -> None:
+        await self.inner.aclose()
+
+
+class BoundedTransport(PinnedTransport):
+    """A pinned transport that refuses compressed and oversized responses."""
+
+    async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+        response = await super().handle_async_request(request)
+        if response.headers.get("content-encoding", "identity") != "identity":
+            await response.aclose()
+            raise OutboundResponseError("Compressed responses are not supported.")
+        response.stream = BoundedStream(response.stream)
+        return response
+
+
+def outbound_client(headers: dict[str, str]) -> httpx.AsyncClient:
+    """A client for a service a person configured. It connects only to vetted
+    addresses, ignores proxy settings, follows no redirects and reads at most
+    RESPONSE_MAX_BYTES."""
+    return httpx.AsyncClient(
+        transport=BoundedTransport(),
+        headers={**headers, "Accept-Encoding": "identity"},
+        timeout=settings.OUTBOUND_TIMEOUT_SECONDS,
+        follow_redirects=False,
+        trust_env=False,
+    )
+
+
+def outbound_sync_client() -> httpx.Client:
+    """The blocking client for a configured service, pinned the same way."""
+    return httpx.Client(
+        transport=PinnedSyncTransport(),
+        timeout=settings.OUTBOUND_TIMEOUT_SECONDS,
+        follow_redirects=False,
+        trust_env=False,
+    )

secondshift/connections/repository_ref.py

@@ -0,0 +1,19 @@
+from dataclasses import dataclass
+
+from secondshift.connections.models import Connection
+
+
+@dataclass(frozen=True)
+class RepositoryRef:
+    """The repository a job works on, as external tools see it.
+
+    The repositories app builds it, so connections never read repository rows.
+    """
+
+    workspace_id: int
+    # owner/repo on GitHub. None when the repository has no GitHub identity.
+    github_name: str | None
+    # GitHub's id, only for repositories added from GitHub.
+    github_id: int | None = None
+    # The source connection a GitHub repository was added with.
+    source: Connection | None = None

secondshift/connections/schemas.py

@@ -0,0 +1,313 @@
+from datetime import datetime
+from typing import Any, Literal
+
+from ninja import Schema
+from pydantic import Field, SecretStr
+
+from secondshift.connections.dao import has_pending_cleanup
+from secondshift.connections.execution import recent_login_runs
+from secondshift.connections.models import AgentConnection
+from secondshift.workspaces.access import Action, allows
+
+
+class FeatureCheckSchema(Schema):
+    feature: str
+    label: str
+    # None when it cannot be checked without using it.
+    ok: bool | None
+    message: str
+
+
+class ConnectionCheckSchema(Schema):
+    ok: bool
+    message: str
+    checked_at: datetime
+    # Source connections: what each feature can do, and where to fix it.
+    features: list[FeatureCheckSchema] = Field(default_factory=list)
+    fix_url: str = ""
+
+
+class ConnectionUsageSchema(Schema):
+    jobs: int
+    runs_7d: int
+    spend_7d_usd: float
+    spend_partial: bool
+    last_used_at: datetime | None
+
+
+class OfferedToolSchema(Schema):
+    name: str
+    description: str
+    read_only: bool
+
+
+class ConnectionSchema(Schema):
+    id: int
+    provider: str
+    kind: str
+    binding_key: str
+    display_name: str
+    status: str
+    expires_at: datetime | None
+    available_tools: list[str]
+    target: str
+    allowed_models: list[str]
+    default_model: str
+    is_default: bool
+    available_models: list[str]
+    last_check: ConnectionCheckSchema | None = None
+    usage: ConnectionUsageSchema | None = None
+    # MCP only: the organization or project an owner chose. Never secret.
+    settings: dict[str, str] = Field(default_factory=dict)
+    # MCP only: "oauth" for a browser sign-in, "token" for a pasted token.
+    auth: str = ""
+    scopes: list[str] = Field(default_factory=list)
+    requested_scopes: list[str] = Field(default_factory=list)
+    # Custom MCP servers: every tool the server offers, for the owner to choose.
+    offered_tools: list[OfferedToolSchema] = Field(default_factory=list)
+    # Capability connections: whether each job's request waits for an owner.
+    require_approval: bool = False
+    # GitHub sources: whether pull requests, issues and commits name SecondShift.
+    attribution: bool = False
+
+
+class AttributionSchema(Schema):
+    attribution: bool
+
+
+class WebhookConnectionCreateSchema(Schema):
+    binding_key: str = Field(
+        min_length=1, max_length=120, pattern=r"^[a-z0-9]+(?:-[a-z0-9]+)*$"
+    )
+    display_name: str = Field(default="", max_length=200)
+    url: str = Field(min_length=1, max_length=2000)
+    require_approval: bool = False
+
+
+class WebhookConnectionUpdateSchema(Schema):
+    display_name: str | None = Field(default=None, max_length=200)
+    url: str | None = Field(default=None, max_length=2000)
+    require_approval: bool | None = None
+
+
+class GitHubTokenConnectSchema(Schema):
+    token: str = Field(min_length=1, max_length=2000)
+    # Replace the token of this connection instead of adding one.
+    connection_id: int | None = None
+
+
+class MCPConnectionCreateSchema(Schema):
+    provider: str = Field(min_length=1, max_length=40)
+    binding_key: str = Field(
+        min_length=1, max_length=120, pattern=r"^[a-z0-9]+(?:-[a-z0-9]+)*$"
+    )
+    display_name: str = Field(default="", max_length=200)
+    # Empty when auth is "source": the connection borrows repository sources.
+    token: str = Field(default="", max_length=2000)
+    auth: str = Field(default="token", pattern=r"^(token|source)$")
+    settings: dict[str, str] = Field(default_factory=dict, max_length=10)
+    require_approval: bool = False
+
+
+class MCPConnectionUpdateSchema(Schema):
+    display_name: str | None = Field(default=None, max_length=200)
+    token: str | None = Field(default=None, max_length=2000)
+    settings: dict[str, str] | None = Field(default=None, max_length=10)
+    tools: list[str] | None = Field(default=None, max_length=200)
+    require_approval: bool | None = None
+
+
+class APISpecSchema(Schema):
+    spec_url: str = Field(default="", max_length=2000)
+    spec_text: str = Field(default="", max_length=5 * 1024 * 1024)
+
+
+class APISpecSummarySchema(Schema):
+    title: str
+    servers: list[str]
+    operations: int
+    read_only: int
+
+
+class APIConnectionCreateSchema(APISpecSchema):
+    binding_key: str = Field(
+        min_length=1, max_length=120, pattern=r"^[a-z0-9]+(?:-[a-z0-9]+)*$"
+    )
+    display_name: str = Field(default="", max_length=200)
+    base_url: str = Field(min_length=1, max_length=200)
+    auth: str = Field(default="bearer", max_length=20)
+    header: str = Field(default="", max_length=64)
+    token: str = Field(min_length=1, max_length=2000)
+    require_approval: bool = False
+
+
+class APIConnectionUpdateSchema(Schema):
+    display_name: str | None = Field(default=None, max_length=200)
+    token: str | None = Field(default=None, max_length=2000)
+    base_url: str | None = Field(default=None, max_length=200)
+    operations: list[str] | None = Field(default=None, max_length=500)
+    require_approval: bool | None = None
+
+
+class MCPInspectSchema(Schema):
+    provider: str = Field(min_length=1, max_length=40)
+    settings: dict[str, str] = Field(default_factory=dict, max_length=10)
+
+
+class MCPInspectResultSchema(Schema):
+    oauth: bool
+    scopes: list[str]
+
+
+class MCPSignInSchema(Schema):
+    provider: str = Field(min_length=1, max_length=40)
+    binding_key: str = Field(
+        default="", max_length=120, pattern=r"^([a-z0-9]+(?:-[a-z0-9]+)*)?$"
+    )
+    display_name: str = Field(default="", max_length=200)
+    settings: dict[str, str] = Field(default_factory=dict, max_length=10)
+    scopes: list[str] = Field(default_factory=list, max_length=20)
+    # New connections only. A reconnect keeps the connection's setting.
+    require_approval: bool = False
+    # Reconnect this connection instead of creating one.
+    connection_id: int | None = None
+
+
+class MCPSignInStartSchema(Schema):
+    authorize_url: str
+
+
+class MCPSettingSchema(Schema):
+    key: str
+    label: str
+    pattern: str
+    required: bool
+    help: str
+    placeholder: str
+
+
+class MCPProviderSchema(Schema):
+    key: str
+    label: str
+    description: str
+    default_binding: str
+    token_label: str
+    token_hint: str
+    setup_url: str
+    setup_help: str
+    settings: list[MCPSettingSchema]
+    tools: list[str]
+    auth_methods: list[str]
+    owner_selects_tools: bool
+
+
+class ModelConnectionCreateSchema(Schema):
+    provider: Literal["anthropic", "openai", "openrouter", "openai_compatible"]
+    display_name: str = Field(default="", max_length=200)
+    api_key: str = Field(default="", max_length=1000)
+    base_url: str = Field(default="", max_length=2000)
+    allowed_models: list[str] = Field(default_factory=list, max_length=100)
+    default_model: str = Field(default="", max_length=200)
+    is_default: bool | None = None
+
+
+class ModelConnectionUpdateSchema(Schema):
+    display_name: str | None = Field(default=None, max_length=200)
+    api_key: str | None = Field(default=None, max_length=1000)
+    base_url: str | None = Field(default=None, max_length=2000)
+    allowed_models: list[str] | None = Field(default=None, max_length=100)
+    default_model: str | None = Field(default=None, max_length=200)
+    is_default: bool | None = None
+
+
+class ConnectionTestSchema(Schema):
+    ok: bool
+    status_code: int | None
+    message: str
+    models: list[str] = Field(default_factory=list)
+
+
+class WebhookSampleSchema(Schema):
+    # A run report as a webhook receives it, built from made-up records.
+    payload: dict[str, Any]
+    # How many characters of the report the summary field can hold.
+    summary_max_chars: int
+
+
+class AgentConnectionSchema(Schema):
+    id: int
+    backend: str
+    display_name: str
+    connected: bool
+    owner_id: int
+    updated_at: datetime
+    max_runs_per_day: int | None = None
+    # Runs in the last 24 hours, the window the cap counts over.
+    recent_runs: int = 0
+    can_manage: bool = False
+    last_check: dict[str, Any] | None = None
+    # The runner still holds a cached copy of an earlier login.
+    runner_cleanup_pending: bool = False
+
+    @staticmethod
+    def resolve_recent_runs(obj: AgentConnection) -> int:
+        # A list counts every login at once and attaches the counts.
+        counted = getattr(obj, "recent_runs", None)
+        if counted is None:
+            counted = recent_login_runs(obj.workspace_id)[obj.pk]
+        return counted
+
+    @staticmethod
+    def resolve_runner_cleanup_pending(obj: AgentConnection) -> bool:
+        return has_pending_cleanup(obj)
+
+    @staticmethod
+    def resolve_can_manage(obj: AgentConnection, context: dict[str, Any]) -> bool:
+        request = context["request"]
+        return obj.owner_id == request.user.pk and allows(
+            request, Action.EDIT_CONNECTIONS
+        )
+
+
+class AgentRunLimitInput(Schema):
+    # Empty removes the cap.
+    max_runs_per_day: int | None = Field(default=None, ge=1, le=1000)
+
+
+class AgentConnectionInput(Schema):
+    backend: str
+    display_name: str = Field(default="", max_length=200)
+    auth: SecretStr = Field(max_length=32000)
+
+
+class ExecutionChoiceSchema(Schema):
+    key: str
+    label: str
+    disabled: bool = False
+    # Why this choice cannot run jobs, or empty when it can.
+    problem: str = ""
+
+
+class ExecutionStatusSchema(Schema):
+    label: str
+    backend: str
+    problem: str
+    # False until an owner picks a model provider or a login.
+    chosen: bool
+    # Nothing to choose yet: no model provider and no login saved.
+    needs_setup: bool
+    fix_connection_id: int | None
+
+
+class ExecutionConfigurationSchema(Schema):
+    enabled: bool
+    choices: list[ExecutionChoiceSchema]
+    selected: str
+    label: str
+    backend: str
+    ready: bool
+    problem: str
+
+
+class ExecutionPreferenceSchema(Schema):
+    key: str = Field(max_length=100)