secondshift 0.1.0b12__py3-none-any.whl

secondshift/jobs/dao.py

@@ -0,0 +1,258 @@
+"""Queries and writes on jobs, capability requests and grants."""
+
+from collections.abc import Collection, Iterable
+from datetime import datetime
+from typing import Any
+
+from django.db.models import Count, QuerySet
+
+from secondshift.connections.models import Connection
+from secondshift.jobs.models import CapabilityRequest, Job, JobGrant
+from secondshift.repositories.models import Repository
+from secondshift.workspaces.models import Workspace
+
+# Fields a grant review changes. Bulk updates save only these.
+GRANT_REVIEW_FIELDS = [
+    "connection",
+    "status",
+    "allowed_tools",
+    "reviewed_by",
+    "reviewed_at",
+    "updated_at",
+]
+
+
+# Jobs
+
+
+def create_job(**fields: Any) -> Job:
+    return Job.objects.create(**fields)
+
+
+def create_jobs(jobs: list[Job]) -> None:
+    Job.objects.bulk_create(jobs)
+
+
+def active_jobs(
+    workspace_id: int, repository_id: int | None = None, enabled: bool | None = None
+) -> QuerySet[Job]:
+    """The workspace's jobs whose repository was not removed, with the repository."""
+    jobs = Job.objects.filter(
+        workspace_id=workspace_id, repository__removed_at__isnull=True
+    ).select_related("repository")
+    if repository_id is not None:
+        jobs = jobs.filter(repository_id=repository_id)
+    if enabled is not None:
+        jobs = jobs.filter(enabled=enabled)
+    return jobs
+
+
+def active_jobs_with_grants(
+    workspace_id: int, repository_id: int | None = None, enabled: bool | None = None
+) -> QuerySet[Job]:
+    """active_jobs with each capability request's grant and connection."""
+    return active_jobs(workspace_id, repository_id, enabled).prefetch_related(
+        "capability_requests__grant__connection"
+    )
+
+
+def enabled_job_count(workspace_id: int) -> int:
+    return active_jobs(workspace_id, enabled=True).count()
+
+
+def find_active_job(workspace: Workspace | None, job_id: int) -> Job | None:
+    """The job, with its repository, workspace, requests, grants and connections."""
+    return (
+        Job.objects.select_related("repository", "workspace")
+        .prefetch_related("capability_requests__grant__connection")
+        .filter(pk=job_id, workspace=workspace, repository__removed_at__isnull=True)
+        .first()
+    )
+
+
+def jobs_by_id(job_ids: Iterable[int]) -> dict[int, Job]:
+    """The jobs with these ids, with their repositories."""
+    return Job.objects.select_related("repository").in_bulk(list(job_ids))
+
+
+def schedulable_jobs() -> QuerySet[Job]:
+    """Enabled jobs with a schedule in enabled repositories, oldest first."""
+    return (
+        Job.objects.filter(enabled=True, repository__enabled=True)
+        .exclude(cron="")
+        .order_by("pk")
+    )
+
+
+def scheduled_job(job_id: int) -> Job | None:
+    """The job with its repository while both are enabled."""
+    return (
+        Job.objects.select_related("repository")
+        .filter(pk=job_id, enabled=True, repository__enabled=True)
+        .first()
+    )
+
+
+def save_next_runs(jobs: list[Job]) -> None:
+    Job.objects.bulk_update(jobs, ["next_run_at"])
+
+
+def lock_job(job: Job) -> None:
+    """Lock the job for this transaction, so two starts wait for each other."""
+    Job.objects.select_for_update().filter(pk=job.pk).first()
+
+
+def retire_missing_jobs(
+    repository: Repository, keys: Collection[str], now: datetime
+) -> None:
+    """Disable and mark removed the repository's jobs the manifest no longer names."""
+    repository.jobs.exclude(key__in=keys).filter(removed_at__isnull=True).update(
+        enabled=False, removed_at=now
+    )
+
+
+def enabled_repository_jobs(repository: Repository) -> list[Job]:
+    return list(repository.jobs.filter(enabled=True))
+
+
+def disable_repository_jobs(repository: Repository) -> None:
+    repository.jobs.update(enabled=False)
+
+
+def repository_jobs_with_grants(repository: Repository) -> QuerySet[Job]:
+    return repository.jobs.prefetch_related("capability_requests__grant")
+
+
+# Capability requests
+
+
+def create_capability_requests(requests: list[CapabilityRequest]) -> None:
+    CapabilityRequest.objects.bulk_create(requests)
+
+
+def delete_capability_requests(request_ids: list[int]) -> None:
+    CapabilityRequest.objects.filter(pk__in=request_ids).delete()
+
+
+def repository_requests(repository: Repository) -> QuerySet[CapabilityRequest]:
+    """Every request of the repository's jobs, with its grant."""
+    return CapabilityRequest.objects.filter(job__repository=repository).select_related(
+        "grant"
+    )
+
+
+def job_requests(job: Job) -> QuerySet[CapabilityRequest]:
+    return CapabilityRequest.objects.filter(job=job)
+
+
+def job_requests_with_grants(job: Job) -> QuerySet[CapabilityRequest]:
+    """The job's requests with their grants and granted connections."""
+    return job.capability_requests.select_related("grant__connection")
+
+
+def find_request(
+    repository: Repository, job_key: str, binding: str, kind: str
+) -> CapabilityRequest | None:
+    """The request of the repository's job for this binding, with its grant."""
+    return (
+        CapabilityRequest.objects.filter(
+            job__repository=repository, job__key=job_key, binding=binding, kind=kind
+        )
+        .select_related("grant")
+        .first()
+    )
+
+
+def workspace_requests(workspace: Workspace) -> QuerySet[CapabilityRequest]:
+    """Requests of the workspace's active jobs, by binding then job name."""
+    return (
+        CapabilityRequest.objects.filter(
+            job__workspace=workspace, job__repository__removed_at__isnull=True
+        )
+        .select_related("job__repository", "grant")
+        .order_by("binding", "job__name")
+    )
+
+
+def active_requests() -> QuerySet[CapabilityRequest]:
+    """Requests of jobs whose repository was not removed, with grant and job."""
+    return CapabilityRequest.objects.filter(
+        job__repository__removed_at__isnull=True
+    ).select_related("grant", "job__repository")
+
+
+def repository_active_requests(repository: Repository) -> list[CapabilityRequest]:
+    return list(active_requests().filter(job__repository=repository))
+
+
+def binding_active_requests(workspace_id: int, binding: str) -> list[CapabilityRequest]:
+    """The workspace's active requests for this binding."""
+    return list(
+        active_requests().filter(job__workspace_id=workspace_id, binding=binding)
+    )
+
+
+# Grants
+
+
+def create_grants(grants: list[JobGrant]) -> None:
+    JobGrant.objects.bulk_create(grants)
+
+
+def save_grant_reviews(grants: list[JobGrant]) -> None:
+    JobGrant.objects.bulk_update(grants, GRANT_REVIEW_FIELDS)
+
+
+def find_grant(request: CapabilityRequest) -> JobGrant | None:
+    return JobGrant.objects.filter(capability_request=request).first()
+
+
+def approved_tool_grant(
+    job: Job, kind: str, binding: str, connection_id: int | None
+) -> JobGrant | None:
+    """The job's approved grant for this binding and connection, if any."""
+    return (
+        JobGrant.objects.select_related("connection", "capability_request")
+        .filter(
+            capability_request__job=job,
+            capability_request__kind=kind,
+            capability_request__binding=binding,
+            connection_id=connection_id,
+            status=JobGrant.Status.APPROVED,
+        )
+        .first()
+    )
+
+
+def approved_jobs_per_connection(workspace_id: int) -> dict[int, int]:
+    """How many jobs each of the workspace's connections is approved for."""
+    return {
+        row["connection_id"]: row["jobs"]
+        for row in JobGrant.objects.filter(
+            connection__workspace_id=workspace_id, status=JobGrant.Status.APPROVED
+        )
+        .values("connection_id")
+        .annotate(jobs=Count("capability_request__job", distinct=True))
+        .order_by()
+    }
+
+
+def reopen_github_mcp_grants(repository: Repository) -> None:
+    """Send the repository's approved GitHub MCP grants back for review."""
+    JobGrant.objects.filter(
+        capability_request__job__repository=repository,
+        capability_request__kind=Connection.Kind.MCP,
+        connection__provider=Connection.Provider.GITHUB,
+        status=JobGrant.Status.APPROVED,
+    ).update(status=JobGrant.Status.PENDING, reviewed_by=None, reviewed_at=None)
+
+
+def release_connection_grants(connection: Connection) -> None:
+    """Return the connection's grants to pending, without a connection."""
+    connection.job_grants.update(
+        connection=None,
+        status=JobGrant.Status.PENDING,
+        allowed_tools=[],
+        reviewed_by=None,
+        reviewed_at=None,
+    )

secondshift/jobs/drafts.py

@@ -0,0 +1,112 @@
+"""Check a draft .secondshift.yaml against a workspace, before it is committed.
+
+Nothing is saved. Skill files are not read, so a missing skill shows up only
+when the committed manifest is synced. A binding is ready only when the same
+job already has an approved grant that covers what the draft asks for.
+Syncing never grants access, so anything new waits for an owner.
+"""
+
+from dataclasses import dataclass, field
+
+import yaml
+
+from secondshift.config.constants import MANIFEST_FILE
+from secondshift.connections.dao import connections_by_binding
+from secondshift.connections.models import Connection
+from secondshift.jobs.dao import find_request
+from secondshift.jobs.manifest import JobManifest, ManifestError, parse_manifest
+from secondshift.jobs.models import CapabilityRequest, JobGrant
+from secondshift.jobs.needs import NeedStatus, connection_problem
+from secondshift.repositories.models import Repository
+from secondshift.workspaces.models import Workspace
+
+NEW_ACCESS_DETAIL = "Connected. An owner approves it after the manifest is synced."
+APPROVED_DETAIL = "Connected and already approved for this job."
+
+
+@dataclass(frozen=True)
+class DraftBinding:
+    binding: str
+    kind: str
+    required: bool
+    requested_tools: list[str]
+    status: NeedStatus
+    detail: str
+
+
+@dataclass(frozen=True)
+class DraftJob:
+    key: str
+    name: str
+    schedule: str | None
+    bindings: list[DraftBinding] = field(default_factory=list)
+
+
+def requested(job: JobManifest) -> list[tuple[str, str, bool, list[str]]]:
+    """Each binding the job asks for: binding, kind, required and tools."""
+    return [
+        *(
+            (item.binding, CapabilityRequest.Kind.MCP, item.required, item.tools)
+            for item in job.requires.mcps
+        ),
+        *(
+            (item.binding, CapabilityRequest.Kind.API, item.required, item.tools)
+            for item in job.requires.apis
+        ),
+        *(
+            (item.binding, CapabilityRequest.Kind.NOTIFICATION, item.required, [])
+            for item in job.notify
+        ),
+    ]
+
+
+def approved_grant(
+    repository: Repository | None,
+    key: str,
+    binding: str,
+    kind: str,
+    connection: Connection,
+    tools: set[str],
+) -> bool:
+    if repository is None:
+        return False
+    request = find_request(repository, key, binding, kind)
+    grant = getattr(request, "grant", None) if request else None
+    return bool(
+        grant
+        and grant.status == JobGrant.Status.APPROVED
+        and grant.connection_id == connection.pk
+        and tools <= set(grant.allowed_tools)
+    )
+
+
+def check_draft(
+    workspace: Workspace, text: str, repository: Repository | None = None
+) -> list[DraftJob]:
+    """Each draft job's bindings and their status. Raises ManifestError."""
+    try:
+        manifest = parse_manifest(text)
+    except (ValueError, yaml.YAMLError) as error:
+        # The same wording a sync shows for the committed file.
+        raise ManifestError(f"Invalid {MANIFEST_FILE}: {error}") from error
+    connections = connections_by_binding(workspace.pk)
+    repositories = [repository] if repository else []
+    jobs = []
+    for key, job in manifest.jobs.items():
+        bindings = []
+        for binding, kind, required, tools in requested(job):
+            connection = connections.get(binding)
+            problem = connection_problem(kind, connection, set(tools), repositories)
+            if problem:
+                status, detail = problem
+            elif connection and approved_grant(
+                repository, key, binding, kind, connection, set(tools)
+            ):
+                status, detail = "ready", APPROVED_DETAIL
+            else:
+                status, detail = "needs_approval", NEW_ACCESS_DETAIL
+            bindings.append(
+                DraftBinding(binding, kind, required, sorted(tools), status, detail)
+            )
+        jobs.append(DraftJob(key, job.name, job.schedule, bindings))
+    return jobs

secondshift/jobs/grants.py

@@ -0,0 +1,116 @@
+"""An owner's decision on the access a job requests."""
+
+from django.contrib.auth.models import AbstractBaseUser
+from django.db import transaction
+from django.utils import timezone
+
+from secondshift.connections.external import external_access
+from secondshift.connections.models import Connection
+from secondshift.connections.services import ConnectionConflict
+from secondshift.jobs.dao import find_grant, release_connection_grants
+from secondshift.jobs.models import CapabilityRequest, JobGrant
+from secondshift.jobs.tool_rules import (
+    ToolRuleError,
+    ToolRules,
+    kept_rules,
+    validate_rules,
+)
+from secondshift.repositories.dao import has_repositories
+from secondshift.repositories.github import repository_ref
+
+
+class GrantError(Exception):
+    def __init__(self, status: int, message: str) -> None:
+        super().__init__(message)
+        # The HTTP status the API answers with.
+        self.status = status
+
+
+def check_connection(
+    capability: CapabilityRequest, connection: Connection, approving: bool
+) -> None:
+    """The connection must fit the request, be connected, and reach the repository."""
+    if (
+        connection.kind != capability.kind
+        or connection.binding_key != capability.binding
+    ):
+        raise GrantError(422, "Connection kind and binding must match capability.")
+    if not connection.is_connected:
+        raise GrantError(409, "The selected connection is not connected.")
+    if not approving:
+        return
+    access = external_access(connection, repository_ref(capability.job.repository))
+    if access and not access.ok:
+        raise GrantError(409, access.text)
+
+
+def granted_tools(
+    capability: CapabilityRequest,
+    decision: str,
+    connection: Connection | None,
+    chosen: list[str] | None,
+) -> set[str]:
+    """The tools the decision allows. Approving means every requested tool."""
+    requested = set(capability.requested_tools)
+    allowed = set(capability.requested_tools if chosen is None else chosen)
+    if decision == JobGrant.Status.DENIED:
+        return set()
+    if allowed - requested:
+        raise GrantError(422, "Allowed tools must be requested by the job.")
+    if decision == JobGrant.Status.APPROVED:
+        if allowed != requested:
+            raise GrantError(
+                422, "Approve every requested tool or deny the capability."
+            )
+        if connection and requested - set(connection.available_tools):
+            raise GrantError(409, "The connection lacks a requested tool.")
+    return allowed
+
+
+@transaction.atomic
+def decide_grant(
+    capability: CapabilityRequest,
+    decision: str,
+    connection: Connection | None,
+    chosen_tools: list[str] | None,
+    reviewer: AbstractBaseUser,
+    tool_rules: ToolRules | None = None,
+) -> JobGrant:
+    """`tool_rules` None keeps the rules for tools that stay approved."""
+    approving = decision == JobGrant.Status.APPROVED
+    if connection is not None:
+        check_connection(capability, connection, approving)
+    elif approving:
+        raise GrantError(422, "Choose a connection before approving the capability.")
+    allowed = granted_tools(capability, decision, connection, chosen_tools)
+
+    grant = find_grant(capability) or JobGrant(capability_request=capability)
+    rules = kept_rules(grant.tool_rules if tool_rules is None else tool_rules, allowed)
+    if tool_rules is not None and set(tool_rules) - allowed:
+        raise GrantError(422, "Rules can only apply to approved tools.")
+    if rules and connection is not None:
+        try:
+            validate_rules(rules, connection.tool_definitions, sorted(allowed))
+        except ToolRuleError as error:
+            raise GrantError(422, str(error)) from error
+    grant.connection = connection
+    grant.status = decision
+    grant.allowed_tools = sorted(allowed)
+    grant.tool_rules = rules
+    grant.reviewed_by = reviewer
+    grant.reviewed_at = timezone.now()
+    grant.full_clean()
+    grant.save()
+    return grant
+
+
+@transaction.atomic
+def delete_connection(connection: Connection) -> None:
+    if connection.kind == Connection.Kind.SOURCE and has_repositories(connection):
+        raise ConnectionConflict(
+            "This installation has repository history. Disconnect it instead."
+        )
+    # Grants are review records: they return to pending instead of disappearing,
+    # so the jobs show up again under "Needed by your jobs".
+    release_connection_grants(connection)
+    connection.delete()

secondshift/jobs/limits.py

@@ -0,0 +1,26 @@
+"""The run limits of one job. See secondshift.workspaces.limits."""
+
+from secondshift.jobs.models import Job
+from secondshift.workspaces.limits import (
+    LimitView,
+    RunLimit,
+    capped,
+    limit_views,
+    workspace_value,
+)
+
+
+def job_value(job: Job, limit: RunLimit) -> int:
+    chosen = getattr(job, limit)
+    if chosen is not None:
+        return capped(limit, chosen)
+    return workspace_value(job.workspace, limit)
+
+
+def effective_limits(job: Job) -> dict[str, int]:
+    """The limits a run of this job gets, for its execution plan."""
+    return {str(limit): job_value(job, limit) for limit in RunLimit}
+
+
+def job_limit_views(job: Job) -> list[LimitView]:
+    return limit_views(job, lambda limit: workspace_value(job.workspace, limit))