secondshift 0.1.0b12__py3-none-any.whl

secondshift/jobs/planning.py

@@ -0,0 +1,386 @@
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+
+from django.conf import settings
+
+from secondshift.config.constants import (
+    COMMIT_SHA_PATTERN,
+)
+from secondshift.connections.dao import connection_for_binding, connections_by_binding
+from secondshift.connections.execution import workspace_execution, workspace_model
+from secondshift.connections.external import EXTERNAL_KINDS
+from secondshift.connections.model_providers import provider_family
+from secondshift.connections.models import Connection
+from secondshift.jobs.automatic_grants import approves_automatically
+from secondshift.jobs.capabilities import RequestedCapability, manifest_capabilities
+from secondshift.jobs.dao import job_requests_with_grants
+from secondshift.jobs.limits import effective_limits
+from secondshift.jobs.manifest import JobManifest, TaskManifest, manifest_hash
+from secondshift.jobs.models import Job, JobGrant
+from secondshift.jobs.plan import (
+    AgentPlan,
+    CapabilityPlan,
+    ExecutionChoice,
+    ExecutionPlan,
+    IssuesPlan,
+    JobPlan,
+    Missing,
+    ModelPlan,
+    PlanLimits,
+    PlanReason,
+    PullRequestPlan,
+    RepositoryPlan,
+    RunnerPlan,
+    SkillSourcePlan,
+)
+from secondshift.jobs.tool_policy import plan_tools
+from secondshift.jobs.tool_rules import ToolRules, broken_rules, kept_rules
+from secondshift.repositories.github import repository_ref
+from secondshift.repositories.models import Repository
+from secondshift.workspaces.limits import RunLimit, workspace_limits
+from secondshift.workspaces.models import Workspace
+
+
+def model_plan(workspace: Workspace) -> ModelPlan | None:
+    """The workspace's default model provider, frozen into the run's plan."""
+    connection = workspace_model(workspace)
+    if connection is None:
+        return None
+    return ModelPlan(
+        connection_id=connection.pk,
+        connection_name=connection.display_name,
+        provider=connection.provider,
+        family=provider_family(connection),
+        default_model=connection.default_model,
+        allowed_models=tuple(connection.allowed_models),
+    )
+
+
+@dataclass(frozen=True)
+class CapabilityApproval:
+    """Server-resolved policy. Never accepted from repository or client input."""
+
+    status: str
+    connection: Connection | None
+    allowed_tools: tuple[str, ...]
+    tool_rules: ToolRules = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class PlannedCapability:
+    request: RequestedCapability
+    approval: CapabilityApproval | None = None
+
+
+@dataclass(frozen=True)
+class ExecutionDefinition:
+    key: str
+    skill_path: str
+    config_hash: str
+    skill_source: SkillSourcePlan | None = None
+
+
+GRANT_REASONS = {
+    JobGrant.Status.PENDING: PlanReason.GRANT_PENDING,
+    JobGrant.Status.DENIED: PlanReason.GRANT_DENIED,
+}
+CONNECTION_REASONS = {
+    Connection.Status.EXPIRED: PlanReason.CONNECTION_EXPIRED,
+    Connection.Status.REVOKED: PlanReason.CONNECTION_REVOKED,
+}
+WAITING_REASONS = {
+    PlanReason.NO_GRANT,
+    PlanReason.GRANT_PENDING,
+    PlanReason.TOOLS_NOT_GRANTED,
+}
+
+
+def blocking_reasons(
+    workspace: Workspace,
+    request: RequestedCapability,
+    grant: CapabilityApproval | None,
+    connection: Connection | None,
+) -> list[str]:
+    """Why a grant cannot be used for this request. Empty means it can."""
+    requested = set(request.requested_tools)
+    reasons: list[str] = []
+    if not grant:
+        reasons.append(PlanReason.NO_GRANT)
+    elif grant.status != JobGrant.Status.APPROVED:
+        reasons.append(GRANT_REASONS[grant.status])
+    if connection:
+        reasons.extend(connection_reasons(workspace, request, connection))
+    if grant and grant.status == JobGrant.Status.APPROVED and connection is None:
+        reasons.append(PlanReason.CONNECTION_MISSING)
+    if grant and requested - set(grant.allowed_tools):
+        reasons.append(PlanReason.TOOLS_NOT_GRANTED)
+    if connection and requested - set(connection.available_tools):
+        reasons.append(PlanReason.TOOLS_UNAVAILABLE)
+    return reasons
+
+
+def connection_reasons(
+    workspace: Workspace, request: RequestedCapability, connection: Connection
+) -> list[str]:
+    reasons = []
+    if connection.current_status != Connection.Status.CONNECTED:
+        reasons.append(CONNECTION_REASONS[connection.current_status])
+    if connection.workspace_id != workspace.pk:
+        reasons.append(PlanReason.CONNECTION_WORKSPACE_MISMATCH)
+    if connection.kind != request.kind:
+        reasons.append(PlanReason.CONNECTION_KIND_MISMATCH)
+    if connection.binding_key != request.binding:
+        reasons.append(PlanReason.CONNECTION_BINDING_MISMATCH)
+    return reasons
+
+
+def rules_broken(grant: CapabilityApproval, connection: Connection) -> bool:
+    """The connection's schemas changed under the owner's rules. Fail closed."""
+    definitions = connection.tool_definitions
+    return bool(grant.tool_rules and broken_rules(grant.tool_rules, definitions))
+
+
+def missing_reason(
+    workspace: Workspace,
+    request: RequestedCapability,
+    grant: CapabilityApproval | None,
+    reasons: list[str],
+) -> Missing:
+    if grant and grant.status == JobGrant.Status.DENIED:
+        return Missing.DENIED
+    connection = (grant.connection if grant else None) or connection_for_binding(
+        workspace.pk, request.binding
+    )
+    if connection is None or connection.status != Connection.Status.CONNECTED:
+        return Missing.NOT_CONNECTED
+    if WAITING_REASONS & set(reasons):
+        return Missing.WAITING_FOR_APPROVAL
+    return Missing.UNAVAILABLE
+
+
+def capability_plan(
+    workspace: Workspace, repository: Repository, capability: PlannedCapability
+) -> CapabilityPlan:
+    """One requested capability, with the tools the run may use."""
+    request = capability.request
+    grant = capability.approval
+    connection = grant.connection if grant else None
+    requested = set(request.requested_tools)
+    allowed = set(grant.allowed_tools) if grant else set()
+    available = set(connection.available_tools) if connection else set()
+    effective = requested & allowed & available
+    reasons = blocking_reasons(workspace, request, grant, connection)
+    if grant and connection and rules_broken(grant, connection):
+        reasons.append(PlanReason.TOOL_RULES_BROKEN)
+
+    adapter = EXTERNAL_KINDS.get(request.kind)
+    access = (
+        adapter.access(connection, repository_ref(repository))
+        if adapter and connection and connection.kind == request.kind
+        else None
+    )
+    if access and not access.ok:
+        reasons.append(access.reason)
+
+    authorized = not reasons
+    frozen = bool(adapter and connection and access and access.scope and authorized)
+    return CapabilityPlan(
+        binding=request.binding,
+        kind=request.kind,
+        required=request.required,
+        requested_tools=tuple(sorted(requested)),
+        connection_id=connection.pk if connection else None,
+        connection_name=connection.display_name if connection else None,
+        effective_tools=tuple(sorted(effective)) if authorized else (),
+        authorized=authorized,
+        reasons=tuple(reasons),
+        missing=(
+            None if authorized else missing_reason(workspace, request, grant, reasons)
+        ),
+        external=(
+            adapter.freeze(connection, access.scope, effective) if frozen else None
+        ),
+        tool_rules=kept_rules(grant.tool_rules, effective) if frozen else {},
+    )
+
+
+def compile_execution_plan(
+    workspace: Workspace,
+    repository: Repository,
+    definition: ExecutionDefinition,
+    resolved_capabilities: Sequence[PlannedCapability],
+    limits: dict[str, int],
+    commit_sha: str,
+    pull_request: PullRequestPlan | None = None,
+    issues: IssuesPlan | None = None,
+    subagents: int = 0,
+) -> ExecutionPlan:
+    """Compile server-resolved policy into the plan a run keeps."""
+    if repository.workspace_id != workspace.pk:
+        raise ValueError("Repository and execution policy must share a workspace.")
+    capabilities = tuple(
+        capability_plan(workspace, repository, item) for item in resolved_capabilities
+    )
+    selection = workspace_execution(workspace)
+    connection = selection.connection
+    ready = selection.ready and all(
+        item.authorized for item in capabilities if item.required
+    )
+    agent = AgentPlan(
+        limits=PlanLimits(
+            max_turns=limits[RunLimit.MAX_TURNS],
+            max_tool_calls=limits[RunLimit.MAX_TOOL_CALLS],
+            max_tool_result_chars=settings.AGENT_MAX_TOOL_RESULT_CHARS,
+            token_budget=limits[RunLimit.TOKEN_BUDGET],
+            timeout_seconds=limits[RunLimit.TIMEOUT_SECONDS],
+        ),
+        tools=tuple(tool.name for tool in plan_tools(ready, capabilities)),
+        # Backends that cannot start sub-agents get none. Lowering the server
+        # maximum applies before the manifest is synced again.
+        subagents=(
+            min(subagents, settings.RUN_SUBAGENTS_MAX)
+            if selection.backend.starts_subagents
+            else 0
+        ),
+    )
+    return ExecutionPlan(
+        execution=ExecutionChoice(
+            backend=selection.backend,
+            label=selection.label,
+            ready=selection.ready,
+            problem=selection.problem,
+            connection_id=connection.pk if connection else None,
+            credential_key=str(connection.credential_key) if connection else None,
+            owner_id=connection.owner_id if connection else None,
+        ),
+        ready=ready,
+        agent=agent,
+        job=JobPlan(
+            key=definition.key,
+            skill_path=definition.skill_path,
+            config_hash=definition.config_hash,
+            skill_source=definition.skill_source,
+        ),
+        repository=RepositoryPlan(
+            full_name=repository.full_name, config_sha=commit_sha
+        ),
+        runner=RunnerPlan(
+            image=settings.RUNNER_IMAGE,
+            network_policy="none",
+            collect_changes=pull_request is not None,
+        ),
+        pull_request=pull_request,
+        issues=issues,
+        model=model_plan(workspace) if selection.backend.uses_model_api else None,
+        capabilities=capabilities,
+    )
+
+
+def job_skill_source(job: Job) -> SkillSourcePlan | None:
+    """A synced job's source is always pinned to a commit."""
+    if not job.skill_source:
+        return None
+    return SkillSourcePlan(
+        repository=job.skill_source["repository"],
+        commit_sha=job.skill_source["ref"],
+    )
+
+
+def build_execution_plan(job: Job) -> ExecutionPlan:
+    """The plan for a run of this saved job, with its grants."""
+    capabilities = []
+    for request in job_requests_with_grants(job):
+        grant = getattr(request, "grant", None)
+        capabilities.append(
+            PlannedCapability(
+                RequestedCapability(
+                    request.binding,
+                    request.kind,
+                    tuple(request.requested_tools),
+                    request.required,
+                ),
+                CapabilityApproval(
+                    grant.status,
+                    grant.connection,
+                    tuple(grant.allowed_tools),
+                    grant.tool_rules,
+                )
+                if grant is not None
+                else None,
+            )
+        )
+    return compile_execution_plan(
+        job.workspace,
+        job.repository,
+        ExecutionDefinition(
+            job.key,
+            job.skill_path,
+            job.config_hash,
+            job_skill_source(job),
+        ),
+        capabilities,
+        effective_limits(job),
+        job.repository.config_sha,
+        # Jobs run on the followed branch, so their drafts target it.
+        PullRequestPlan(
+            base=job.repository.default_branch, run_ci=job.repository.pull_request_ci
+        )
+        if job.open_pull_request
+        else None,
+        issues=(
+            IssuesPlan(max=issues.max, labels=tuple(issues.labels))
+            if (issues := job.issue_settings)
+            else None
+        ),
+        subagents=job.subagents,
+    )
+
+
+def build_skill_execution_plan(
+    repository: Repository,
+    definition: JobManifest | TaskManifest,
+    *,
+    key: str,
+    commit_sha: str,
+    pull_request: PullRequestPlan | None = None,
+    skill_source: SkillSourcePlan | None = None,
+) -> ExecutionPlan:
+    """Plan an ad hoc repository skill without saving jobs, requests or grants.
+
+    The caller resolves the repository revision to a commit before planning,
+    and a skill from another repository to its commit.
+    Logical capabilities use only connections that allow automatic approval;
+    grants for other jobs are never inherited. Limits and execution selection
+    come from the workspace, and sandbox policy comes from server settings.
+    This does not check out skill files or submit an execution.
+    """
+    if not re.fullmatch(COMMIT_SHA_PATTERN, commit_sha):
+        raise ValueError("A full commit SHA is required before planning a skill.")
+    requests = manifest_capabilities(definition)
+    connections = connections_by_binding(
+        repository.workspace_id, [request.binding for request in requests]
+    )
+    capabilities = []
+    for request in requests:
+        connection = connections.get(request.binding)
+        approval = None
+        if connection is not None and approves_automatically(
+            request, connection, repository
+        ):
+            approval = CapabilityApproval(
+                JobGrant.Status.APPROVED, connection, request.requested_tools
+            )
+        capabilities.append(PlannedCapability(request, approval))
+    return compile_execution_plan(
+        repository.workspace,
+        repository,
+        ExecutionDefinition(
+            key, definition.skill or "", manifest_hash(definition), skill_source
+        ),
+        capabilities,
+        workspace_limits(repository.workspace),
+        commit_sha,
+        pull_request,
+        subagents=definition.subagents,
+    )

secondshift/jobs/readiness.py

@@ -0,0 +1,46 @@
+"""Why a job cannot run yet, in plain words."""
+
+from secondshift.jobs.models import Job
+from secondshift.jobs.plan import ExecutionPlan, PlanReason
+from secondshift.jobs.planning import build_execution_plan
+from secondshift.repositories.models import Repository
+
+MISMATCH = "is approved for a connection that does not match"
+REASON_TEXT = {
+    PlanReason.NO_GRANT: "waits for an owner's approval",
+    PlanReason.GRANT_PENDING: "waits for an owner's approval",
+    PlanReason.GRANT_DENIED: "was denied by an owner",
+    PlanReason.CONNECTION_MISSING: "has no approved connection",
+    PlanReason.CONNECTION_EXPIRED: "uses a connection whose sign-in expired",
+    PlanReason.CONNECTION_REVOKED: "uses a connection that was disconnected",
+    PlanReason.CONNECTION_WORKSPACE_MISMATCH: MISMATCH,
+    PlanReason.CONNECTION_KIND_MISMATCH: MISMATCH,
+    PlanReason.CONNECTION_BINDING_MISMATCH: MISMATCH,
+    PlanReason.TOOLS_NOT_GRANTED: "requests tools the owner did not approve",
+    PlanReason.TOOLS_UNAVAILABLE: "requests tools the connection does not offer",
+    PlanReason.TOOL_RULES_BROKEN: (
+        "has tool restrictions that no longer fit the connection"
+    ),
+}
+OTHER_REASON = "cannot reach what the job needs through its connection"
+
+
+def execution_problems(repository: Repository, plan: ExecutionPlan) -> list[str]:
+    problems = []
+    if not repository.accepts_runs:
+        problems.append("The repository is disabled.")
+    if not plan.execution.ready:
+        problems.append(plan.execution.problem)
+    for capability in plan.capabilities:
+        if not capability.required or capability.authorized:
+            continue
+        texts = dict.fromkeys(
+            REASON_TEXT.get(reason, OTHER_REASON) for reason in capability.reasons
+        )
+        problems.append(f"{capability.binding} {' and '.join(texts)}.")
+    return problems
+
+
+def readiness_problems(job: Job) -> list[str]:
+    """Empty when Run now would start the job."""
+    return execution_problems(job.repository, build_execution_plan(job))

secondshift/jobs/repository_sync.py

@@ -0,0 +1,199 @@
+"""Bring a repository's jobs up to date with its committed configuration.
+
+The repositories app reads the source. This module reads the manifest and
+skills from it and syncs the jobs, so source code never knows about jobs.
+Git runs outside database transactions.
+"""
+
+from pathlib import Path
+
+import yaml
+from django.db import transaction
+
+from secondshift.config.constants import MANIFEST_FILE
+from secondshift.connections.models import Connection
+from secondshift.github.errors import GitHubError
+from secondshift.jobs.automatic_grants import approve_waiting
+from secondshift.jobs.dao import reopen_github_mcp_grants, repository_active_requests
+from secondshift.jobs.manifest import RepositoryManifest, parse_manifest, skill_key
+from secondshift.jobs.sync import sync_jobs
+from secondshift.repositories.dao import (
+    get_repository_with_connection,
+    restore_repository_fields,
+    set_sync_error,
+)
+from secondshift.repositories.external_skills import (
+    ExternalSkillError,
+    resolve_external_skill,
+)
+from secondshift.repositories.github_access import connection_access
+from secondshift.repositories.github_sync import (
+    fetched_branch,
+    record_synced_revision,
+    save_attached_repository,
+)
+from secondshift.repositories.local import (
+    LocalRepositoryError,
+    LocalRepositoryLocation,
+    committed_file,
+    locate_local_repository,
+    save_local_location,
+)
+from secondshift.repositories.models import Repository
+from secondshift.workspaces.models import Workspace
+
+
+class RepositoryConfigError(LocalRepositoryError):
+    """The branch's manifest, or a skill it names, is missing or invalid.
+
+    The repository can still be added. Its page shows the problem.
+    """
+
+
+class SyncError(Exception):
+    pass
+
+
+def read_committed_config(
+    root: Path, revision: str, branch: str
+) -> tuple[RepositoryManifest, dict[str, str]]:
+    """The manifest and its skills at a commit. Raises RepositoryConfigError."""
+    try:
+        text = committed_file(root, revision, MANIFEST_FILE)
+    except LocalRepositoryError as error:
+        raise RepositoryConfigError(
+            f"The branch {branch} has no committed {MANIFEST_FILE} at its root."
+        ) from error
+    try:
+        manifest = parse_manifest(text)
+    except (ValueError, yaml.YAMLError) as error:
+        raise RepositoryConfigError(f"Invalid {MANIFEST_FILE}: {error}") from error
+    skills = {}
+    try:
+        for job in manifest.jobs.values():
+            if source := job.skill_source:
+                skills[skill_key(job)] = resolve_external_skill(
+                    source.repository, job.skill, source.ref
+                ).text
+            else:
+                skills[skill_key(job)] = committed_file(root, revision, job.skill)
+    except ExternalSkillError as error:
+        raise RepositoryConfigError(f"Invalid {MANIFEST_FILE}: {error}") from error
+    except LocalRepositoryError as error:
+        raise RepositoryConfigError(str(error)) from error
+    return manifest, skills
+
+
+def inspect_local_repository(
+    raw_path: str, branch: str | None = None
+) -> LocalRepositoryLocation:
+    """Check the newest commit on a branch has a valid configuration.
+
+    Only committed files count, so edits in the working folder never change
+    what runs.
+    """
+    location = locate_local_repository(raw_path, branch)
+    read_committed_config(location.root, location.revision, location.branch)
+    return location
+
+
+def attach_local_repository(
+    workspace: Workspace, raw_path: str, branch: str | None = None
+) -> Repository:
+    """Add a local repository, or sync one again from its branch.
+
+    A missing or invalid configuration does not stop it. The repository is
+    saved with the problem in last_sync_error, and its jobs stay as the last
+    good sync left them.
+    """
+    location = locate_local_repository(raw_path, branch)
+    try:
+        manifest, skills = read_committed_config(
+            location.root, location.revision, location.branch
+        )
+        config_error = ""
+    except RepositoryConfigError as error:
+        manifest, skills, config_error = None, {}, str(error)
+    with transaction.atomic():
+        repository, origin_changed = save_local_location(
+            workspace, location, config_error, manifest is not None
+        )
+        if manifest is not None:
+            sync_jobs(repository, manifest, skills)
+        if origin_changed:
+            reopen_github_mcp_grants(repository)
+            approve_waiting(repository_active_requests(repository))
+    return repository
+
+
+def sync_github_repository(repository: Repository) -> Repository:
+    """Fetch the branch into the mirror and sync jobs from its newest commit.
+
+    A missing or invalid configuration is saved in last_sync_error, and the
+    jobs stay as the last good sync left them. The mirror's lock keeps two
+    syncs of one repository in order, so an older one never saves last.
+    """
+    repository = get_repository_with_connection(repository.pk)
+    with fetched_branch(repository) as branch:
+        try:
+            manifest, skills = read_committed_config(
+                branch.mirror, branch.revision, repository.default_branch
+            )
+        except RepositoryConfigError as error:
+            # Keep the last good sync. Runs go on using it.
+            set_sync_error(repository.pk, str(error), checked_sha=branch.revision)
+            repository.refresh_from_db()
+            return repository
+        branch.mark_synced()
+        with transaction.atomic():
+            locked = record_synced_revision(repository.pk, branch.revision)
+            sync_jobs(locked, manifest, skills)
+    repository.refresh_from_db()
+    return repository
+
+
+def attach_github_repository(
+    workspace: Workspace, connection: Connection, repository_id: int, branch: str
+) -> Repository:
+    """Add a repository, or restore a removed one, then sync it.
+
+    The row is saved first in a short transaction, so the GitHub fetch never
+    holds a database lock. If the first sync fails, the row is put back.
+    """
+    metadata = connection_access(connection, repository_id).metadata
+    with transaction.atomic():
+        repository, previous = save_attached_repository(
+            workspace, connection, repository_id, branch, metadata
+        )
+    try:
+        return sync_github_repository(repository)
+    except Exception:
+        if previous is None:
+            repository.delete()
+        else:
+            restore_repository_fields(repository.pk, previous)
+        raise
+
+
+def sync_repository(repository: Repository) -> Repository:
+    """Sync from the branch's newest commit, from GitHub or the local folder."""
+    if repository.source_type == Repository.SourceType.GITHUB:
+        return sync_github_repository(repository)
+    return attach_local_repository(
+        repository.workspace, repository.local_path, repository.default_branch
+    )
+
+
+def resync(repository: Repository) -> Repository:
+    """Read the manifest and skills again from the branch's newest commit.
+
+    A configuration problem is saved on the repository and raised, so the
+    person who pressed Resync sees it.
+    """
+    try:
+        synced = sync_repository(repository)
+    except (GitHubError, LocalRepositoryError) as error:
+        raise SyncError(str(error)) from error
+    if synced.last_sync_error:
+        raise SyncError(synced.last_sync_error)
+    return synced

secondshift/jobs/scheduling.py

@@ -0,0 +1,26 @@
+from datetime import datetime
+from zoneinfo import ZoneInfo
+
+from croniter import croniter
+from django.utils import timezone
+
+from secondshift.jobs.dao import save_next_runs
+from secondshift.jobs.models import Job
+
+
+def next_occurrence(job: Job, after: datetime | None = None) -> datetime:
+    base = after or timezone.now()
+    local_base = base.astimezone(ZoneInfo(job.timezone))
+    next_local = croniter(job.cron, local_base).get_next(datetime)
+    return next_local.astimezone(ZoneInfo("UTC"))
+
+
+def restart_schedules(jobs: list[Job]) -> None:
+    """Set the next run from now for jobs whose schedule was off.
+
+    A job keeps its old next run while it is off. That time is in the past
+    when the job turns back on.
+    """
+    for job in jobs:
+        job.next_run_at = next_occurrence(job) if job.cron else None
+    save_next_runs(jobs)