secondshift 0.1.0b12__py3-none-any.whl

secondshift/runs/models.py

@@ -0,0 +1,340 @@
+import uuid
+from functools import cached_property
+
+from django.conf import settings
+from django.db import models
+from django.utils import timezone
+
+from secondshift.config.constants import RunStart
+from secondshift.connections.models import Connection
+from secondshift.jobs.models import Job
+from secondshift.jobs.plan import ExecutionPlan
+from secondshift.repositories.models import Repository
+from secondshift.runs.inputs import RunInput
+from secondshift.runs.outputs import RunOutputs
+
+
+class RunFailure(models.TextChoices):
+    """Why a run failed. The label is what pages show beside the reason."""
+
+    # The stage a workflow error stopped the run in.
+    STARTING = "starting", "Start"
+    CHECKOUT = "checkout", "Checkout"
+    SANDBOX = "sandbox", "Sandbox"
+    SKILL = "skill", "Skill"
+    FINISHING = "finishing", "Recording"
+    # A run limit stopped the agent.
+    TIMEOUT = "timeout", "Timed out"
+    BUDGET = "budget", "Budget used up"
+    MODEL_PROVIDER = "model_provider", "Model provider"
+    TOOL_RULE = "tool_rule", "Tool restriction"
+    # The queue never took the run, so it does not count as started.
+    NOT_QUEUED = "not_queued", "Run"
+    OTHER = "other", "Run"
+
+
+class Task(models.Model):
+    """One assignment in a repository. Each run of it is an attempt.
+
+    A job run and an ad hoc task each get one. Ad hoc tasks also keep how
+    they were submitted, so a repeated submission finds its first run.
+    """
+
+    # Runs protect their task, so only a task without attempts goes with it.
+    repository = models.ForeignKey(
+        Repository, on_delete=models.CASCADE, related_name="tasks"
+    )
+    # The client's idempotency key for an ad hoc task. Empty for job runs.
+    submission_key = models.CharField(max_length=120, blank=True, default="")
+    # What was submitted with the key. Another submission may not reuse it.
+    fingerprint = models.CharField(max_length=64, blank=True, default="")
+    submitted_by = models.ForeignKey(
+        settings.AUTH_USER_MODEL,
+        on_delete=models.SET_NULL,
+        related_name="secondshift_tasks",
+        null=True,
+        blank=True,
+    )
+    created_at = models.DateTimeField(default=timezone.now, editable=False)
+
+    class Meta:
+        ordering = ["-created_at"]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["repository", "submission_key"],
+                condition=~models.Q(submission_key=""),
+                name="unique_repository_submission",
+            ),
+        ]
+
+
+class Run(models.Model):
+    class Status(models.TextChoices):
+        QUEUED = "queued", "Queued"
+        RUNNING = "running", "Running"
+        SUCCEEDED = "succeeded", "Succeeded"
+        FAILED = "failed", "Failed"
+        CANCELED = "canceled", "Canceled"
+
+    # A run in one of these has not finished yet.
+    ACTIVE_STATUSES = (Status.QUEUED, Status.RUNNING)
+    # A run in one of these is done and will not change status again.
+    FINISHED_STATUSES = (Status.SUCCEEDED, Status.FAILED, Status.CANCELED)
+
+    job = models.ForeignKey(
+        Job, on_delete=models.PROTECT, related_name="runs", null=True, blank=True
+    )
+    task = models.ForeignKey(Task, on_delete=models.PROTECT, related_name="runs")
+    status = models.CharField(
+        max_length=16,
+        choices=Status.choices,
+        default=Status.QUEUED,
+    )
+    commit_sha = models.CharField(max_length=64)
+    scheduled_for = models.DateTimeField()
+    started_by = models.CharField(
+        max_length=16, choices=[(item.value, item.value.title()) for item in RunStart]
+    )
+    started_at = models.DateTimeField(null=True, blank=True)
+    finished_at = models.DateTimeField(null=True, blank=True)
+    sandbox_runtime = models.CharField(max_length=32, blank=True)
+    exit_code = models.IntegerField(null=True, blank=True)
+    summary = models.TextField(blank=True)
+    failure = models.CharField(max_length=16, choices=RunFailure.choices, blank=True)
+    failure_reason = models.TextField(blank=True)
+    # Inputs beyond the plan: an ad hoc task's definition and input, a
+    # trigger's payload and repeat key, and the earlier runs it reads.
+    input_snapshot = models.JSONField(default=dict, blank=True)
+    # The frozen jobs.plan.ExecutionPlan. Read it through `plan`.
+    execution_plan = models.JSONField()
+    # The agent login the plan chose. Its runs per day count against its cap.
+    agent_connection = models.ForeignKey(
+        "connections.AgentConnection",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        related_name="runs",
+    )
+    # The full report is in run_artifacts storage at output_path. Only its
+    # metadata and the tail are kept here.
+    output_excerpt = models.TextField(blank=True)
+    # The rendered input (instructions, prompt and the skill at its commit),
+    # stored next to the output in run_artifacts.
+    input_path = models.CharField(max_length=500, blank=True)
+    input_size_bytes = models.BigIntegerField(null=True, blank=True)
+    output_path = models.CharField(max_length=500, blank=True)
+    output_size_bytes = models.BigIntegerField(null=True, blank=True)
+    output_sha256 = models.CharField(max_length=64, blank=True)
+    output_truncated = models.BooleanField(default=False)
+    # What the sandbox left behind, stored in run_artifacts: the agent's file
+    # changes and the files it saved. See artifacts.store_collection.
+    outputs = models.JSONField(default=dict, blank=True, db_default={})
+    dbos_workflow_id = models.CharField(max_length=200, blank=True, db_index=True)
+    # Sent to the runner so its session can never match one from an older run
+    # that had the same ID, for example after a database reset or restore.
+    runner_session_key = models.UUIDField(default=uuid.uuid4, editable=False)
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        ordering = ["-created_at"]
+        indexes = [
+            models.Index(
+                fields=["status", "created_at"], name="runs_status_created_at_idx"
+            )
+        ]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["job", "scheduled_for"],
+                name="unique_job_scheduled_run",
+            ),
+        ]
+
+    @property
+    def repository(self) -> Repository:
+        return self.task.repository
+
+    @cached_property
+    def inputs(self) -> RunInput:
+        return RunInput.model_validate(self.input_snapshot)
+
+    @cached_property
+    def collected(self) -> RunOutputs | None:
+        """What the sandbox left behind, or None when nothing was collected."""
+        return RunOutputs.model_validate(self.outputs) if self.outputs else None
+
+    @cached_property
+    def plan(self) -> ExecutionPlan:
+        return ExecutionPlan.model_validate(self.execution_plan)
+
+    @property
+    def workspace_id(self) -> int:
+        return self.task.repository.workspace_id
+
+    @property
+    def job_key(self) -> str:
+        """The job's key, or "" for an ad hoc task."""
+        return self.plan.job.key if self.job_id is not None else ""
+
+    @property
+    def display_name(self) -> str:
+        return (
+            self.job.name
+            if self.job is not None
+            else self.inputs.name or "Ad hoc skill"
+        )
+
+    def __str__(self) -> str:
+        return f"{self.display_name} at {self.scheduled_for:%Y-%m-%d %H:%M}"
+
+
+class RunEvent(models.Model):
+    run = models.ForeignKey(Run, on_delete=models.CASCADE, related_name="events")
+    level = models.CharField(max_length=16, default="info")
+    message = models.TextField()
+    metadata = models.JSONField(default=dict, blank=True)
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        ordering = ["created_at"]
+
+    def __str__(self) -> str:
+        return f"{self.run_id}: {self.message[:80]}"
+
+
+class RunDelivery(models.Model):
+    """One place a finished run's report is sent, kept so a failure can be retried.
+
+    A retry sends the stored report again. It never runs the agent again.
+    """
+
+    class Kind(models.TextChoices):
+        NOTIFICATION = "notification", "Notification"
+        PULL_REQUEST = "pull_request", "Pull request"
+        ISSUES = "issues", "Issues"
+
+    class Status(models.TextChoices):
+        # Not tried yet.
+        PENDING = "pending", "Pending"
+        # Tried and failed, and another attempt is coming.
+        RETRYING = "retrying", "Retrying"
+        DELIVERED = "delivered", "Delivered"
+        SKIPPED = "skipped", "Skipped"
+        FAILED = "failed", "Failed"
+
+    run = models.ForeignKey(Run, on_delete=models.CASCADE, related_name="deliveries")
+    kind = models.CharField(max_length=24, choices=Kind.choices)
+    connection = models.ForeignKey(
+        Connection,
+        on_delete=models.SET_NULL,
+        related_name="run_deliveries",
+        null=True,
+        blank=True,
+    )
+    # The connection's name when the delivery was made, kept if it is deleted.
+    target = models.CharField(max_length=200)
+    status = models.CharField(
+        max_length=16, choices=Status.choices, default=Status.PENDING
+    )
+    attempts = models.PositiveIntegerField(default=0)
+    status_code = models.IntegerField(null=True, blank=True)
+    message = models.TextField(blank=True)
+    # Where the delivered result lives, such as the draft pull request.
+    external_url = models.URLField(max_length=500, blank=True)
+    created_at = models.DateTimeField(auto_now_add=True)
+    updated_at = models.DateTimeField(auto_now=True)
+
+    @property
+    def event_name(self) -> str:
+        """How the run's events name this delivery."""
+        if self.kind == self.Kind.NOTIFICATION:
+            return f"Notification to {self.target}"
+        return self.get_kind_display()
+
+    class Meta:
+        ordering = ["created_at", "pk"]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["run", "kind", "connection"],
+                name="unique_run_delivery_target",
+            ),
+            models.UniqueConstraint(
+                fields=["run"],
+                condition=models.Q(kind="pull_request"),
+                name="unique_run_pull_request",
+            ),
+            models.UniqueConstraint(
+                fields=["run"],
+                condition=models.Q(kind="issues"),
+                name="unique_run_issues",
+            ),
+        ]
+
+    def __str__(self) -> str:
+        return f"{self.run_id}: {self.kind} to {self.target} {self.status}"
+
+
+class RunIssue(models.Model):
+    """An issue the agent proposed. SecondShift opens it after the run.
+
+    Opened issues keep their number, so a retry opens only the rest.
+    """
+
+    run = models.ForeignKey(Run, on_delete=models.CASCADE, related_name="issues")
+    # The model's tool call. A replayed call finds its proposal instead of
+    # adding it again.
+    call_id = models.CharField(max_length=128)
+    title = models.CharField(max_length=256)
+    body = models.TextField(blank=True)
+    number = models.PositiveIntegerField(null=True, blank=True)
+    url = models.URLField(max_length=500, blank=True)
+    # An open issue already had this title, so none was opened.
+    already_open = models.BooleanField(default=False)
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        ordering = ["created_at", "pk"]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["run", "call_id"], name="unique_run_issue_call"
+            ),
+        ]
+
+    def __str__(self) -> str:
+        return f"{self.run_id}: {self.title}"
+
+
+class RunAccessNeed(models.Model):
+    """Access the agent said it needed and this run did not have.
+
+    Only a record for people to read. It never grants anything.
+    """
+
+    run = models.ForeignKey(Run, on_delete=models.CASCADE, related_name="access_needs")
+    # The first tool call that named the service. A replayed call finds it.
+    call_id = models.CharField(max_length=128)
+    # As the agent named it, such as "Linear" or "linear".
+    service = models.CharField(max_length=100)
+    tools = models.JSONField(default=list)
+    reason = models.TextField(blank=True)
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        ordering = ["created_at", "pk"]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["run", "call_id"], name="unique_run_access_need_call"
+            ),
+        ]
+
+    def __str__(self) -> str:
+        return f"{self.run_id}: {self.service}"
+
+
+class AgentLease(models.Model):
+    """One active agent process, with a revocable run-scoped broker token."""
+
+    run = models.OneToOneField(Run, on_delete=models.CASCADE)
+    token_hash = models.CharField(max_length=64, unique=True)
+    session_id = models.CharField(max_length=100)
+    active = models.BooleanField(default=True)
+    tool_calls = models.PositiveIntegerField(default=0)

secondshift/runs/orchestration/admission.py

@@ -0,0 +1,15 @@
+"""Admission policy for standalone tasks, shared by dispatch and execution."""
+
+from secondshift.repositories.models import Repository
+from secondshift.runs.models import Run
+from secondshift.workspaces.access import Action, role_allows
+from secondshift.workspaces.dao import active_member_role
+
+
+def task_admission_problem(run: Run, repository: Repository) -> str:
+    if not repository.accepts_runs:
+        return "The repository was disabled or removed before this task started."
+    role = active_member_role(repository.workspace_id, run.task.submitted_by_id)
+    if not role_allows(role, Action.RUN_JOBS):
+        return "The person who submitted this task can no longer run tasks in this workspace."
+    return ""

secondshift/runs/orchestration/dbos_runtime.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from importlib import import_module
+
+from dbos import DBOS, DBOSClient, DBOSConfig
+from django.conf import settings
+
+
+def dbos_config() -> DBOSConfig:
+    return {
+        "name": settings.DBOS_APPLICATION_NAME,
+        "application_version": settings.DBOS_APPLICATION_VERSION,
+        "system_database_url": settings.DBOS_SYSTEM_DATABASE_URL,
+    }
+
+
+def new_dbos_client() -> DBOSClient:
+    return DBOSClient(
+        system_database_url=settings.DBOS_SYSTEM_DATABASE_URL,
+        application_name=settings.DBOS_APPLICATION_NAME,
+    )
+
+
+def launch_dbos_worker() -> None:
+    # Importing registers decorated workflows before DBOS starts recovery.
+    import_module("secondshift.runs.orchestration.workflows")
+
+    if settings.DBOS_WORKER_CONCURRENCY < 1 or settings.DBOS_GLOBAL_CONCURRENCY < 0:
+        raise ValueError(
+            "Run concurrency must be positive; the global limit may be zero."
+        )
+    global_limit = settings.DBOS_GLOBAL_CONCURRENCY or None
+    worker_limit = settings.DBOS_WORKER_CONCURRENCY
+    if global_limit is not None:
+        worker_limit = min(worker_limit, global_limit)
+    DBOS(config=dbos_config())
+    DBOS.listen_queues([settings.DBOS_QUEUE_NAME])
+    DBOS.launch()
+    DBOS.register_queue(
+        settings.DBOS_QUEUE_NAME,
+        worker_concurrency=worker_limit,
+        global_concurrency=global_limit,
+    )

secondshift/runs/orchestration/dispatch.py

@@ -0,0 +1,130 @@
+import logging
+
+from dbos import EnqueueOptions
+from django.conf import settings
+from django.db import transaction
+from django.utils import timezone
+
+from secondshift.config.constants import EXECUTE_RUN_WORKFLOW
+from secondshift.repositories.dao import lock_repository
+from secondshift.runs.dao import (
+    add_event,
+    add_event_once,
+    first_due_task,
+    has_dispatched_task,
+    repository_task_run,
+    waiting_tasks,
+)
+from secondshift.runs.models import Run, RunFailure
+from secondshift.runs.orchestration.admission import task_admission_problem
+from secondshift.runs.orchestration.dbos_runtime import new_dbos_client
+from secondshift.runs.orchestration.reconciliation import (
+    WorkflowStatus,
+    mark_run_terminal,
+)
+from secondshift.runs.transitions import finish_run
+
+logger = logging.getLogger(__name__)
+
+RUN_ENQUEUE_FAILED = "The durable workflow could not be enqueued."
+
+
+class RunEnqueueFailed(Exception):
+    """The workflow could not be queued and the run was marked failed."""
+
+    def __init__(self, run: Run) -> None:
+        super().__init__(RUN_ENQUEUE_FAILED)
+        self.run = run
+
+
+def enqueue_run(run: Run) -> str:
+    if run.job_id is None:
+        return enqueue_task(run)
+    try:
+        workflow_id = queue_workflow(run)
+    except Exception as error:
+        add_event(run.pk, RUN_ENQUEUE_FAILED, level="error")
+        failed = finish_run(
+            run.pk,
+            Run.Status.FAILED,
+            RUN_ENQUEUE_FAILED,
+            only_from={Run.Status.QUEUED},
+            failure=RunFailure.NOT_QUEUED,
+        )
+        if failed is not None:
+            raise RunEnqueueFailed(failed) from error
+        raise
+    return record_workflow(run, workflow_id)
+
+
+def queue_workflow(run: Run) -> str:
+    """Hand the run to DBOS. Raises when DBOS refuses."""
+    workflow_id = f"secondshift-run-{run.pk}"
+    options: EnqueueOptions = {
+        "workflow_name": EXECUTE_RUN_WORKFLOW,
+        "queue_name": settings.DBOS_QUEUE_NAME,
+        "workflow_id": workflow_id,
+        "application_name": settings.DBOS_APPLICATION_NAME,
+    }
+    client = new_dbos_client()
+    try:
+        workflow_id = client.enqueue(options, run.pk).get_workflow_id()
+    finally:
+        try:
+            client.destroy()
+        except Exception:
+            logger.warning("Could not close execution queue client for run %s", run.pk)
+    return workflow_id
+
+
+def record_workflow(run: Run, workflow_id: str) -> str:
+    run.dbos_workflow_id = workflow_id
+    run.save(update_fields=["dbos_workflow_id"])
+    return workflow_id
+
+
+@transaction.atomic
+def enqueue_task(run: Run) -> str:
+    """Hand the oldest eligible task to DBOS, once per repository at a time.
+
+    The repository lock covers the external handoff and its acknowledgement.
+    It serializes submissions, worker retries and queued cancellations.
+    """
+    repository = lock_repository(run.task.repository_id)
+    run = repository_task_run(repository.pk, run.pk)
+    if run.status != Run.Status.QUEUED:
+        return ""
+    if run.dbos_workflow_id:
+        return run.dbos_workflow_id
+    if run.scheduled_for > timezone.now():
+        return ""
+    if problem := task_admission_problem(run, repository):
+        mark_run_terminal(
+            run.pk,
+            Run.Status.FAILED,
+            problem,
+            workflow_status=WorkflowStatus.NOT_ADMITTED,
+        )
+        return ""
+    if has_dispatched_task(repository.pk):
+        return ""
+    first = first_due_task(repository.pk, timezone.now())
+    if first is None or first.pk != run.pk:
+        return ""
+    try:
+        return record_workflow(run, queue_workflow(run))
+    except Exception:
+        # The Run is the durable outbox. Retry with the same workflow ID,
+        # including after a crash between DBOS accepting and our save.
+        add_event_once(
+            run.pk,
+            "Waiting for the execution queue. The worker will retry automatically.",
+            "warning",
+        )
+        return ""
+
+
+def dispatch_waiting_tasks() -> int:
+    """Recover committed submissions and advance the queue on each worker pass."""
+    waiting = waiting_tasks(timezone.now(), 500)
+    return sum(bool(enqueue_task(run)) for run in waiting)

secondshift/runs/orchestration/periodic.py

@@ -0,0 +1,39 @@
+"""Work the worker repeats on a timer, each with its own interval."""
+
+import logging
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+
+logger = logging.getLogger(__name__)
+
+
+def always() -> bool:
+    return True
+
+
+@dataclass
+class PeriodicTask:
+    name: str
+    # Seconds between runs. 0 runs it on every pass of the worker loop.
+    every: float
+    run: Callable[[], object]
+    # Checked each pass, so a setting can turn the task off.
+    enabled: Callable[[], bool] = always
+    clock: Callable[[], float] = time.monotonic
+    last_run: float | None = field(default=None, init=False)
+
+    def due(self) -> bool:
+        if not self.enabled():
+            return False
+        return self.last_run is None or self.clock() - self.last_run >= self.every
+
+    def run_if_due(self) -> None:
+        """Run when due. A failure is logged and the worker carries on."""
+        if not self.due():
+            return
+        self.last_run = self.clock()
+        try:
+            self.run()
+        except Exception:
+            logger.exception("periodic task %r failed", self.name)

secondshift/runs/orchestration/reconciliation.py

@@ -0,0 +1,98 @@
+from enum import StrEnum
+from typing import Literal
+
+from dbos import DBOSClient
+from django.conf import settings
+from django.db import transaction
+
+from secondshift.runs.dao import active_workflows, add_event
+from secondshift.runs.models import Run, RunFailure
+from secondshift.runs.orchestration.dbos_runtime import new_dbos_client
+from secondshift.runs.transitions import finish_run
+
+
+class WorkflowStatus(StrEnum):
+    """DBOS workflow statuses, plus the one recorded before DBOS starts a run."""
+
+    ERROR = "ERROR"
+    MAX_RECOVERY_ATTEMPTS_EXCEEDED = "MAX_RECOVERY_ATTEMPTS_EXCEEDED"
+    CANCELLED = "CANCELLED"
+    SUCCESS = "SUCCESS"
+    NOT_ADMITTED = "NOT_ADMITTED"
+
+
+TERMINAL_WORKFLOW_STATUSES = {
+    WorkflowStatus.ERROR,
+    WorkflowStatus.MAX_RECOVERY_ATTEMPTS_EXCEEDED,
+    WorkflowStatus.CANCELLED,
+    WorkflowStatus.SUCCESS,
+}
+
+
+@transaction.atomic
+def mark_run_terminal(
+    run_id: int,
+    status: Literal["failed", "canceled"],
+    summary: str,
+    *,
+    workflow_status: WorkflowStatus,
+    metadata: dict[str, object] | None = None,
+    failure: RunFailure = RunFailure.OTHER,
+    reason: str = "",
+) -> bool:
+    run = finish_run(run_id, status, summary, failure=failure, reason=reason)
+    if run is None:
+        return False
+    add_event(
+        run.pk,
+        summary,
+        level="error" if status == Run.Status.FAILED else "info",
+        metadata={**(metadata or {}), "workflow_status": workflow_status},
+    )
+    return True
+
+
+def reconcile_terminal_workflows(client: DBOSClient | None = None) -> int:
+    pending_runs = active_workflows(500)
+    if not pending_runs:
+        return 0
+
+    owns_client = client is None
+    client = client or new_dbos_client()
+    try:
+        workflows = client.list_workflows(
+            workflow_ids=[workflow_id for _, workflow_id in pending_runs],
+            application_name=settings.DBOS_APPLICATION_NAME,
+            load_input=False,
+            load_output=False,
+        )
+    finally:
+        if owns_client:
+            client.destroy()
+
+    run_ids = {workflow_id: run_id for run_id, workflow_id in pending_runs}
+    reconciled = 0
+    for workflow in workflows:
+        if workflow.status not in TERMINAL_WORKFLOW_STATUSES:
+            continue
+        run_id = run_ids.get(workflow.workflow_id)
+        if run_id is None:
+            continue
+        if workflow.status == WorkflowStatus.CANCELLED:
+            status = Run.Status.CANCELED
+            summary = "The durable workflow was canceled before the run completed."
+        elif workflow.status == WorkflowStatus.SUCCESS:
+            status = Run.Status.FAILED
+            summary = "The workflow completed without persisting a terminal run result."
+        else:
+            status = Run.Status.FAILED
+            summary = "The durable workflow failed before the run completed."
+        reconciled += int(
+            mark_run_terminal(
+                run_id,
+                status,
+                summary,
+                workflow_status=workflow.status,
+            )
+        )
+    return reconciled