secondshift 0.1.0b12__py3-none-any.whl

secondshift/triggers/schemas.py

@@ -0,0 +1,58 @@
+from datetime import datetime
+
+from ninja import Schema
+from pydantic import Field
+
+from secondshift.config.constants import TriggerKind
+
+
+class TriggerEventSchema(Schema):
+    id: int
+    outcome: str
+    reason: str
+    run_id: int | None
+    received_at: datetime
+    # Requests with the same outcome and reason share one row.
+    # received_at is the newest of them.
+    count: int = 1
+
+
+class TriggerSchema(Schema):
+    kind: TriggerKind
+    # The repository's configuration names this trigger.
+    declared: bool
+    # Someone turned it on at least once, so it has a URL and a secret.
+    set_up: bool
+    enabled: bool
+    url: str | None
+    secret_rotated_at: datetime | None
+    events: list[TriggerEventSchema]
+    # The cap that applies, the owner's own value if set, and the server's.
+    runs_per_hour: int
+    custom_runs_per_hour: int | None
+    default_runs_per_hour: int
+    max_runs_per_hour: int
+    # From the repository's webhook settings. repeat_window includes the default.
+    skip_when: dict[str, str | int | float | bool]
+    repeat_key: str | None
+    repeat_window: str | None
+
+
+class TriggerChangeSchema(Schema):
+    trigger: TriggerSchema
+    # Only right after it was created or rotated. Never returned again.
+    secret: str | None = None
+
+
+class TriggerEnabledSchema(Schema):
+    enabled: bool
+
+
+class TriggerLimitSchema(Schema):
+    # Empty goes back to the server default.
+    runs_per_hour: int | None = Field(default=None, ge=1)
+
+
+class TriggerReceivedSchema(Schema):
+    detail: str
+    run_id: int | None = None

secondshift/triggers/serialization.py

@@ -0,0 +1,91 @@
+from django.conf import settings
+
+from secondshift.config.constants import TriggerKind
+from secondshift.jobs.manifest import WebhookManifest
+from secondshift.jobs.models import Job
+from secondshift.triggers.dao import recent_events
+from secondshift.triggers.models import JobTrigger
+from secondshift.triggers.schemas import (
+    TriggerChangeSchema,
+    TriggerEventSchema,
+    TriggerSchema,
+)
+from secondshift.triggers.services import TriggerChange, runs_per_hour
+
+# How many received requests the job page shows for each trigger.
+EVENTS_SHOWN = 20
+
+
+def trigger_url(trigger: JobTrigger) -> str:
+    return f"{settings.FRONTEND_URL.rstrip('/')}/api/triggers/{trigger.public_id}"
+
+
+def grouped_events(trigger: JobTrigger) -> list[TriggerEventSchema]:
+    """The newest requests, with repeats of the same outcome and reason folded
+    into one row at their newest time. Rows for started runs stay apart.
+
+    A burst of ignored repeats, even for several alerts at once, would
+    otherwise push the runs they repeat out of view.
+    """
+    groups: list[TriggerEventSchema] = []
+    folded: dict[tuple[str, str], TriggerEventSchema] = {}
+    for event in recent_events(trigger, settings.TRIGGER_EVENTS_KEPT * 3):
+        key = (event.outcome, event.reason)
+        if event.run_id is None and key in folded:
+            folded[key].count += 1
+            continue
+        if len(groups) == EVENTS_SHOWN:
+            continue
+        row = TriggerEventSchema(
+            id=event.pk,
+            outcome=event.outcome,
+            reason=event.reason,
+            run_id=event.run_id,
+            received_at=event.received_at,
+        )
+        groups.append(row)
+        if event.run_id is None:
+            folded[key] = row
+    return groups
+
+
+def serialize_trigger(
+    job: Job, kind: TriggerKind, trigger: JobTrigger | None
+) -> TriggerSchema:
+    events = grouped_events(trigger) if trigger else []
+    webhook = (
+        job.webhook_settings if kind == TriggerKind.WEBHOOK else None
+    ) or WebhookManifest()
+    return TriggerSchema(
+        kind=kind,
+        declared=kind in job.triggers,
+        set_up=trigger is not None,
+        enabled=bool(trigger and trigger.enabled),
+        url=trigger_url(trigger) if trigger else None,
+        secret_rotated_at=trigger.secret_rotated_at if trigger else None,
+        events=events,
+        runs_per_hour=runs_per_hour(trigger),
+        custom_runs_per_hour=trigger.max_runs_per_hour if trigger else None,
+        default_runs_per_hour=settings.TRIGGER_MAX_RUNS_PER_HOUR,
+        max_runs_per_hour=settings.TRIGGER_MAX_RUNS_PER_HOUR_CEILING,
+        skip_when=webhook.skip_when,
+        repeat_key=webhook.repeat_key,
+        repeat_window=webhook.window if webhook.repeat_key else None,
+    )
+
+
+def serialize_job_triggers(job: Job) -> list[TriggerSchema]:
+    """Every trigger the job names, and any set up earlier that it no longer names."""
+    existing = {TriggerKind(item.kind): item for item in job.job_triggers.all()}
+    kinds = [TriggerKind(kind) for kind in job.triggers] + [
+        kind for kind in existing if kind not in job.triggers
+    ]
+    return [serialize_trigger(job, kind, existing.get(kind)) for kind in kinds]
+
+
+def serialize_change(job: Job, change: TriggerChange) -> TriggerChangeSchema:
+    trigger = change.trigger
+    return TriggerChangeSchema(
+        trigger=serialize_trigger(job, TriggerKind(trigger.kind), trigger),
+        secret=change.secret,
+    )

secondshift/triggers/services.py

@@ -0,0 +1,246 @@
+"""Turn job triggers on and off, and start runs from the requests they receive.
+
+The repository names the trigger. Turning it on creates its URL and secret.
+"""
+
+import secrets
+import time
+from collections.abc import Mapping
+from dataclasses import dataclass
+from datetime import timedelta
+
+from django.conf import settings
+from django.utils import timezone
+
+from secondshift.config.constants import RunStart, TriggerKind
+from secondshift.jobs.manifest import WebhookManifest, duration_seconds
+from secondshift.jobs.models import JOB_REMOVED, Job
+from secondshift.runs.dao import run_for_repeat_key, started_run_count
+from secondshift.runs.models import Run
+from secondshift.runs.orchestration.dispatch import RunEnqueueFailed
+from secondshift.runs.services import RunInProgress, RunNotReady, RunRefused, start_run
+from secondshift.triggers.dao import (
+    add_event,
+    create_trigger,
+    has_recent_event,
+    job_trigger,
+)
+from secondshift.triggers.filters import (
+    parse_body,
+    repeat_key,
+    skip_reason,
+)
+from secondshift.triggers.models import JobTrigger, TriggerEvent
+from secondshift.triggers.verification import verify
+
+# A failed sign-in is recorded at most this often, so guessing cannot flood
+# the log.
+UNAUTHORIZED_RECORD_SECONDS = 60
+
+# Secret scanners can match trigger secrets by this prefix.
+SECRET_PREFIX = "sst_"
+
+ENQUEUE_RETRY_SECONDS = 60
+
+
+class TriggerError(Exception):
+    pass
+
+
+@dataclass(frozen=True)
+class TriggerChange:
+    trigger: JobTrigger
+    # Only when a secret was just created or rotated. It is shown once.
+    secret: str | None = None
+
+
+@dataclass(frozen=True)
+class Received:
+    """What a trigger request led to. status is the HTTP status to answer."""
+
+    status: int
+    detail: str
+    run: Run | None = None
+    retry_after: int | None = None
+
+
+def new_secret() -> str:
+    return SECRET_PREFIX + secrets.token_urlsafe(32)
+
+
+def set_trigger_enabled(job: Job, kind: TriggerKind, enabled: bool) -> TriggerChange:
+    """Turn a trigger on or off. The first time on creates its URL and secret."""
+    trigger = job_trigger(job, kind)
+    if enabled and kind not in job.triggers:
+        raise TriggerError(
+            f"The job's configuration does not name a {kind} trigger. "
+            f"Add triggers: [{kind}] to the job in .secondshift.yaml."
+        )
+    if trigger is None:
+        if not enabled:
+            raise TriggerError("This trigger is not set up.")
+        secret = new_secret()
+        trigger = create_trigger(job, kind, secret, timezone.now())
+        return TriggerChange(trigger, secret)
+    trigger.enabled = enabled
+    trigger.save(update_fields=["enabled", "updated_at"])
+    return TriggerChange(trigger)
+
+
+def rotate_secret(job: Job, kind: TriggerKind) -> TriggerChange:
+    """Replace the secret. Requests signed with the old one are refused."""
+    trigger = job_trigger(job, kind)
+    if trigger is None:
+        raise TriggerError("This trigger is not set up.")
+    secret = new_secret()
+    trigger.secret = secret
+    trigger.secret_rotated_at = timezone.now()
+    trigger.save(update_fields=["secret", "secret_rotated_at", "updated_at"])
+    return TriggerChange(trigger, secret)
+
+
+def set_runs_per_hour(job: Job, kind: TriggerKind, value: int | None) -> JobTrigger:
+    """Set the trigger's hourly cap. None goes back to the server default."""
+    trigger = job_trigger(job, kind)
+    if trigger is None:
+        raise TriggerError("This trigger is not set up.")
+    ceiling = settings.TRIGGER_MAX_RUNS_PER_HOUR_CEILING
+    if value is not None and value > ceiling:
+        raise TriggerError(f"This server allows at most {ceiling} runs per hour.")
+    trigger.max_runs_per_hour = value
+    trigger.save(update_fields=["max_runs_per_hour", "updated_at"])
+    return trigger
+
+
+def record(
+    trigger: JobTrigger, outcome: str, reason: str = "", run: Run | None = None
+) -> None:
+    """Keep the event, and only the newest few of each outcome for each trigger.
+
+    Kept per outcome, so a flood of refusals never pushes out started runs.
+    """
+    add_event(trigger, outcome, reason[:500], run, settings.TRIGGER_EVENTS_KEPT)
+
+
+def refuse(trigger: JobTrigger, status: int, reason: str) -> Received:
+    record(trigger, TriggerEvent.Outcome.REFUSED, reason)
+    return Received(status, reason)
+
+
+def ignore(trigger: JobTrigger, reason: str) -> Received:
+    """Answer 2xx, so the sender does not retry a request the job skips."""
+    record(trigger, TriggerEvent.Outcome.IGNORED, reason)
+    return Received(200, reason)
+
+
+def refuse_unauthorized(trigger: JobTrigger, reason: str) -> Received:
+    since = timezone.now() - timedelta(seconds=UNAUTHORIZED_RECORD_SECONDS)
+    if not has_recent_event(trigger, TriggerEvent.Outcome.REFUSED, reason, since):
+        record(trigger, TriggerEvent.Outcome.REFUSED, reason)
+    return Received(401, reason)
+
+
+def runs_per_hour(trigger: JobTrigger | None) -> int:
+    """The owner's cap or the server default, never above the server ceiling."""
+    chosen = (trigger and trigger.max_runs_per_hour) or (
+        settings.TRIGGER_MAX_RUNS_PER_HOUR
+    )
+    return min(chosen, settings.TRIGGER_MAX_RUNS_PER_HOUR_CEILING)
+
+
+def earlier_run(job: Job, key: str, webhook: WebhookManifest) -> Run | None:
+    """A run for the same event that is active or finished within the window."""
+    since = timezone.now() - timedelta(seconds=duration_seconds(webhook.window))
+    return run_for_repeat_key(job, key, since)
+
+
+def repeat_reason(run: Run) -> str:
+    return f"Repeat of the event that started run #{run.pk}."
+
+
+def not_ready_reason(error: RunNotReady) -> str:
+    return "The job is not ready: " + " ".join(error.problems)
+
+
+def refusal(trigger: JobTrigger) -> Received | None:
+    """Why the trigger itself cannot start a run, before the body is read."""
+    if not trigger.enabled:
+        return refuse(trigger, 409, "This trigger is turned off.")
+    job = trigger.job
+    if job.repository.removed_at is not None:
+        return refuse(trigger, 409, "The job's repository was removed.")
+    if job.removed_at is not None:
+        return refuse(trigger, 409, JOB_REMOVED)
+    if trigger.kind not in job.triggers:
+        return refuse(
+            trigger,
+            409,
+            f"The job's configuration no longer names a {trigger.kind} trigger.",
+        )
+    return None
+
+
+def hourly_refusal(trigger: JobTrigger) -> Received | None:
+    limit = runs_per_hour(trigger)
+    started = started_run_count(
+        trigger.job,
+        RunStart.WEBHOOK,
+        timezone.now() - timedelta(hours=1),
+    )
+    if started < limit:
+        return None
+    runs = "run" if limit == 1 else "runs"
+    return refuse(
+        trigger, 429, f"This trigger already started {limit} {runs} in the last hour."
+    )
+
+
+def start(trigger: JobTrigger, body: bytes, key: str | None) -> Received:
+    try:
+        run = start_run(trigger.job, RunStart.WEBHOOK, body, key)
+    except RunInProgress as error:
+        if key is not None and error.run.inputs.repeat_key == key:
+            return ignore(trigger, repeat_reason(error.run))
+        return refuse(
+            trigger, 409, f"Run #{error.run.pk} of this job has not finished."
+        )
+    except RunNotReady as error:
+        return refuse(trigger, 409, not_ready_reason(error))
+    except RunRefused as error:
+        return refuse(trigger, 409, str(error))
+    except RunEnqueueFailed as error:
+        reason = "The run could not be queued."
+        record(trigger, TriggerEvent.Outcome.REFUSED, reason, run=error.run)
+        return Received(503, reason, error.run, retry_after=ENQUEUE_RETRY_SECONDS)
+    record(trigger, TriggerEvent.Outcome.STARTED, run=run)
+    return Received(202, "Run started.", run)
+
+
+def receive(trigger: JobTrigger, headers: Mapping[str, str], body: bytes) -> Received:
+    """Check one request and start a run if everything allows it.
+
+    In order: the secret, the trigger, the job's webhook settings (skip and
+    repeats), the hourly cap, then the job itself.
+    """
+    verdict = verify(
+        headers,
+        body,
+        trigger.secret,
+        now=time.time(),
+        max_age_seconds=settings.TRIGGER_SIGNATURE_MAX_AGE_SECONDS,
+    )
+    if not verdict.ok:
+        return refuse_unauthorized(trigger, verdict.reason)
+    if refused := refusal(trigger):
+        return refused
+    declared = trigger.job.webhook_settings
+    data = parse_body(body) if declared else None
+    webhook = declared or WebhookManifest()
+    if reason := skip_reason(data, webhook.skip_when):
+        return ignore(trigger, reason)
+    key = repeat_key(data, webhook.repeat_key)
+    if key is not None and (earlier := earlier_run(trigger.job, key, webhook)):
+        return ignore(trigger, repeat_reason(earlier))
+    if refused := hourly_refusal(trigger):
+        return refused
+    return start(trigger, body, key)

secondshift/triggers/verification.py

@@ -0,0 +1,84 @@
+"""Check that a trigger request came from someone who holds its secret.
+
+A request passes with either:
+
+- an Authorization header whose credentials are the secret, with any scheme,
+  such as "Bearer <secret>".
+- a hex HMAC-SHA256 of the body, keyed with the secret, in SIGNATURE_HEADER.
+  With TIMESTAMP_HEADER, the signature covers "<timestamp>:<body>" and the
+  timestamp must be recent.
+
+Senders that sign with their own header names, such as Grafana, let you set
+the names. Point them at these.
+"""
+
+import hashlib
+import hmac
+from collections.abc import Mapping
+from dataclasses import dataclass
+
+SIGNATURE_HEADER = "X-SecondShift-Signature"
+TIMESTAMP_HEADER = "X-SecondShift-Timestamp"
+SIGNATURE_PREFIX = "sha256="
+
+
+@dataclass(frozen=True)
+class Verdict:
+    ok: bool
+    reason: str = ""
+
+
+MISSING = Verdict(False, "The request had no Authorization header or signature.")
+WRONG = Verdict(False, "The request's credentials did not match the secret.")
+STALE = Verdict(False, "The signed timestamp is missing, invalid or too old.")
+
+
+def header(headers: Mapping[str, str], name: str) -> str | None:
+    value = headers.get(name)
+    return None if value is None else value.strip()
+
+
+def same(given: str, expected: str) -> bool:
+    return hmac.compare_digest(given.encode(), expected.encode())
+
+
+def authorization_matches(value: str, secret: str) -> bool:
+    parts = value.split(None, 1)
+    credentials = parts[1] if len(parts) == 2 else ""
+    return bool(credentials) and same(credentials.strip(), secret)
+
+
+def signature(secret: str, body: bytes, timestamp: str | None) -> str:
+    signed = f"{timestamp}:".encode() + body if timestamp is not None else body
+    return hmac.new(secret.encode(), signed, hashlib.sha256).hexdigest()
+
+
+def fresh(timestamp: str, now: float, max_age_seconds: int) -> bool:
+    try:
+        sent = float(timestamp)
+    except ValueError:
+        return False
+    return abs(now - sent) <= max_age_seconds
+
+
+def verify(
+    headers: Mapping[str, str],
+    body: bytes,
+    secret: str,
+    *,
+    now: float,
+    max_age_seconds: int,
+) -> Verdict:
+    authorization = headers.get("Authorization")
+    if authorization is not None:
+        return Verdict(True) if authorization_matches(authorization, secret) else WRONG
+    given = header(headers, SIGNATURE_HEADER)
+    if given is None:
+        return MISSING
+    given = given.removeprefix(SIGNATURE_PREFIX).lower()
+    timestamp = header(headers, TIMESTAMP_HEADER)
+    if not same(given, signature(secret, body, timestamp)):
+        return WRONG
+    if timestamp is not None and not fresh(timestamp, now, max_age_seconds):
+        return STALE
+    return Verdict(True)