hawk 3.2.0__tar.gz → 3.3.0__tar.gz

{hawk-3.2.0 → hawk-3.3.0}/.dockerignore

@@ -17,22 +17,6 @@ services/**
 !services/modules/*/pyproject.toml
 !services/modules/*/README.md
 !services/modules/*/uv.lock
-!services/modules/viewer/nginx.conf
-
-!www
-www/**
-!www/package.json
-!www/pnpm-lock.yaml
-!www/pnpm-workspace.yaml
-!www/index.html
-!www/eslint.config.js
-!www/tsconfig.json
-!www/vite.config.ts
-!www/tailwind.config.js
-!www/src
-!www/src/**
-!www/public
-!www/public/**
 
 hawk/tests/**
 **/.venv/**

{hawk-3.2.0 → hawk-3.3.0}/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## [3.3.0](https://github.com/METR/hawk/compare/v3.2.0...v3.3.0) (2026-09-08)
+
+
+### Features
+
+* **runner:** enable adaptive model concurrency and unblock sandbox starts ([#1607](https://github.com/METR/hawk/issues/1607)) ([ed86d01](https://github.com/METR/hawk/commit/ed86d016b59dd29d13db5ea59347804de7ea45c1))
+
+
+### Bug Fixes
+
+* **api:** export scan results by scanner key and 404 a missing parquet [SEN-189] ([#1259](https://github.com/METR/hawk/issues/1259)) ([1e15689](https://github.com/METR/hawk/commit/1e15689957cbbfca22ea31bcf5c2aeb4c3b3eafe))
+* **authz:** derive a scan's ACL from the evals it read ([#1142](https://github.com/METR/hawk/issues/1142)) ([e763980](https://github.com/METR/hawk/commit/e763980a51e807fd3be43d3c20ecdc0d4c4bd4ba))
+* **db:** raise statement_timeout for the a726f26fdd42 scan recompute ([#1634](https://github.com/METR/hawk/issues/1634)) ([7e9bb91](https://github.com/METR/hawk/commit/7e9bb916d9a462943c2c8380073c1b207ce643cb))
+* **human-eval:** report the jumphost cause when key registration fails [SEN-240] ([#1447](https://github.com/METR/hawk/issues/1447)) ([5251885](https://github.com/METR/hawk/commit/52518856f2a915b64a96256b6f5882e008af8a91))
+* **runner:** cache token-broker credentials on the pod ([#1585](https://github.com/METR/hawk/issues/1585)) ([15d1616](https://github.com/METR/hawk/commit/15d1616d8748b9217d25e2e9f9112cc2b34386a3))
+* **runner:** let tasks and solvers resolve model roles at construction ([#1619](https://github.com/METR/hawk/issues/1619)) ([9b24b19](https://github.com/METR/hawk/commit/9b24b190247a664100dc1bba33786829032b995f))
+* **shell:** log the failing command's output, not just its headline ([#1488](https://github.com/METR/hawk/issues/1488)) ([60a0f42](https://github.com/METR/hawk/commit/60a0f42d2f0c8f97318bbde87c3766104dc6914c))
+
 ## [3.2.0](https://github.com/METR/hawk/compare/v3.1.1...v3.2.0) (2026-09-07)
 
 

{hawk-3.2.0 → hawk-3.3.0}/Dockerfile

@@ -1,14 +1,10 @@
 # Keep this default aligned with infra.lib.config.DEFAULT_BASE_PYTHON_IMAGE.
 ARG PYTHON_BASE_IMAGE=dhi.io/python:3.13-dev
 ARG AWS_CLI_VERSION=2.27.26
-ARG DOCKER_VERSION=29.7.1
-ARG KUBECTL_VERSION=1.36.2
 ARG UV_VERSION=0.12.3
 
 FROM amazon/aws-cli:${AWS_CLI_VERSION} AS aws-cli
-FROM docker:${DOCKER_VERSION}-cli AS docker-cli
 FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv
-FROM rancher/kubectl:v${KUBECTL_VERSION} AS kubectl
 FROM ${PYTHON_BASE_IMAGE} AS python
 
 FROM --platform=$BUILDPLATFORM alpine:3.24 AS helm
@@ -62,7 +58,13 @@ ENV UV_LINK_MODE=copy
 FROM base AS builder-base
 WORKDIR /source
 COPY pyproject.toml uv.lock ./
-COPY services/modules services/modules
+# `uv sync --locked` only reads the path dependencies' pyproject.toml (they are
+# in dev-only groups, never installed here); copying their sources would rerun
+# every venv install on each lambda .py change.
+COPY services/modules/eval_log_importer/pyproject.toml services/modules/eval_log_importer/
+COPY services/modules/eval_log_reader/pyproject.toml services/modules/eval_log_reader/
+COPY services/modules/job_status_updated/pyproject.toml services/modules/job_status_updated/
+COPY services/modules/sample_editor/pyproject.toml services/modules/sample_editor/
 
 FROM builder-base AS builder-runner
 RUN --mount=type=cache,target=/root/.cache/uv \
@@ -73,8 +75,19 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --no-dev \
         --no-install-project
 
+# inspect resolves restic from ~/.cache/inspect_ai/bin/restic_<version>_<platform>
+# (inspect_ai/util/_restic/resolver.py), version from the installed package, so
+# this stays in lockstep with the pinned inspect-ai. Build the whole destination
+# tree with fixed ownership and mtimes so the runner's COPY --link layer is
+# byte-stable. Linux only: the runner host and the sandboxes are linux
+# (_checkpoint/_sandbox_restic/repo.py maps amd64/arm64 to linux_*).
 FROM builder-runner AS restic-cache
-RUN HOME=/home/nonroot inspect download restic
+RUN HOME=/out/home/nonroot inspect download restic \
+ && rm /out/home/nonroot/.cache/inspect_ai/bin/restic_*_darwin_* \
+       /out/home/nonroot/.cache/inspect_ai/bin/restic_*_windows_* \
+ && chown -R ${USER_ID}:${GROUP_ID} /out/home/nonroot \
+ && chown root:root /out /out/home \
+ && find /out -exec touch -h -t 202001010000.00 {} +
 
 FROM builder-base AS builder-api
 RUN --mount=type=cache,target=/root/.cache/uv \
@@ -95,10 +108,10 @@ FROM base AS runner
 ENV LD_PRELOAD=libjemalloc.so.2
 ENV MALLOC_CONF=background_thread:true,dirty_decay_ms:30000,muzzy_decay_ms:30000,metadata_thp:auto
 
-COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker
-COPY --from=docker-cli /usr/local/libexec/docker/cli-plugins/docker-buildx /usr/local/libexec/docker/cli-plugins/docker-buildx
 COPY --from=helm /helm /usr/local/bin/helm
-COPY --from=kubectl /bin/kubectl /usr/local/bin/
+# --link: the layer is built from the restic-cache stage alone, not this stage's
+# snapshot, so its digest is identical on every build.
+COPY --link --from=restic-cache /out/ /
 
 WORKDIR /home/nonroot/app
 COPY --from=builder-runner ${UV_PROJECT_ENVIRONMENT} ${UV_PROJECT_ENVIRONMENT}
@@ -115,13 +128,12 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 USER nonroot
 STOPSIGNAL SIGINT
 
-COPY --from=restic-cache --chown=nonroot:nonroot /home/nonroot/.cache/inspect_ai /home/nonroot/.cache/inspect_ai
 ENTRYPOINT ["python", "-m", "hawk.runner.entrypoint"]
 
 FROM base AS api
 # Install graphviz. x11-common's postinst needs update-rc.d from init-system-helpers
 RUN apt-get update \
- && apt-get install -y init-system-helpers graphviz \
+ && apt-get install -y --no-install-recommends init-system-helpers graphviz \
  && rm -rf /var/lib/apt/lists/*
 
 COPY --from=aws-cli /usr/local/aws-cli/v2/current /usr/local

{hawk-3.2.0 → hawk-3.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: hawk
-Version: 3.2.0
+Version: 3.3.0
 Summary: Platform for running Inspect AI evaluations and Inspect Scout scans at scale on Kubernetes
 Project-URL: Homepage, https://hawk.metr.org/
 Project-URL: Documentation, https://hawk.metr.org/

{hawk-3.2.0 → hawk-3.3.0}/hawk/api/EvalSetConfig.schema.json

@@ -1,5 +1,43 @@
 {
   "$defs": {
+    "AdaptiveConnectionsBounds": {
+      "additionalProperties": false,
+      "description": "Bounds for inspect-ai's adaptive connections controller.",
+      "properties": {
+        "min": {
+          "description": "Lowest concurrency the controller may wind a throttled model down to.",
+          "exclusiveMinimum": 0,
+          "title": "Min",
+          "type": "integer"
+        },
+        "max": {
+          "description": "Ceiling the controller may scale up to.",
+          "exclusiveMinimum": 0,
+          "title": "Max",
+          "type": "integer"
+        },
+        "start": {
+          "anyOf": [
+            {
+              "exclusiveMinimum": 0,
+              "type": "integer"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Concurrency to start at, within [min, max]. Unset: inspect-ai's default start, clamped into the bounds.",
+          "title": "Start"
+        }
+      },
+      "required": [
+        "min",
+        "max"
+      ],
+      "title": "AdaptiveConnectionsBounds",
+      "type": "object"
+    },
     "AgentConfig": {
       "description": "Configuration for an agent.",
       "properties": {
@@ -1465,10 +1503,20 @@
         {
           "exclusiveMinimum": 0,
           "type": "integer"
+        },
+        {
+          "pattern": "^\\d+-\\d+(-\\d+)?$",
+          "type": "string"
+        },
+        {
+          "$ref": "#/$defs/AdaptiveConnectionsBounds"
+        },
+        {
+          "type": "null"
         }
       ],
-      "default": false,
-      "description": "Enable inspect-ai's adaptive connections controller, which automatically scales model API concurrency based on rate-limit feedback. Set to an integer N to also raise the controller's concurrency ceiling to N (inspect-ai shorthand for `AdaptiveConcurrency(max=N)`); `true` uses inspect-ai's default bounds. Note that an explicit per-model `max_connections` takes precedence over the adaptive controller (adaptive is silently disabled for that model), so remove `max_connections` for adaptive to apply. May become the default in a future release. See https://inspect.aisi.org.uk/models-concurrency.html#adaptive-connections.",
+      "default": null,
+      "description": "inspect-ai's adaptive connections controller, which scales model API concurrency up and down from rate-limit feedback instead of holding a fixed ceiling. Unset (the default) defers to inspect-ai, which enables it; `false` opts out. Set an integer N to raise the ceiling to N (inspect-ai shorthand for `AdaptiveConcurrency(max=N)`), or set the bounds directly as a mapping such as `{min: 1, max: 100}` or the shorthand `\"1-20-100\"` (`min-max` or `min-start-max`) \u2014 a `min` of 1 lets a throttled model be wound down to a single in-flight request. Note that an explicit per-model `max_connections` takes precedence (adaptive is silently disabled for that model), so remove `max_connections` for adaptive to apply. See https://inspect.aisi.org.uk/models-concurrency.html#adaptive-connections.",
       "title": "Adaptive Connections"
     },
     "monitor": {

{hawk-3.2.0 → hawk-3.3.0}/hawk/api/human_eval_server.py

@@ -361,8 +361,10 @@ async def create_human_eval(
             public_key=human.ssh_public_key,
             comment=comment,
         )
-    except Exception:
+    except Exception as e:
         logger.exception("Failed to register SSH key for human eval %s", eval_set_id)
+        # The caller can't see server logs, so the cause travels with the error.
+        cause = f"{type(e).__name__}: {e}"
         rolled_back = await _uninstall_eval_set(helm_client, eval_set_id, settings)
         if rolled_back:
             # Helm release is gone, no key was ever registered — drop the
@@ -374,24 +376,24 @@ async def create_human_eval(
             raise problem.AppError(
                 title="Jump host key registration failed",
                 message=(
-                    "Failed to register SSH key on jump host. The Helm release "
-                    "has been uninstalled; any Datadog monitors and S3 files "
-                    "created during eval-set setup remain and will need to be "
-                    "cleaned up out of band."
+                    f"Failed to register SSH key on jump host ({cause}). The Helm "
+                    f"release has been uninstalled and the launch rolled back, so "
+                    f"you can retry. Any S3 files and Datadog monitor left over "
+                    f"from eval-set {eval_set_id}'s setup are inert."
                 ),
                 status_code=HTTPStatus.BAD_GATEWAY,
-            )
+            ) from e
         # Rollback failed — leave the assignment row in place so a future
         # DELETE /eval_sets/{id} re-attempts cleanup of the still-running release.
         raise problem.AppError(
             title="Jump host key registration failed; rollback also failed",
             message=(
-                f"Failed to register SSH key on jump host AND failed to tear down "
-                f"the eval-set. Eval-set {eval_set_id} is still running and needs "
-                f"manual cleanup via DELETE /eval_sets/{eval_set_id}."
+                f"Failed to register SSH key on jump host ({cause}) AND failed to "
+                f"tear down the eval-set. Eval-set {eval_set_id} is still running "
+                f"and needs manual cleanup via DELETE /eval_sets/{eval_set_id}."
             ),
             status_code=HTTPStatus.BAD_GATEWAY,
-        )
+        ) from e
 
     return CreateHumanEvalResponse(
         eval_set_id=eval_set_id,

{hawk-3.2.0 → hawk-3.3.0}/hawk/api/meta_server.py

@@ -10,6 +10,7 @@ from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, cast
 
 import fastapi
+import fastapi.concurrency
 import pydantic
 import sqlalchemy as sa
 import sqlalchemy.sql.elements as sql_elements
@@ -2153,6 +2154,31 @@ async def export_scan_results(
     # Fetch Arrow results (async for S3 metadata)
     results = await hawk.core.scan_export.get_scan_results_arrow(info.scan_location)
 
+    # Open the parquet before the response starts: starlette sends the 200
+    # before it touches the body iterator, so a missing file found inside the
+    # generator can't become a 404 (SEN-189).
+    try:
+        reader = await fastapi.concurrency.run_in_threadpool(
+            hawk.core.scan_export.open_scan_results_reader,
+            results,
+            info.scanner_key,
+        )
+    except FileNotFoundError:
+        # A 404 doesn't reach Sentry; this log is the only record of the miss.
+        log.warning(
+            "scan_export_missing",
+            extra={
+                "event": "scan_export_missing",
+                "scanner_result_uuid": scanner_result_uuid,
+                "scan_location": info.scan_location,
+                "scanner_key": info.scanner_key,
+            },
+        )
+        raise fastapi.HTTPException(
+            status_code=404,
+            detail="Scan results data not found",
+        ) from None
+
     safe_scan_id = utils.sanitize_filename(info.scan_id)
     safe_scanner_name = utils.sanitize_filename(info.scanner_name)
     filename = f"{safe_scan_id}_{safe_scanner_name}.csv"
@@ -2160,7 +2186,7 @@ async def export_scan_results(
     # Return streaming response with sync generator
     # (FastAPI handles sync iterators correctly)
     return StreamingResponse(
-        hawk.core.scan_export.stream_scan_results_csv(results, info.scanner_name),
+        hawk.core.scan_export.stream_scan_results_csv(reader),
         media_type="text/csv",
         headers={"Content-Disposition": f'attachment; filename="{filename}"'},
     )

hawk-3.3.0/hawk/core/db/alembic/versions/a726f26fdd42_scan_model_groups_inherit_eval_groups.py

@@ -0,0 +1,174 @@
+"""scan model_groups: inherit the source evals' restriction
+
+Rewrites `compute_scan_model_groups()` so a scan's ACL is never less
+restrictive than the evals whose transcripts it read, then recomputes
+`scan.model_groups` for the existing rows that are safe to recompute.
+
+Before this the function read model *names* only -- the scan's own `model`, its
+`model_role` rows, and `sample_model` -- and never the source `eval`, so a
+restricted model was caught only if it surfaced in `sample_model`. Live
+ingestion made that a fail-open: the `__unresolved__` sentinel fires only when
+some `scanner_result.sample_pk IS NULL`, which was always true for a running
+eval set until live rows began resolving `sample_pk` early while `sample_model`
+was still a partial lower bound. Sentinel suppressed and no restricted model yet
+streamed, the scan lands on `model_groups = {}` -- read here as visible to every
+authenticated user.
+
+On prd every scan already satisfied the new invariant, making the backfill a
+no-op there; it is the dev/stg live-ingest window where it can change rows.
+
+The new *term* only ever widens, but recomputing a row does not: a scan parked
+at `__unresolved__` by a deferred import can have every `sample_pk` resolved by
+now, so re-deriving drops the sentinel and -- absent an inherited group -- lands
+on `{}`. The backfill therefore skips rows already carrying it, as
+`c7d8e9f0a1b2` does.
+
+Revision ID: a726f26fdd42
+Revises: e54f61f05480
+Create Date: 2026-07-31 12:05:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "a726f26fdd42"
+down_revision: str | None = "e54f61f05480"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+_COMPUTE_SCAN_MODEL_GROUPS_NEW = """
+CREATE OR REPLACE FUNCTION compute_scan_model_groups(target_scan_pk uuid)
+RETURNS text[]
+LANGUAGE sql
+STABLE
+SECURITY DEFINER
+SET search_path = middleman, public, pg_catalog, pg_temp
+AS $$
+    SELECT COALESCE(array_agg(DISTINCT g.name ORDER BY g.name), ARRAY[]::text[])
+    FROM (
+        SELECT mg.name
+        FROM (
+            SELECT model AS m FROM scan WHERE pk = target_scan_pk AND model IS NOT NULL
+            UNION
+            SELECT model AS m FROM model_role WHERE scan_pk = target_scan_pk
+            UNION
+            SELECT sm.model AS m FROM sample_model sm
+            JOIN sample s ON s.pk = sm.sample_pk
+            JOIN scanner_result sr ON sr.sample_pk = s.pk
+            WHERE sr.scan_pk = target_scan_pk
+        ) models
+        JOIN middleman.model md ON (
+                md.name = models.m
+                OR (models.m <> ''
+                    AND right(md.name, char_length(models.m) + 1)
+                        = '/' || models.m)
+                OR (md.name <> ''
+                    AND right(models.m, char_length(md.name) + 1)
+                        = '/' || md.name)
+            )
+        JOIN middleman.model_group mg ON mg.pk = md.model_group_pk
+
+        UNION
+
+        SELECT source.name
+        FROM scanner_result sr
+        JOIN sample s ON s.pk = sr.sample_pk
+        JOIN eval e ON e.pk = s.eval_pk
+        CROSS JOIN LATERAL unnest(e.model_groups) AS source(name)
+        WHERE sr.scan_pk = target_scan_pk
+
+        UNION
+
+        SELECT '__unresolved__'::text
+        WHERE EXISTS (
+            SELECT 1 FROM scanner_result sr
+            WHERE sr.scan_pk = target_scan_pk
+              AND sr.transcript_source_type = 'eval_log'
+              AND sr.sample_pk IS NULL
+        )
+    ) g
+$$
+"""
+
+
+# The shape this replaces, as of c7d8e9f0a1b2: name-derived groups only, but
+# already joining middleman prefix-tolerantly. Downgrading must keep that join
+# or it re-opens the canonical-name fail-open (METR/hawk#1198).
+_COMPUTE_SCAN_MODEL_GROUPS_OLD = """
+CREATE OR REPLACE FUNCTION compute_scan_model_groups(target_scan_pk uuid)
+RETURNS text[]
+LANGUAGE sql
+STABLE
+SECURITY DEFINER
+SET search_path = middleman, public, pg_catalog, pg_temp
+AS $$
+    SELECT COALESCE(array_agg(DISTINCT mg.name ORDER BY mg.name), ARRAY[]::text[])
+        || CASE WHEN EXISTS (
+            SELECT 1 FROM scanner_result sr
+            WHERE sr.scan_pk = target_scan_pk
+              AND sr.transcript_source_type = 'eval_log'
+              AND sr.sample_pk IS NULL
+        ) THEN ARRAY['__unresolved__']
+          ELSE ARRAY[]::text[]
+        END
+    FROM (
+        SELECT model AS m FROM scan WHERE pk = target_scan_pk AND model IS NOT NULL
+        UNION
+        SELECT model AS m FROM model_role WHERE scan_pk = target_scan_pk
+        UNION
+        SELECT sm.model AS m FROM sample_model sm
+        JOIN sample s ON s.pk = sm.sample_pk
+        JOIN scanner_result sr ON sr.sample_pk = s.pk
+        WHERE sr.scan_pk = target_scan_pk
+    ) models
+    JOIN middleman.model md ON (
+            md.name = models.m
+            OR (models.m <> ''
+                AND right(md.name, char_length(models.m) + 1)
+                    = '/' || models.m)
+            OR (md.name <> ''
+                AND right(models.m, char_length(md.name) + 1)
+                    = '/' || md.name)
+        )
+    JOIN middleman.model_group mg ON mg.pk = md.model_group_pk
+$$
+"""
+
+
+# This bulk pass never recomputes a row parked at the ``__unresolved__`` sentinel
+# (a deferred/failed import that must stay hidden): re-deriving could return an
+# empty array and flip it from hidden to world-visible, and a backfill has no way
+# to tell such a row from one it should fix. Same guard as `c7d8e9f0a1b2`. The
+# runtime path is unguarded on purpose -- see `hawk.core.db.scan_writes`.
+_RECOMPUTE_SCAN = """
+UPDATE scan AS s
+SET model_groups = compute_scan_model_groups(s.pk)
+WHERE s.model_groups IS DISTINCT FROM compute_scan_model_groups(s.pk)
+  AND (s.model_groups IS NULL OR NOT (s.model_groups @> ARRAY['__unresolved__']))
+"""
+
+
+def upgrade() -> None:
+    op.execute(_COMPUTE_SCAN_MODEL_GROUPS_NEW)
+    # The recompute calls compute_scan_model_groups() for every in-scope scan
+    # (WHERE), and again for each row it changes (SET). Each call walks that
+    # scan's scanner_result -> sample -> eval rows; on prd (5.1k scans, 8.2M
+    # scanner_result rows) one full pass measured ~17 min on the reader, so the
+    # 5-min default statement_timeout (connection.get_url_and_engine_args)
+    # cancelled it and failed the deploy. Like `3af9c05e1d76`, raise it for
+    # this transaction only. Keep it below the db_migrate.py 60-min poll cap:
+    # past that the harness stops the task and nothing commits either way.
+    op.execute("SET LOCAL statement_timeout = '50min'")
+    op.execute(_RECOMPUTE_SCAN)
+
+
+def downgrade() -> None:
+    # Asymmetric on purpose, following `c7d8e9f0a1b2`: reinstate the old body but
+    # do NOT recompute. Under the old body a recompute strips the inherited eval
+    # groups, re-publishing precisely the rows this migration restricted -- which
+    # the `upgrade head && downgrade -1 && upgrade head` loop would do routinely.
+    # Leaving the cache stricter stays reversible: re-upgrading rebuilds it.
+    op.execute(_COMPUTE_SCAN_MODEL_GROUPS_OLD)

{hawk-3.2.0 → hawk-3.3.0}/hawk/core/db/functions.py

@@ -520,31 +520,60 @@ refresh_eval_model_groups_ddls: Final = [
 
 # --- Denormalized model_groups for scan ---
 
-# Mirrors compute_eval_model_groups, from scan.model, model_role (scan_pk), and
-# sample_model via scanner_result. Appends `__unresolved__` when a scanner_result
-# is missing its sample_pk (groups are then an incomplete lower bound).
+# The scan's own models, unioned with the groups of the evals it read. Empty =
+# fail-open (visible to all), so a missed group is a leak.
 COMPUTE_SCAN_MODEL_GROUPS_BODY: Final = f"""\
-SELECT COALESCE(array_agg(DISTINCT mg.name ORDER BY mg.name), ARRAY[]::text[])
-    || CASE WHEN EXISTS (
+SELECT COALESCE(array_agg(DISTINCT g.name ORDER BY g.name), ARRAY[]::text[])
+FROM (
+    -- (a) The scan's own models. The middleman join is prefix-tolerant (see
+    -- model_name_match_sql), as in compute_eval_model_groups.
+    SELECT mg.name
+    FROM (
+        SELECT model AS m FROM scan WHERE pk = target_scan_pk AND model IS NOT NULL
+        UNION
+        SELECT model AS m FROM model_role WHERE scan_pk = target_scan_pk
+        UNION
+        SELECT sm.model AS m FROM sample_model sm
+        JOIN sample s ON s.pk = sm.sample_pk
+        JOIN scanner_result sr ON sr.sample_pk = s.pk
+        WHERE sr.scan_pk = target_scan_pk
+    ) models
+    JOIN middleman.model md ON {model_name_match_sql("md.name", "models.m")}
+    JOIN middleman.model_group mg ON mg.pk = md.model_group_pk
+
+    UNION
+
+    -- (b) The source-side restriction: a scan is no less restricted than the
+    -- transcripts it read. Reusing `eval.model_groups` rather than re-deriving
+    -- it keeps the two sides from drifting, and inherits whatever the eval side
+    -- later learns.
+    --
+    -- NOT fail-closed in general: `eval.model_groups` is itself empty for an
+    -- eval whose models resolve to no group, and empty reads as unrestricted.
+    -- What this closes is a restricted model registered while `sample_model`
+    -- lags; making the eval side fail-closed is METR/hawk#1450.
+    --
+    -- `unnest` of a NULL `model_groups` yields no rows, so an unrestricted
+    -- eval contributes nothing rather than dropping the whole row.
+    SELECT source.name
+    FROM scanner_result sr
+    JOIN sample s ON s.pk = sr.sample_pk
+    JOIN eval e ON e.pk = s.eval_pk
+    CROSS JOIN LATERAL unnest(e.model_groups) AS source(name)
+    WHERE sr.scan_pk = target_scan_pk
+
+    UNION
+
+    -- (c) A transcript with no warehouse row at all: (b) cannot see an eval
+    -- that was never resolved to a sample.
+    SELECT '__unresolved__'::text
+    WHERE EXISTS (
         SELECT 1 FROM scanner_result sr
         WHERE sr.scan_pk = target_scan_pk
           AND sr.transcript_source_type = 'eval_log'
           AND sr.sample_pk IS NULL
-    ) THEN ARRAY['__unresolved__']
-      ELSE ARRAY[]::text[]
-    END
-FROM (
-    SELECT model AS m FROM scan WHERE pk = target_scan_pk AND model IS NOT NULL
-    UNION
-    SELECT model AS m FROM model_role WHERE scan_pk = target_scan_pk
-    UNION
-    SELECT sm.model AS m FROM sample_model sm
-    JOIN sample s ON s.pk = sm.sample_pk
-    JOIN scanner_result sr ON sr.sample_pk = s.pk
-    WHERE sr.scan_pk = target_scan_pk
-) models
-JOIN middleman.model md ON {model_name_match_sql("md.name", "models.m")}
-JOIN middleman.model_group mg ON mg.pk = md.model_group_pk\
+    )
+) g\
 """
 
 

hawk-3.3.0/hawk/core/db/scan_writes.py

@@ -0,0 +1,58 @@
+"""Writes to `scan` rows that are owned by something other than the scan itself."""
+
+from __future__ import annotations
+
+import typing
+
+import sqlalchemy
+from sqlalchemy import sql
+from sqlalchemy.dialects import postgresql
+from sqlmodel import col
+
+from hawk.core.db import models
+
+if typing.TYPE_CHECKING:
+    import uuid
+
+    import sqlalchemy.ext.asyncio as async_sa
+
+
+async def recompute_dependent_scan_model_groups(
+    session: async_sa.AsyncSession, eval_pk: uuid.UUID
+) -> None:
+    """Re-stamp the ACL of every scan that read `eval_pk`'s samples."""
+    # The `scan.model_groups` triggers fire on `scan`, `model_role` and
+    # `scanner_result` -- never on the eval-side inputs the ACL is derived from,
+    # so without this call a scan computed while its source eval was still
+    # importing keeps that answer forever.
+    #
+    # Deliberately not a `sample_model` row trigger: that is the O(n^2)
+    # full-scan aggregate the `hawk.defer_scan_model_groups` GUC exists to
+    # avoid. This runs once per settling eval, bounded by the number of scans
+    # over it, which is normally zero.
+    #
+    # Unguarded on `__unresolved__`, unlike the a726f26fdd42 backfill: that
+    # guard is a backfill-only rule, because a bulk pass cannot tell a row it
+    # should leave alone from one it should fix. Here the eval that parked the
+    # sentinel is the eval now settling, so recomputing it is the point --
+    # `refresh_scan_model_groups` and the scan writer's own finalize are
+    # likewise unguarded.
+    dependent_scans = (
+        sqlalchemy.select(col(models.ScannerResult.scan_pk))
+        .join(
+            models.Sample,
+            col(models.Sample.pk) == col(models.ScannerResult.sample_pk),
+        )
+        .where(col(models.Sample.eval_pk) == eval_pk)
+        .distinct()
+    )
+    await session.execute(
+        sqlalchemy.update(models.Scan)
+        .where(col(models.Scan.pk).in_(dependent_scans))
+        .values(
+            model_groups=sql.func.compute_scan_model_groups(
+                col(models.Scan.pk),
+                type_=postgresql.ARRAY(sqlalchemy.Text),
+            )
+        )
+    )

{hawk-3.2.0 → hawk-3.3.0}/hawk/core/importer/eval/writer/postgres.py

@@ -17,7 +17,7 @@ from sqlalchemy.dialects import postgresql
 from sqlmodel import col
 
 import hawk.core.exceptions as hawk_exceptions
-from hawk.core.db import models, serialization, upsert
+from hawk.core.db import models, scan_writes, serialization, upsert
 from hawk.core.exceptions import exception_context
 from hawk.core.importer.eval import externalization, records, writer
 
@@ -262,6 +262,15 @@ class PostgresWriter(writer.EvalLogWriter):
         await _commit_import_status(
             session=self.session, eval_db_pk=self.eval_pk, status="success"
         )
+        # Deliberately after the stamp commits, not inside its transaction: the
+        # recompute can wait on scan row locks, and a block or timeout there
+        # must not cost the stamp (an unstamped eval is re-imported in full).
+        # The cost of the split is a window where the scan ACL trails the eval
+        # groups it is derived from; it closes on the next settle of this eval.
+        await scan_writes.recompute_dependent_scan_model_groups(
+            self.session, self.eval_pk
+        )
+        await self.session.commit()
 
         logger.info(
             "Eval import committed",
@@ -289,6 +298,13 @@ class PostgresWriter(writer.EvalLogWriter):
             await _commit_import_status(
                 session=self.session, eval_db_pk=self.eval_pk, status="failed"
             )
+            # A failed import still settles the eval's groups, and nothing else
+            # would repair a scan left stale against them. After the stamp, for
+            # the same reason as in finalize().
+            await scan_writes.recompute_dependent_scan_model_groups(
+                self.session, self.eval_pk
+            )
+            await self.session.commit()
         except Exception as e:  # noqa: BLE001
             logger.error(
                 "Failed to mark eval import as failed; original error preserved",