inspect-steward 0.2.0__tar.gz

inspect_steward-0.2.0/.gitignore

@@ -0,0 +1,225 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv*
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# pyright
+.pyright/
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# mypy type checker
+.mypy_cache/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer,
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Dynamic version
+src/inspect_steward/_version.py
+
+# Inspect logs and scans
+/logs/
+/logs-*/
+/runs/
+/scans/
+/transcripts/
+
+# MacOS
+.DS_Store
+
+# Claude Code
+**/.claude/settings.local.json
+.claude
+
+# Quarto Docs
+/.luarc.json
+

inspect_steward-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Meridian Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

inspect_steward-0.2.0/PKG-INFO

@@ -0,0 +1,54 @@
+Metadata-Version: 2.5
+Name: inspect_steward
+Version: 0.2.0
+Summary: An agent that supervises evaluations, monitoring progress, diagnosing problems, and adjusting runtime state on your behalf.
+Project-URL: Source Code, https://github.com/meridianlabs-ai/inspect_steward
+Project-URL: Issue Tracker, https://github.com/meridianlabs-ai/inspect_steward/issues
+Author: Meridian Labs
+License: MIT License
+License-File: LICENSE
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: POSIX :: Linux
+Requires-Python: >=3.12
+Requires-Dist: apprise>=1.9
+Requires-Dist: click>=8.0
+Requires-Dist: httpx>=0.27
+Requires-Dist: inspect-ai>=0.3.263
+Requires-Dist: inspect-scout>=0.5.0
+Requires-Dist: platformdirs>=4.0
+Requires-Dist: psutil>=5.9
+Requires-Dist: python-dotenv>=1.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: flow
+Requires-Dist: inspect-flow>=0.12.0; extra == 'flow'
+Provides-Extra: hawk
+Requires-Dist: hawk[cli,runner]>=2.4.0; (python_version >= '3.13') and extra == 'hawk'
+Requires-Dist: uv>=0.5; (python_version >= '3.13') and extra == 'hawk'
+Description-Content-Type: text/markdown
+
+# Inspect Steward
+
+Welcome to Inspect Steward, an agent that supervises evaluations on your behalf.
+
+Steward plays the role of the human sitting in front of a running eval. It:
+
+-   Watches progress as a run unfolds
+-   Diagnoses problems as they surface
+-   Adjusts runtime state in response
+
+A steward acts on behalf of someone who isn't there. That's the relationship this package models: you start a run and step away, and the steward minds it.
+
+Concretely, Steward runs each task of an eval set in its own process — so one crash costs one task rather than the run, and CPU-bound work actually runs in parallel — and reconciles the log directory against what your definition asks for on a schedule. A timer guarantees that happens with nobody watching; what needs judgement waits for someone who can exercise it.
+
+```bash
+pip install inspect-steward
+
+steward init my-sweep     # create a workspace
+steward tasks evalset.py  # see what a definition resolves to, before running it
+steward launch            # start it, arm the timer, and return
+steward status            # ask how it's going, whenever you like
+```
+
+Steward runs on macOS and Linux. Windows is not supported: a run has to outlive the process that started it, and the detached-process, control-socket, and process-table mechanisms that achieve it are POSIX-only.
+
+Learn more about using Steward at <https://meridianlabs-ai.github.io/inspect_steward>.

inspect_steward-0.2.0/README.md

@@ -0,0 +1,26 @@
+# Inspect Steward
+
+Welcome to Inspect Steward, an agent that supervises evaluations on your behalf.
+
+Steward plays the role of the human sitting in front of a running eval. It:
+
+-   Watches progress as a run unfolds
+-   Diagnoses problems as they surface
+-   Adjusts runtime state in response
+
+A steward acts on behalf of someone who isn't there. That's the relationship this package models: you start a run and step away, and the steward minds it.
+
+Concretely, Steward runs each task of an eval set in its own process — so one crash costs one task rather than the run, and CPU-bound work actually runs in parallel — and reconciles the log directory against what your definition asks for on a schedule. A timer guarantees that happens with nobody watching; what needs judgement waits for someone who can exercise it.
+
+```bash
+pip install inspect-steward
+
+steward init my-sweep     # create a workspace
+steward tasks evalset.py  # see what a definition resolves to, before running it
+steward launch            # start it, arm the timer, and return
+steward status            # ask how it's going, whenever you like
+```
+
+Steward runs on macOS and Linux. Windows is not supported: a run has to outlive the process that started it, and the detached-process, control-socket, and process-table mechanisms that achieve it are POSIX-only.
+
+Learn more about using Steward at <https://meridianlabs-ai.github.io/inspect_steward>.

inspect_steward-0.2.0/pyproject.toml

@@ -0,0 +1,239 @@
+[project]
+name = "inspect_steward"
+dynamic = ["version"]
+description = "An agent that supervises evaluations, monitoring progress, diagnosing problems, and adjusting runtime state on your behalf."
+authors = [{ name = "Meridian Labs" }]
+readme = "README.md"
+requires-python = ">=3.12"
+license = { text = "MIT License" }
+# POSIX only, and deliberately so: Steward's central requirement is that a run
+# outlives the process that started it, which it achieves by spawning workers
+# into their own session. Windows has no equivalent that `subprocess` reaches
+# the same way, and the rest of the design leans the same direction (AF_UNIX
+# control sockets, process-table identification). Supporting it would be a
+# second execution model rather than a flag, so it is declined rather than
+# deferred. Classifiers are metadata, not enforcement — pip will still install
+# here; nothing pretends the guarantee holds.
+classifiers = [
+    "Operating System :: POSIX :: Linux",
+    "Operating System :: MacOS",
+]
+dependencies = [
+    # the released wheels carry what this project needs since 2026-09-04 --
+    # record-only selection scanning, the ambient scan-model default, the
+    # `log_dir` default for `eval_set()`, scout's unique summary tmp names --
+    # so these floors are the whole story; only `inspect-harbor` still resolves
+    # through `[tool.uv.sources]` below, for the reason given there
+    "inspect-ai>=0.3.263",
+    "inspect-scout>=0.5.0",
+    "click>=8.0",
+    "pyyaml>=6.0",
+    # the process table is how Steward finds a worker that has not yet reached
+    # its eval, and so has no log and no control-discovery entry. inspect-ai
+    # depends on psutil too, but this is a direct import and a transitive
+    # dependency is not a promise.
+    "psutil>=5.9",
+    # a worker's control server is HTTP over an AF_UNIX socket, and reading a
+    # running fleet for `status` goes to it directly rather than through
+    # `inspect ctl` (see _worker/live.py). inspect-ai depends on httpx too, but
+    # this is a direct import and a transitive dependency is not a promise.
+    "httpx>=0.27",
+    # arming a timer checks which credentials a workspace's `.env` really
+    # defines, and only the resolved value answers that -- `KEY=""` and
+    # `KEY=${UNSET}` both look like assignments and both arrive as nothing. Read
+    # with the same parser inspect will use so the two cannot disagree.
+    # inspect-ai depends on python-dotenv too, but this is a direct import and a
+    # transitive dependency is not a promise.
+    "python-dotenv>=1.0",
+    # notification is the gate on autonomy rather than a nicety (workflow.md
+    # §11): an unattended run that cannot reach a person is the failure the
+    # whole design exists to prevent. inspect-ai makes apprise optional because
+    # an eval that never asks a human needs no channel; a supervisor always
+    # might, so a workspace must not be able to configure a channel Steward
+    # then cannot reach.
+    "apprise>=1.9",
+    # `log_store: auto` names the machine's own store, which is a per-platform
+    # data directory rather than a path Steward could hardcode. inspect-flow
+    # depends on platformdirs too — and resolving `auto` through *flow* would
+    # make the common case pay the `[flow]` extra to find out where it is
+    # pointing, which is the whole thing the directory implementation exists to
+    # avoid. A direct import, and a transitive dependency is not a promise.
+    "platformdirs>=4.0",
+]
+
+[project.optional-dependencies]
+flow = ["inspect_flow>=0.12.0"]
+# cli: the `python -m hawk` click entry point Steward invokes.
+# runner: hawk.runner.run_eval_set imports k8s_sandbox at module scope.
+# uv: hawk shells out to `uv pip install` on every invocation but does not
+#   declare it, so without this the extra installs and then fails at the first
+#   read with FileNotFoundError. Ours to carry until hawk declares it.
+#   Declaring it is necessary but not sufficient: pip puts the binary beside
+#   the interpreter, and hawk resolves a bare `uv` through PATH, which only
+#   contains that directory when the venv happens to be activated. The hawk
+#   branch of `definition_command` prepends it for the child process.
+#
+# Marker-gated because hawk requires Python 3.13 and inspect_steward does not.
+# An extra is opt-in, so hawk's floor binds only those who ask for hawk: on
+# 3.12 the extra resolves to nothing and hawk support is simply unavailable.
+#
+# No upper bound, but note the coupling it leaves open: the hawk test fixture
+# names `hawk/e2e_hello`, hawk's own internal smoke-test task, because
+# `--direct` installs nothing new and it is the only registered task available
+# without one. A hawk release that renames or drops it breaks that fixture with
+# no change on our side.
+hawk = [
+    "hawk[cli,runner]>=2.4.0; python_version >= '3.13'",
+    "uv>=0.5; python_version >= '3.13'",
+]
+
+[project.urls]
+"Source Code" = "https://github.com/meridianlabs-ai/inspect_steward"
+"Issue Tracker" = "https://github.com/meridianlabs-ai/inspect_steward/issues"
+
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project.entry-points.inspect_ai]
+inspect_steward = "inspect_steward._registry"
+
+[project.scripts]
+steward = "inspect_steward._cli.main:main"
+
+
+[dependency-groups]
+dev = [
+    "ruff",
+    "pytest",
+    "pyright",
+    "pytest-xdist",
+    "pytest-dotenv",
+    "pytest-timeout>=2.4.0",
+    "types-psutil", # psutil ships no inline types and pyright runs strict
+    "inspect_flow>=0.12.0", # for the flow definition tests
+    # for the hawk definition tests; 3.13+ only, so those tests skip on 3.12
+    "hawk[cli,runner]>=2.4.0; python_version >= '3.13'",
+    "uv>=0.5; python_version >= '3.13'",
+    # the eval-set definitions under runs/ import inspect_harbor and name
+    # openai/ and anthropic/ models. inspect-ai makes provider SDKs optional
+    # because an eval that never calls a provider needs none, so nothing else
+    # here pulls them in and a `uv sync` takes them back out again — which is
+    # how a workspace that ran clean an hour ago comes back
+    # ModuleNotFoundError. The floors are inspect's own gates
+    # (`validate_openai_client`, `validate_anthropic_client`), not guesses.
+    "openai>=3.1.0",
+    "anthropic>=1.0.0",
+    "inspect_harbor",
+    "inspect-sandboxes",
+    {include-group = "doc"},
+]
+doc = [
+    "jupyter",
+    "panflute",
+    "markdown",
+    "griffe>=2",
+    "rich",
+    "pyyaml",
+    "types-PyYAML",
+    "quarto-cli>=1.9.36",
+]
+dist = ["twine", "build"]
+
+[tool.hatch.build]
+only-packages = true
+exclude = ["docs/", "tests/"]
+artifacts = ["src/**/*.yaml", "src/**/*.md", "src/**/*.txt"]
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.build.hooks.vcs]
+version-file = "src/inspect_steward/_version.py"
+
+[tool.hatch.version.raw-options]
+local_scheme = "no-local-version" # Important for PyPI compatibility
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.uv]
+managed = true
+# openai, for a conflict that has no satisfying resolution: harbor depends on
+# litellm, which caps `openai>=2.20.0,<3.0.0`, while inspect-ai's OpenAI
+# provider gates on `openai>=3.1.0`. Left alone uv resolves it the only way it
+# can — by walking inspect-harbor back to 0.4.17 and harbor to 0.2.0, which
+# predate the registry tasks the runs/ definitions name, so the lock "succeeds"
+# and the eval set then fails to import its own tasks. The override says take
+# inspect's floor and let litellm's cap go unmet. That is a real claim, not a
+# formality: it ships a combination litellm declares unsupported. It holds
+# because nothing on the inspect path calls litellm — harbor's use of it is in
+# its own orchestrators, which Steward's runs never enter. Revisit when litellm
+# widens the cap; the pin can go the day `litellm` accepts openai 3.x.
+override-dependencies = ["openai"]
+
+[tool.uv.sources]
+
+# inspect-harbor from git rather than PyPI, because the registry tasks the
+# runs/ definitions name (cais/swebenchpro, datacurve/deep-swe) reach the
+# generated `_tasks.py` ahead of a release: the logs of the last clean run
+# record `inspect_harbor: 0.7.6.dev1`, a commit past the 0.7.5 wheel. A run
+# resolved to the wheel is a different eval, and the task identifier hash says
+# so.
+inspect-harbor = { git = "https://github.com/meridianlabs-ai/inspect_harbor", branch = "main" }
+
+[tool.ruff]
+extend-exclude = ["docs"]
+src = ["."]
+
+[tool.ruff.lint]
+select = [
+    "E",      # pycodestyle errors
+    "W",      # pycodestyle warnings
+    "F",      # flake8
+    "D",      # pydocstyle
+    "I",      # isort
+    "B",      # flake8-bugbear
+    "SIM101", # duplicate isinstance
+    # "RET", # flake8-return
+    # "RUF", # ruff rules
+]
+ignore = ["E203", "E501", "D10", "D212", "D415"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+testpaths = ["tests"]
+# `not network` by default, and the reason is isolation rather than speed:
+# hawk's CLI runs `uv pip install` into the *running interpreter*, which
+# uninstalls this project's editable inspect-ai and rewrites the package tree
+# from hawk's own git pin. Two consequences, both observed. Concurrently, a
+# sibling xdist worker importing inspect_ai mid-rewrite dies with a bare
+# ModuleNotFoundError nowhere near its own subject. Afterwards, the venv is
+# left pointing at hawk's snapshot, so every later run -- and every `steward`
+# invocation -- silently tests a different inspect than the checkout.
+# Opt in with `pytest -m network -n0`, and `uv sync` when it is done.
+addopts = "-n auto -m 'not network'"
+markers = [
+    # Requires network and the `uv` binary. Hawk's CLI runs `uv pip install`
+    # into the running interpreter on every invocation, resolving inspect-ai
+    # from a pinned git commit, so a hawk definition cannot be read offline
+    # -- and cannot be read without replacing what is installed. Deselected by
+    # default (see addopts above).
+    "network: needs network access, an external tool on PATH, and sole use of the environment",
+    # A marking run's real, detached runner. Every other test has the launch
+    # stood in for and applies the run in-process (see tests/conftest.py).
+    "runner: starts a real marking runner process",
+]
+
+[tool.pyright]
+include = ["src", "tests"]
+typeCheckingMode = "strict"
+pythonPlatform = "All"
+
+# tests may exercise private helpers directly
+[[tool.pyright.executionEnvironments]]
+root = "tests"
+reportPrivateUsage = false

inspect_steward-0.2.0/src/inspect_steward/__init__.py

@@ -0,0 +1,21 @@
+from ._evalset.manifest import (
+    Manifest,
+    ManifestSource,
+    ManifestTask,
+)
+from ._evalset.read import ReadEvalSetError, read_eval_set
+
+try:
+    from ._version import __version__
+except ImportError:
+    __version__ = "unknown"
+
+
+__all__ = [
+    "Manifest",
+    "ManifestSource",
+    "ManifestTask",
+    "ReadEvalSetError",
+    "read_eval_set",
+    "__version__",
+]

inspect_steward-0.2.0/src/inspect_steward/__main__.py

@@ -0,0 +1,9 @@
+"""`python -m inspect_steward` — the entry point a scheduler uses.
+
+The console script would be shorter, but it lives in a venv's `bin/` that a stripped cron or launchd environment has no reason to have on `PATH`. An absolute interpreter and a module name need no `PATH` at all, and a crontab line is read by people.
+"""
+
+from ._cli.main import main
+
+if __name__ == "__main__":
+    main()