inspect-flow 0.0.1__tar.gz

inspect_flow-0.0.1/.gitignore

@@ -0,0 +1,229 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+#!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+!*.code-workspace
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+# python version
+.python-version
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Dynamic version
+src/inspect_flow/_version.py
+
+# Inspect logs
+logs/
+
+# MacOS
+.DS_Store

inspect_flow-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Meridian Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

inspect_flow-0.0.1/PKG-INFO

@@ -0,0 +1,51 @@
+Metadata-Version: 2.4
+Name: inspect_flow
+Version: 0.0.1
+Summary: Inspect Flow is a workflow stack built on Inspect AI that enables research organizations to run AI evaluations at scale
+Project-URL: Source Code, https://github.com/meridianlabs-ai/inspect_flow
+Project-URL: Issue Tracker, https://github.com/meridianlabs-ai/inspect_flow/issues
+Author: Meridian Labs
+License: MIT License
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: click>=8.2.1
+Requires-Dist: inspect-ai>=0.3.131
+Requires-Dist: pydantic>=2.11.2
+Requires-Dist: python-dotenv>=1.1.1
+Requires-Dist: typing-extensions>=4.9.0
+Description-Content-Type: text/markdown
+
+Welcome to Inspect Flow. Inspect Flow is a workflow stack built on Inspect AI that enables research organizations to run AI evaluations at scale.
+
+## Installation
+
+Latest development version:
+
+```bash
+pip install git+https://github.com/meridianlabs-ai/inspect_flow
+```
+
+## Development
+
+To work on development of Inspect Flow, clone the repository and install with the `-e` flag and `[dev]` optional dependencies:
+
+```bash
+git clone https://github.com/meridianlabs-ai/inspect_flow
+cd inspect_flow
+uv sync
+```
+
+Optionally install pre-commit hooks via
+
+```bash
+make hooks
+```
+
+Run linting, formatting, and tests via
+
+```bash
+make check
+make test
+```
+
+

inspect_flow-0.0.1/README.md

@@ -0,0 +1,34 @@
+Welcome to Inspect Flow. Inspect Flow is a workflow stack built on Inspect AI that enables research organizations to run AI evaluations at scale.
+
+## Installation
+
+Latest development version:
+
+```bash
+pip install git+https://github.com/meridianlabs-ai/inspect_flow
+```
+
+## Development
+
+To work on development of Inspect Flow, clone the repository and install with the `-e` flag and `[dev]` optional dependencies:
+
+```bash
+git clone https://github.com/meridianlabs-ai/inspect_flow
+cd inspect_flow
+uv sync
+```
+
+Optionally install pre-commit hooks via
+
+```bash
+make hooks
+```
+
+Run linting, formatting, and tests via
+
+```bash
+make check
+make test
+```
+
+

inspect_flow-0.0.1/pyproject.toml

@@ -0,0 +1,112 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "inspect_flow"
+dynamic = ["version"]
+description = "Inspect Flow is a workflow stack built on Inspect AI that enables research organizations to run AI evaluations at scale"
+authors = [{ name = "Meridian Labs" }]
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT License" }
+dependencies = [
+    "click>=8.2.1",
+    "inspect-ai>=0.3.131",
+    "pydantic>=2.11.2",
+    "python-dotenv>=1.1.1",
+    "typing_extensions>=4.9.0",
+]
+
+[project.urls]
+"Source Code" = "https://github.com/meridianlabs-ai/inspect_flow"
+"Issue Tracker" = "https://github.com/meridianlabs-ai/inspect_flow/issues"
+
+[project.scripts]
+flow = "inspect_flow._cli.main:main"
+
+[dependency-groups]
+dev = [
+    "ruff",
+    "pytest",
+    "pytest-cov",
+    "anthropic",
+    "openai",
+    "pytest-asyncio",
+    "pytest-dotenv",
+    "pyright",
+    "types-pyyaml>=6.0.12.20250915",
+    "datamodel-code-generator",
+    "pre-commit",
+]
+
+
+[tool.hatch.build]
+only-packages = true
+exclude = ["docs/", "tests/"]
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.build.hooks.vcs]
+version-file = "src/inspect_flow/_version.py"
+
+[tool.hatch.version.raw-options]
+local_scheme = "no-local-version" # Important for PyPI compatibility
+
+[tool.uv]
+managed = true
+add-bounds = "lower"
+default-groups = ["dev"]
+
+[tool.ruff]
+extend-exclude = ["docs"]
+src = ["."]
+
+[tool.ruff.lint]
+select = [
+    "E",      # pycodestyle errors
+    "W",      # pycodestyle warnings
+    "F",      # flake8
+    "D",      # pydocstyle
+    "I",      # isort
+    "B",      # flake8-bugbear
+    "SIM101", # duplicate isinstance
+    # "RET", # flake8-return
+    # "RUF", # ruff rules
+]
+ignore = [
+    "E203",  # whitespace before ':'
+    "E501",  # line too long
+    "D105",  # missing docstring in magic method
+    "D107",  # missing docstring in __init__
+    "D212",  # multi-line docstring summary should start at the first line
+    "D415",  # first line should end with a period, question mark, or exclamation point
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*.py" = ["D"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+testpaths = ["tests"]
+
+[tool.coverage.run]
+source = ["src"]
+omit = ["*/tests/*", "*/__pycache__/*", "*/test_*.py"]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "@abstractmethod",
+]
+show_missing = true
+precision = 2

inspect_flow-0.0.1/src/inspect_flow/README.md

@@ -0,0 +1,54 @@
+# Inspect Flow Internals
+
+## _types Module
+
+[_types](./_types) defines the types used in inspect flow configurations. 
+
+[flow_config.py](./_types/flow_config.py) defines the pydantic types for the configuration. These types have a "Flow" prefix, e.g. `FlowTask`.
+
+[type_gen.py](./_types/type_gen.py) defines the code generation logic that generates TypedDict classes based on the pydantic types.
+These generated types are in [generated.py](./_types/generated.py).
+
+`FlowTaskDict` and other types with a `Dict` suffix are TypedDicts corresponding to the Pydantic types.
+These are used primarily to unpack kwargs in the _with functions.
+`FlowTaskMatrixDict` and the other types with a `MatrixDict` suffix are TypedDicts that store lists instead of single values.
+These are for use in the matrix functions with lists for their field types.
+
+[factories.py](./_types/factories.py) defines three types of functions.
+The `_with` functions apply fields to all objects in a list. For example `task_with` sets fields on a list of tasks (specified as a list of string, `FTask`, `FlowTask`, and `FlowTaskDict`).
+The `_matrix` functions, like `tasks_matrix` generate lists of types from the product of lists of field values. 
+
+## _api Module
+
+[_api](./_api) defines the public API for inspect flow. 
+This includes the main entry points for running flows and interacting with the framework.
+These functions correspond to the CLI commands defined in the [_cli](./_cli) module.
+
+## _cli Module
+
+[_cli](./_cli) defines the command line interface for inspect flow. The main command is `run`, defined in [run.py](./_cli/run.py).
+
+## _config Module
+
+[_config](./_config) is responsible for loading and validating the flow configuration. The main function is `load_config`, defined in [config.py](./_config/config.py).
+
+## _launcher Module
+
+[_launcher](./_launcher) is responsible for creating the virtual environment, installing package dependencies, and starting the flow runner process.
+The main function is `launch`, defined in [launch.py](./_launcher/launch.py).
+
+## _runner Module
+
+[_runner](./_runner) is responsible for running the flow tasks within the virtual environment.
+The main function is `flow_run`, defined in [run.py](./_runner/run.py).
+This involves:
+1. Resolving the flow configuration into a canonical representation of the tasks with all defaults explicitly set.
+This may require loading source files to determine the exported task functions.
+2. Instantiating the tasks and all dependencies.
+This converts from the configuration into the inspect AI objects required to run the tasks.
+3. Running the tasks.
+Currently eval_set is used, although this may change in the future.
+
+## _util Module
+
+[_util](./_util) defines utility functions and classes used throughout the inspect flow codebase.

inspect_flow-0.0.1/src/inspect_flow/__init__.py

@@ -0,0 +1,57 @@
+"""inspect_flow methods for constructing flow configs."""
+
+try:
+    from ._version import __version__
+except ImportError:
+    __version__ = "unknown"
+
+from inspect_flow._types.factories import (
+    agents_matrix,
+    agents_with,
+    configs_matrix,
+    configs_with,
+    models_matrix,
+    models_with,
+    solvers_matrix,
+    solvers_with,
+    tasks_matrix,
+    tasks_with,
+)
+from inspect_flow._types.flow_types import (
+    FlowAgent,
+    FlowDefaults,
+    FlowEpochs,
+    FlowGenerateConfig,
+    FlowJob,
+    FlowModel,
+    FlowOptions,
+    FlowSolver,
+    FlowTask,
+)
+from inspect_flow._types.merge import (
+    merge,
+)
+
+__all__ = [
+    "__version__",
+    "FlowAgent",
+    "FlowJob",
+    "FlowDefaults",
+    "FlowEpochs",
+    "FlowGenerateConfig",
+    "FlowModel",
+    "FlowOptions",
+    "FlowSolver",
+    "FlowTask",
+    "agents_matrix",
+    "agents_with",
+    "configs_matrix",
+    "configs_with",
+    "merge",
+    "models_matrix",
+    "models_with",
+    "solvers_matrix",
+    "solvers_with",
+    "tasks_matrix",
+    "tasks_with",
+]

inspect_flow-0.0.1/src/inspect_flow/_api/api.py

@@ -0,0 +1,36 @@
+import click
+
+from inspect_flow._config.write import config_to_yaml
+from inspect_flow._launcher.launch import launch
+from inspect_flow._types.flow_types import FlowJob
+
+
+def run(
+    config: FlowJob,
+    dry_run: bool = False,
+) -> None:
+    """Run an inspect_flow evaluation.
+
+    Args:
+        config: The flow configuration.
+        dry_run: If True, do not run eval, but show a count of tasks that would be run.
+    """
+    run_args = ["--dry-run"] if dry_run else []
+    launch(config=config, run_args=run_args)
+
+
+def config(
+    config: FlowJob,
+    resolve: bool = False,
+) -> None:
+    """Print the flow configuration.
+
+    Args:
+        config: The flow configuration.
+        resolve: If True, resolve the configuration before printing.
+    """
+    if resolve:
+        launch(config=config, run_args=["--config"])
+    else:
+        dump = config_to_yaml(config)
+        click.echo(dump)

inspect_flow-0.0.1/src/inspect_flow/_cli/config.py

@@ -0,0 +1,30 @@
+import click
+from typing_extensions import Unpack
+
+from inspect_flow._api.api import config
+from inspect_flow._cli.options import (
+    ConfigOptionArgs,
+    config_options,
+    parse_config_options,
+)
+from inspect_flow._config.load import load_config
+
+
+@click.command("config", help="Output config")
+@click.argument("config-file", type=str, required=True)
+@click.option(
+    "--resolve",
+    type=bool,
+    is_flag=True,
+    envvar="INSPECT_FLOW_RESOLVE",
+    help="Fully resolve the config. Will create a venv and create all objects.",
+)
+@config_options
+def config_command(
+    config_file: str,
+    resolve: bool,
+    **kwargs: Unpack[ConfigOptionArgs],
+) -> None:
+    config_options = parse_config_options(**kwargs)
+    fconfig = load_config(config_file, config_options=config_options)
+    config(fconfig, resolve=resolve)

inspect_flow-0.0.1/src/inspect_flow/_cli/main.py

@@ -0,0 +1,45 @@
+import click
+from dotenv import find_dotenv, load_dotenv
+
+from inspect_flow._cli.config import config_command
+
+from .. import __version__
+from .run import run_command
+
+
+@click.group(invoke_without_command=True)
+@click.option(
+    "--version",
+    type=bool,
+    is_flag=True,
+    default=False,
+    help="Print the flow version.",
+)
+@click.pass_context
+def flow(
+    ctx: click.Context,
+    version: bool,
+) -> None:
+    # if this was a subcommand then allow it to execute
+    if ctx.invoked_subcommand is not None:
+        return
+
+    if version:
+        click.echo(__version__)
+        ctx.exit()
+    else:
+        click.echo(ctx.get_help())
+        ctx.exit()
+
+
+flow.add_command(run_command)
+flow.add_command(config_command)
+
+
+def main() -> None:
+    load_dotenv(find_dotenv(usecwd=True))
+    flow(auto_envvar_prefix="INSPECT_FLOW")
+
+
+if __name__ == "__main__":
+    main()