inspect-mlflow 0.1.0__tar.gz

inspect_mlflow-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,60 @@
+name: Bug Report
+description: Report a bug in inspect-mlflow
+labels: [bug]
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: What happened?
+      description: Describe the bug clearly. Include error messages if any.
+    validations:
+      required: true
+
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: Steps to reproduce
+      description: Minimal code or commands to reproduce the issue.
+      render: python
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: What you expected to happen.
+    validations:
+      required: true
+
+  - type: input
+    id: version
+    attributes:
+      label: inspect-mlflow version
+      placeholder: "0.1.0"
+    validations:
+      required: true
+
+  - type: input
+    id: inspect-version
+    attributes:
+      label: inspect-ai version
+      placeholder: "0.3.190"
+    validations:
+      required: true
+
+  - type: input
+    id: mlflow-version
+    attributes:
+      label: MLflow version
+      placeholder: "3.10.1"
+    validations:
+      required: true
+
+  - type: input
+    id: python-version
+    attributes:
+      label: Python version
+      placeholder: "3.12"
+    validations:
+      required: true

inspect_mlflow-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,28 @@
+name: Feature Request
+description: Suggest a new feature or improvement
+labels: [enhancement]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem or use case
+      description: What problem does this solve? Why do you need it?
+    validations:
+      required: true
+
+  - type: textarea
+    id: solution
+    attributes:
+      label: Proposed solution
+      description: How should it work? Include code examples if helpful.
+      render: python
+    validations:
+      required: false
+
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives considered
+      description: Other approaches you thought about and why they don't work.
+    validations:
+      required: false

inspect_mlflow-0.1.0/.github/dependabot.yml

@@ -0,0 +1,15 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 5
+    labels: [dependencies]
+
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 5
+    labels: [dependencies, ci]

inspect_mlflow-0.1.0/.github/pull_request_template.md

@@ -0,0 +1,14 @@
+## What changed and why
+
+## Testing
+
+- [ ] `uv run pytest tests/ -v` passes
+- [ ] `uv run ruff check .` clean
+- [ ] `uv run mypy inspect_mlflow/` clean
+- [ ] New tests added for new functionality
+
+## Checklist
+
+- [ ] Read [CONTRIBUTING.md](CONTRIBUTING.md)
+- [ ] Pre-commit hooks pass (`uv run pre-commit run --all-files`)
+- [ ] No breaking changes (or documented in description)

inspect_mlflow-0.1.0/.github/workflows/api-compat.yml

@@ -0,0 +1,48 @@
+name: API Compatibility
+
+on:
+  schedule:
+    - cron: "0 0 * * 0" # Sunday midnight UTC
+  workflow_dispatch:
+
+jobs:
+  test-latest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+      - name: Install with latest deps
+        run: |
+          uv sync --group dev
+          uv pip install --upgrade inspect-ai mlflow
+      - name: Run tests
+        run: uv run pytest tests/ -v --tb=short
+      - name: Report versions
+        if: always()
+        run: |
+          echo "inspect-ai: $(uv run python -c 'import inspect_ai; print(inspect_ai.__version__)')"
+          echo "mlflow: $(uv run python -c 'import mlflow; print(mlflow.__version__)')"
+
+      - name: Create issue on failure
+        if: failure()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const title = `API compatibility failure: ${new Date().toISOString().slice(0, 10)}`;
+            const existing = await github.rest.issues.listForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              state: 'open',
+              labels: 'api-compat',
+            });
+            if (existing.data.length === 0) {
+              await github.rest.issues.create({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                title,
+                body: `Tests failed against latest inspect-ai and mlflow.\n\nSee: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
+                labels: ['bug', 'api-compat'],
+              });
+            }

inspect_mlflow-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,64 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+      - run: uv sync --group dev
+      - run: uv run ruff check .
+      - run: uv run ruff format --check .
+      - run: uv run mypy inspect_mlflow/
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+      - run: uv python install ${{ matrix.python-version }}
+      - run: uv sync --group dev --python ${{ matrix.python-version }}
+      - name: Run tests with coverage
+        run: uv run pytest tests/ -v --tb=short --cov=inspect_mlflow --cov-report=xml --cov-report=term-missing
+      - name: Enforce coverage threshold
+        run: uv run coverage report --fail-under=70
+      - name: Upload coverage
+        if: matrix.python-version == '3.12' && github.event_name == 'push'
+        uses: codecov/codecov-action@v4
+        with:
+          file: coverage.xml
+          fail_ci_if_error: false
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: hynek/build-and-inspect-python-package@v2
+
+  check:
+    if: always()
+    needs: [lint, test, build]
+    runs-on: ubuntu-latest
+    steps:
+      - name: All jobs passed
+        uses: re-actors/alls-green@release/v1
+        with:
+          jobs: ${{ toJSON(needs) }}

inspect_mlflow-0.1.0/.github/workflows/codeql.yml

@@ -0,0 +1,21 @@
+name: CodeQL
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: "0 6 * * 1"
+
+jobs:
+  analyze:
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: github/codeql-action/init@v3
+        with:
+          languages: python
+      - uses: github/codeql-action/analyze@v3

inspect_mlflow-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,50 @@
+name: Release
+
+on:
+  push:
+    tags: ["v*"]
+
+permissions:
+  contents: write
+  id-token: write
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+      - run: uv sync --group dev
+      - run: uv run pytest tests/ -v --tb=short
+
+  build:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: hynek/build-and-inspect-python-package@v2
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: release
+    steps:
+      - name: Download dist
+        uses: actions/download-artifact@v4
+        with:
+          name: Packages
+          path: dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    needs: publish
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Create GitHub Release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh release create ${{ github.ref_name }} dist/* --generate-notes

inspect_mlflow-0.1.0/.gitignore

@@ -0,0 +1,17 @@
+__pycache__/
+*.py[cod]
+*$py.class
+*.egg-info/
+dist/
+build/
+.eggs/
+*.egg
+.venv/
+.env
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+mlflow.db
+mlartifacts/
+mlruns/
+.coverage

inspect_mlflow-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,25 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.14.0
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: check-added-large-files
+        args: [--maxkb=500]
+      - id: no-commit-to-branch
+        args: [--branch=main]
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.1
+    hooks:
+      - id: codespell
+        args: [--skip=uv.lock]

inspect_mlflow-0.1.0/CHANGELOG.md

@@ -0,0 +1,10 @@
+# Changelog
+
+## 0.1.0 (2026-03-20)
+
+Initial release.
+
+- MLflow tracking hook: hierarchical runs, per-sample metrics, model usage, artifact logging
+- MLflow tracing hook: execution span tree with model calls, tool calls, scoring
+- Auto-registration via Inspect AI entry points
+- 33 unit tests

inspect_mlflow-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,67 @@
+# Contributing
+
+## Development Setup
+
+```bash
+git clone https://github.com/debu-sinha/inspect-mlflow.git
+cd inspect-mlflow
+uv sync --group dev
+uv run pre-commit install
+```
+
+## Running Tests
+
+```bash
+uv run pytest tests/ -v
+uv run pytest tests/ --cov=inspect_mlflow --cov-report=term-missing
+```
+
+## Linting
+
+```bash
+uv run ruff check .
+uv run ruff format .
+uv run mypy inspect_mlflow/
+```
+
+## Pre-commit
+
+Pre-commit hooks run automatically on `git commit`. To run manually:
+
+```bash
+uv run pre-commit run --all-files
+```
+
+## Integration Testing
+
+To test with a real MLflow server and OpenAI API:
+
+```bash
+mlflow server --port 5556
+export MLFLOW_TRACKING_URI="http://127.0.0.1:5556"
+export MLFLOW_INSPECT_TRACING="true"
+export OPENAI_API_KEY="sk-..."
+
+uv run python -c "
+from inspect_ai import Task, eval
+from inspect_ai.dataset import Sample
+from inspect_ai.scorer import match
+from inspect_ai.solver import generate
+
+task = Task(
+    dataset=[Sample(input='What is 2+2?', target='4')],
+    solver=generate(),
+    scorer=match(),
+)
+eval(task, model='openai/gpt-4o-mini')
+"
+```
+
+Open http://127.0.0.1:5556 to see runs and traces.
+
+## Pull Requests
+
+- Keep PRs focused on a single change
+- Include tests for new functionality
+- Run `uv run pre-commit run --all-files` before pushing
+- All CI checks must pass before merge

inspect_mlflow-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Debu Sinha
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

inspect_mlflow-0.1.0/PKG-INFO

@@ -0,0 +1,161 @@
+Metadata-Version: 2.4
+Name: inspect-mlflow
+Version: 0.1.0
+Summary: MLflow integration for Inspect AI: experiment tracking, execution tracing, and Scout analysis
+Project-URL: Homepage, https://github.com/debu-sinha/inspect-mlflow
+Project-URL: Issues, https://github.com/debu-sinha/inspect-mlflow/issues
+Project-URL: Repository, https://github.com/debu-sinha/inspect-mlflow
+Author-email: Debu Sinha <debusinha2009@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Requires-Dist: inspect-ai>=0.3.180
+Requires-Dist: mlflow<4.0,>=3.0
+Provides-Extra: scout
+Requires-Dist: inspect-scout>=0.1.0; extra == 'scout'
+Description-Content-Type: text/markdown
+
+# inspect-mlflow
+
+![logo](https://raw.githubusercontent.com/debu-sinha/inspect-mlflow/main/docs/images/logo.png)
+
+[![CI](https://github.com/debu-sinha/inspect-mlflow/actions/workflows/ci.yml/badge.svg)](https://github.com/debu-sinha/inspect-mlflow/actions/workflows/ci.yml)
+[![CodeQL](https://github.com/debu-sinha/inspect-mlflow/actions/workflows/codeql.yml/badge.svg)](https://github.com/debu-sinha/inspect-mlflow/actions/workflows/codeql.yml)
+[![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)
+
+MLflow integration for [Inspect AI](https://inspect.aisi.org.uk/). Provides experiment tracking, execution tracing, and artifact logging for Inspect AI evaluations.
+
+## Install
+
+```bash
+pip install inspect-mlflow
+```
+
+## Quick Start
+
+No code changes needed. Hooks auto-register via entry points when the package is installed. Set env vars and run evals as usual.
+
+```bash
+# Start MLflow server
+mlflow server --port 5000
+
+# Set env vars
+export MLFLOW_TRACKING_URI="http://localhost:5000"
+export MLFLOW_INSPECT_TRACING="true"
+
+# Run evals. Hooks auto-activate.
+inspect eval my_task.py --model openai/gpt-4o
+```
+
+Then open http://localhost:5000 to see runs and traces.
+
+## What it does
+
+### Tracking Hook
+
+Activated when `MLFLOW_TRACKING_URI` is set. Creates hierarchical MLflow runs mirroring the eval structure.
+
+- Parent run per eval invocation, nested child runs per task
+- Task config logged as parameters (model, dataset, solver, temperature)
+- Per-sample scores as step metrics
+- Model token usage (input/output/total per model)
+- Real-time event counting (model calls, tool calls)
+- Eval artifacts: per-sample results JSON + full eval log JSON
+
+### Tracing Hook
+
+Activated when `MLFLOW_INSPECT_TRACING=true` is also set. Maps eval execution to MLflow trace spans.
+
+```
+eval_run:6fvmKSZv (CHAIN)
+  task:task (CHAIN)
+    sample:gM9UtEAM (CHAIN)
+      solvers -> generate -> model:openai/gpt-4o-mini (LLM)
+      scorers -> match -> score (EVALUATOR)
+    sample:628Qbuhr (CHAIN)
+      ...
+```
+
+Each span captures relevant data:
+
+| Span Type | Data |
+|-----------|------|
+| LLM | model name, token counts, temperature, cache, response |
+| TOOL | function name, arguments, result, errors |
+| EVALUATOR | score value, explanation, target |
+
+## Screenshots
+
+**Traces list** showing an eval run with execution time and status:
+
+![Traces list](docs/images/inspect-tracing-01-traces-list.png)
+
+**Full span tree** showing the eval hierarchy (eval_run -> task -> samples -> solvers/scorers):
+
+![Span tree](docs/images/inspect-tracing-04-timeline.png)
+
+**LLM span detail** with model name, token counts, and response text:
+
+![LLM detail](docs/images/inspect-tracing-05-model-expanded.png)
+
+## Configuration
+
+| Env var | Required | Default | Description |
+|---------|----------|---------|-------------|
+| `MLFLOW_TRACKING_URI` | Yes | - | MLflow server URL |
+| `MLFLOW_EXPERIMENT_NAME` | No | `inspect_ai` | Experiment name |
+| `MLFLOW_INSPECT_TRACING` | No | `false` | Enable execution tracing |
+| `MLFLOW_INSPECT_LOG_ARTIFACTS` | No | `true` | Log eval artifacts |
+
+## Example
+
+```python
+from inspect_ai import Task, eval
+from inspect_ai.dataset import Sample
+from inspect_ai.scorer import match
+from inspect_ai.solver import generate
+
+# No special imports needed. Hooks auto-register on install.
+
+task = Task(
+    dataset=[
+        Sample(input="What is 2 + 2?", target="4"),
+        Sample(input="What is 3 * 5?", target="15"),
+        Sample(input="What is 10 - 7?", target="3"),
+    ],
+    solver=generate(),
+    scorer=match(),
+)
+
+logs = eval(task, model="openai/gpt-4o-mini")
+# Results are now in MLflow: runs with metrics + traces with spans
+```
+
+## Development
+
+```bash
+git clone https://github.com/debu-sinha/inspect-mlflow.git
+cd inspect-mlflow
+uv sync --group dev
+uv run pre-commit install
+uv run pytest tests/ -v
+```
+
+See [CONTRIBUTING.md](CONTRIBUTING.md) for details.
+
+## Related
+
+- [Inspect AI](https://inspect.aisi.org.uk/) - AI evaluation framework by UK AISI
+- [MLflow](https://mlflow.org/) - ML experiment tracking and model management
+- [Inspect AI hooks docs](https://inspect.aisi.org.uk/extensions.html#sec-hooks) - How hooks work
+- [Issue #3547](https://github.com/UKGovernmentBEIS/inspect_ai/issues/3547) - Original proposal