pytest-resilience-agent 0.2.0__tar.gz

pytest_resilience_agent-0.2.0/.github/dependabot.yml

@@ -0,0 +1,25 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: "/"
+    schedule:
+      interval: weekly
+      day: monday
+      time: "06:00"
+      timezone: "Europe/London"
+    open-pull-requests-limit: 5
+    commit-message:
+      prefix: "deps"
+      include: scope
+
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: weekly
+      day: monday
+      time: "06:00"
+      timezone: "Europe/London"
+    open-pull-requests-limit: 3
+    commit-message:
+      prefix: "ci"
+      include: scope

pytest_resilience_agent-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,44 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    name: Lint (ruff)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.12"
+      - name: Install ruff
+        run: pip install "ruff==0.15.13"
+      - name: Ruff check
+        run: ruff check .
+      - name: Ruff format check
+        run: ruff format --check .
+
+  unit-tests:
+    name: Unit tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+      - name: Install package with dev extras
+        run: pip install -e ".[dev]"
+      - name: Run tests (skip integration and slow markers)
+        run: pytest tests -m "not integration and not slow"

pytest_resilience_agent-0.2.0/.github/workflows/codeql.yml

@@ -0,0 +1,38 @@
+name: CodeQL
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: "30 5 * * 1"
+
+permissions:
+  contents: read
+
+jobs:
+  analyze:
+    name: Analyze (Python)
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4
+        with:
+          languages: python
+          queries: security-and-quality
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4
+        with:
+          category: "/language:python"

pytest_resilience_agent-0.2.0/.github/workflows/dependabot-auto-merge.yml

@@ -0,0 +1,29 @@
+name: Dependabot auto-merge
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  contents: read
+
+jobs:
+  auto-merge:
+    if: github.actor == 'dependabot[bot]'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Read PR metadata
+        id: meta
+        uses: dependabot/fetch-metadata@21025c705c08248db411dc16f3619e6b5f9ea21a # v2
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Enable auto-merge for patch + minor
+        if: steps.meta.outputs.update-type == 'version-update:semver-patch' || steps.meta.outputs.update-type == 'version-update:semver-minor'
+        run: gh pr merge --auto --squash "$PR_URL"
+        env:
+          PR_URL: ${{ github.event.pull_request.html_url }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

pytest_resilience_agent-0.2.0/.github/workflows/publish.yml

@@ -0,0 +1,33 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/pytest-resilience-agent
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
+        with:
+          python-version: "3.12"
+
+      - name: Build sdist and wheel
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0

pytest_resilience_agent-0.2.0/.github/workflows/scorecard.yml

@@ -0,0 +1,42 @@
+name: Scorecard supply-chain security
+on:
+  push:
+    branches:
+    - main
+  schedule:
+    - cron:  "30 1 * * 6"
+
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      id-token: write
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          publish_results: true
+
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
+        with:
+          sarif_file: results.sarif

pytest_resilience_agent-0.2.0/.gitignore

@@ -0,0 +1,40 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+.Python
+build/
+dist/
+*.egg-info/
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# Editor
+.vscode/
+.idea/
+*.swp
+.DS_Store
+
+# pytest
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Local env files
+.env
+.env.local
+
+# Build artefacts
+*.whl
+*.tar.gz
+.secrets/
+
+# Slide regeneration artefacts
+videos/slides/*.orig.*
+videos/*.orig.*

pytest_resilience_agent-0.2.0/.pre-commit-config.yaml

@@ -0,0 +1,17 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.7.4
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-yaml
+      - id: check-toml
+      - id: check-merge-conflict
+      - id: check-added-large-files

pytest_resilience_agent-0.2.0/CHANGELOG.md

@@ -0,0 +1,113 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Planned
+- Semantic assertion hooks (composability with eval frameworks)
+- LLM-driven scenario classifier behind the existing `pick_scenarios` interface
+- Scenario composition primitives (e.g. `rate_limit` then `partial_outage`)
+
+## [0.2.0] - 2026-06-11
+
+### Added
+- Multi-turn conversation chaos. The `resilience` marker now accepts
+  `turns=[[...], [...], ...]`, one scenario set per conversation turn, advanced
+  with `chaos.next_turn()`. Each turn is an independent chaos window: advancing
+  reverts the current turn's scenarios and applies the next turn's with fresh
+  call counters, so chaos can appear and clear mid-conversation (turn 1 clean,
+  turn 2 brownout, turn 3 recovered). `turns=` and `scenarios=` are mutually
+  exclusive; combining them, or advancing past the last turn, raises a clear
+  usage error. Each turn boundary emits a `chaos.turn.N` OpenTelemetry span.
+- New `malformed_json` chaos scenario: the gateway returns HTTP 200 with an HTML error body instead of JSON, mirroring a proxy or CDN that swallows the upstream failure and serves its own page. Agent code that calls `response.json()` without guarding against decode errors surfaces this as an unhandled exception rather than a graceful fallback. Brings the built-in scenario count to ten.
+
+## [0.1.0] - 2026-05-27
+
+Initial release. Built for the DevNetwork [AI + ML] Hackathon 2026, Lark and
+TrueFoundry sponsor tracks.
+
+### Added
+
+**Plugin**
+- `pytest_resilience_agent.plugin` registers the `resilience` pytest marker
+  with strict-marker support
+- `ai_gateway` fixture: returns an `AIGatewayClient` configured from
+  `--resilience-gateway-url` CLI option or `TFY_GATEWAY_URL` env var
+- `chaos` fixture: applies named scenarios from the marker for the test
+  duration, with automatic cleanup
+- `--resilience-record PATH` option writes a JSON timeline of every chaos
+  event to disk at session finish
+- `pytest_runtest_logreport` hook attaches a "chaos events" section to each
+  test report so judges and engineers see what was injected next to the
+  test outcome
+
+**Chaos scenarios**
+- `llm_timeout`: gateway sleeps past the request timeout
+- `llm_5xx`: gateway returns 502 for the first N calls, then succeeds
+- `rate_limit`: gateway returns 429 with `Retry-After`
+- `mcp_error`: Lark MCP server raises a JSON-RPC error envelope
+- `partial_outage`: first call 503, retry succeeds
+- `cost_exceeded`: gateway returns 402 quota_exceeded
+- `wrong_model_returned`: gateway silently routes to an unintended model
+- `stream_stall`: gateway returns 200 with empty content (stream drop)
+- `network_blip`: ConnectError on first N calls, recovery after
+
+**Generator**
+- `pytest_resilience_agent.generator.generate_test` writes a runnable
+  pytest file for any failure text, with chaos scenarios picked from a
+  deterministic regex rule set (ADR 0003)
+- 9 regex rules cover the common failure-text patterns: 429, 502, 503,
+  504, 402, connection errors, empty/stream, MCP, model mismatch
+
+**CLI**
+- `pytest-resilience-agent` console script with five subcommands:
+  - `scenarios` lists the registered chaos scenarios
+  - `discover` lists failing tests via Lark MCP
+  - `generate` synthesises resilience tests from Lark failures
+  - `run` executes the generated tests through pytest
+  - `report` pushes resolution status back to Lark
+
+**Demo entry points**
+- `demo/run_demo.py`: drives the sample FastAPI agent through every chaos
+  scenario with a Rich-table summary
+- `demo/run_full_loop.py`: full end-to-end loop - spins up mock Lark
+  server in a background thread, pulls failures, generates resilience
+  tests, runs pytest, reports resolutions back to Lark
+- `demo/mock_truefoundry.py`: in-process mock of the TrueFoundry AI
+  Gateway with a Gemini → Claude → local fallback chain
+- `demo/mock_lark.py`: in-process mock of the Lark MCP server with seeded
+  failing-test data
+- `demo/sample_agent/app.py`: sample FastAPI agent that summarises
+  customer emails through the gateway with retry logic
+
+**Tests and quality**
+- 16 passing tests covering plugin registration, fixture wiring, every
+  chaos scenario, the timeline export, and the report hook
+- 5 example end-user tests in `demo/example_agent_tests/` showing the
+  four patterns we expect adopters to copy
+- pre-commit configuration (ruff format + lint, trailing-whitespace,
+  YAML/TOML/large-file checks)
+- 3 ADRs in `docs/adr/`:
+  - 0001: resilience-first direction, not eval-first
+  - 0002: respx as the chaos injection layer
+  - 0003: rule-based scenario picker for v0.1, LLM-driven option for v0.2
+
+**Observability**
+- OpenTelemetry spans on every chaos scenario apply / revert, ready for
+  OTLP export to Cloud Trace or any compatible backend
+
+### Known limitations
+
+- Semantic-level paraphrased failures are out of scope (use phoenix2pytest
+  or DeepEval for those)
+- Multi-turn conversation chaos is on the v0.2 roadmap
+- Distributed-system chaos (network partitions across services) is out of
+  scope for v0.1; the HTTP-layer approach covers gateway, model, MCP, and
+  rate limiter in one mechanism
+
+[Unreleased]: https://github.com/golikovichev/pytest-resilience-agent/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/golikovichev/pytest-resilience-agent/releases/tag/v0.1.0

pytest_resilience_agent-0.2.0/CONTRIBUTING.md

@@ -0,0 +1,40 @@
+# Contributing
+
+Thanks for your interest in pytest-resilience-agent. This is a small alpha project shipped during the DevNetwork AI+ML Hackathon, so the contribution flow is light.
+
+## Reporting a bug
+
+Open an issue with:
+
+- What you ran (pytest invocation, marker selection, Python version)
+- What you expected the resilience scenario to do
+- What happened instead (timeline output on failure helps)
+- A minimal scenario or fixture snippet that reproduces it (strip any real API keys or gateway tokens first)
+
+## Suggesting a feature
+
+Open an issue first so we can talk through the use case before you write code. The project scope is intentionally narrow: pytest-driven resilience testing for LLM applications via Lark MCP plus TrueFoundry AI Gateway, with rule-based assertions and explicit chaos markers. Feature requests that pull it elsewhere will get a polite redirect.
+
+## Submitting a pull request
+
+1. Fork the repo and create a branch from `main`.
+2. Make your changes. Keep the diff focused on one thing.
+3. Add or update tests in `tests/`. The CI runs `pytest -v` on Python 3.11, 3.12, and 3.13.
+4. Run the tests locally before pushing:
+   ```bash
+   pip install -e ".[dev]"
+   pytest tests -m "not integration"
+   ```
+5. New scenarios, fixtures, or markers need at least one happy-path test and one chaos-injection test that exercises the timeline output.
+6. Open the PR with a short description of what changed and why.
+
+## Code style
+
+- Python 3.11+. Type hints on public functions.
+- Function and variable names in English, snake_case (e.g., `inject_timeout`, `assert_recovery`).
+- One responsibility per function. If a function grows past 30-40 lines, split it.
+- Ruff handles lint and formatting. Run `ruff check . && ruff format .` before opening a PR.
+
+## Security
+
+If you find something that could leak gateway tokens, MCP server credentials, or PII from a real LLM transcript, please report privately. See `SECURITY.md` for the disclosure channel.

pytest_resilience_agent-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mikhail Golikov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.