errorworks 0.1.0__tar.gz

errorworks-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,43 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_call:
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - run: uv sync --all-extras
+      - run: uv run ruff check src tests
+      - run: uv run ruff format --check src tests
+
+  typecheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - run: uv sync --all-extras
+      - run: uv run mypy src
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: uv sync --all-extras
+      - run: uv run pytest --cov --cov-report=term-missing

errorworks-0.1.0/.github/workflows/release.yaml

@@ -0,0 +1,34 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  ci:
+    uses: ./.github/workflows/ci.yml
+
+  build:
+    needs: ci
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - run: uv build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

errorworks-0.1.0/.gitignore

@@ -0,0 +1,36 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+
+# Distribution / packaging
+build/
+dist/
+*.egg-info/
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.mypy_cache/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Environment
+.env
+.env.*
+.hypothesis/

errorworks-0.1.0/CHANGELOG.md

@@ -0,0 +1,51 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.0] - 2026-03-12
+
+Initial release of Errorworks — composable chaos-testing services for various pipelines.
+
+### Added
+
+#### Core Engine (`errorworks.engine`)
+- **InjectionEngine**: Burst state machine with priority and weighted error selection modes
+- **MetricsStore**: Thread-safe SQLite metrics storage with schema-driven DDL and time-series bucketing
+- **LatencySimulator**: Configurable artificial latency injection with jitter support
+- **Config loader**: YAML preset system with deep-merge precedence (CLI > file > preset > defaults)
+- Shared vocabulary banks for content generation (English and Lorem Ipsum)
+- Frozen Pydantic models for immutable, validated configuration throughout
+
+#### ChaosLLM (`errorworks.llm`)
+- OpenAI-compatible fake LLM server (chat completions API) with Starlette/ASGI
+- Streaming and non-streaming response modes
+- Error injection: rate limiting (429), server errors (500/502/503), connection errors, malformed JSON
+- Burst pattern simulation for provider stress testing
+- Response generation with gibberish, lorem, and template modes
+- SQLite-backed metrics recording per endpoint/deployment/model
+- Six built-in presets: `silent`, `gentle`, `realistic`, `chaos`, `stress_aimd`, `stress_extreme`
+- `chaosllm` CLI with full configuration via flags, YAML files, or presets
+
+#### ChaosLLM MCP (`errorworks.llm_mcp`)
+- Model Context Protocol server for ChaosLLM metrics analysis
+- Tools for querying error rates, latency distributions, and time-series trends
+- `chaosllm-mcp` CLI entry point
+
+#### ChaosWeb (`errorworks.web`)
+- Fake web server for scraping pipeline resilience testing
+- Error injection: rate limiting (429), forbidden (403), redirects, connection errors, malformed HTML
+- Anti-scraping simulation: SSRF injection, redirect chains, content corruption
+- HTML content generation with configurable structure and link density
+- Five built-in presets: `silent`, `gentle`, `realistic`, `stress_scraping`, `stress_extreme`
+- `chaosweb` CLI with full configuration via flags, YAML files, or presets
+
+#### Unified CLI
+- `chaosengine` command that mounts `chaosllm` and `chaosweb` as subcommands
+
+#### Testing Support
+- `ChaosLLMFixture`: In-process pytest fixture with marker-based configuration
+- `ChaosWebFixture`: In-process pytest fixture with 23 configurable parameters
+- 583 unit tests with 72% overall code coverage

errorworks-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Digital Transformation Agency
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

errorworks-0.1.0/PKG-INFO

@@ -0,0 +1,63 @@
+Metadata-Version: 2.4
+Name: errorworks
+Version: 0.1.0
+Summary: Composable chaos-testing services for various pipelines
+Project-URL: Repository, https://github.com/johnm-dta/errorworks
+Author-email: John Morrissey <john.morrissey@dta.gov.au>
+Maintainer: Digital Transformation Agency
+License: MIT License
+        
+        Copyright (c) 2026 Digital Transformation Agency
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: chaos,fault-injection,llm,resilience,testing,web
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.12
+Requires-Dist: httpx<1,>=0.27
+Requires-Dist: jinja2<4,>=3.1
+Requires-Dist: mcp<2,>=1.0
+Requires-Dist: pydantic<3,>=2.12
+Requires-Dist: pyyaml<7,>=6.0
+Requires-Dist: starlette<1,>=0.45
+Requires-Dist: structlog<26,>=25.0
+Requires-Dist: typer<1,>=0.21
+Requires-Dist: uvicorn<1,>=0.34
+Provides-Extra: dev
+Requires-Dist: hypothesis<7,>=6.98; extra == 'dev'
+Requires-Dist: mypy<2,>=1.19; extra == 'dev'
+Requires-Dist: pre-commit<5,>=4.0; extra == 'dev'
+Requires-Dist: pytest-asyncio<2,>=1.0; extra == 'dev'
+Requires-Dist: pytest-cov<6,>=4.1; extra == 'dev'
+Requires-Dist: pytest-timeout<3,>=2.3; extra == 'dev'
+Requires-Dist: pytest<10,>=9.0; extra == 'dev'
+Requires-Dist: ruff<1,>=0.15; extra == 'dev'
+Requires-Dist: types-pyyaml<7,>=6.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# errorworks
+
+Composable chaos-testing services for various pipelines.

errorworks-0.1.0/README.md

@@ -0,0 +1,3 @@
+# errorworks
+
+Composable chaos-testing services for various pipelines.

errorworks-0.1.0/pyproject.toml

@@ -0,0 +1,172 @@
+[project]
+name = "errorworks"
+version = "0.1.0"
+description = "Composable chaos-testing services for various pipelines"
+readme = "README.md"
+requires-python = ">=3.12"
+license = { file = "LICENSE" }
+authors = [{ name = "John Morrissey", email = "john.morrissey@dta.gov.au" }]
+maintainers = [{ name = "Digital Transformation Agency" }]
+keywords = ["chaos", "testing", "llm", "web", "fault-injection", "resilience"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Testing",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+dependencies = [
+    # Configuration
+    "pydantic>=2.12,<3",
+    "pyyaml>=6.0,<7",
+
+    # CLI
+    "typer>=0.21,<1",
+
+    # Templating (response/content generation)
+    "jinja2>=3.1,<4",
+
+    # Logging
+    "structlog>=25.0,<26",
+
+    # HTTP server
+    "starlette>=0.45,<1",
+    "uvicorn>=0.34,<1",
+
+    # HTTP client (for test fixtures)
+    "httpx>=0.27,<1",
+
+    # MCP server for ChaosLLM metrics analysis
+    "mcp>=1.0,<2",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=9.0,<10",
+    "pytest-cov>=4.1,<6",
+    "pytest-asyncio>=1.0,<2",
+    "pytest-timeout>=2.3,<3",
+    "hypothesis>=6.98,<7",
+    "mypy>=1.19,<2",
+    "ruff>=0.15,<1",
+    "pre-commit>=4.0,<5",
+    "types-PyYAML>=6.0,<7",
+]
+
+[project.scripts]
+chaosengine = "errorworks.engine.cli:main"
+chaosllm = "errorworks.llm.cli:main"
+chaosllm-mcp = "errorworks.llm.cli:mcp_main_entry"
+chaosweb = "errorworks.web.cli:main"
+
+[project.urls]
+Repository = "https://github.com/johnm-dta/errorworks"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/errorworks"]
+
+[tool.hatch.build.targets.sdist]
+exclude = [
+    ".claude/",
+    "CLAUDE.md",
+    "docs/plans/",
+    "docs/arch-analysis-*",
+    "uv.lock",
+]
+
+# === Ruff Configuration ===
+[tool.ruff]
+target-version = "py312"
+line-length = 140
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = [
+    "E",      # pycodestyle errors
+    "F",      # pyflakes
+    "W",      # pycodestyle warnings
+    "I",      # isort
+    "UP",     # pyupgrade
+    "B",      # flake8-bugbear
+    "SIM",    # flake8-simplify
+    "C4",     # flake8-comprehensions
+    "DTZ",    # flake8-datetimez
+    "T20",    # flake8-print
+    "RUF",    # Ruff-specific rules
+]
+ignore = [
+    "E501",   # line too long (handled by formatter)
+    "SIM108", # ternary operator - prefer explicit if/else
+]
+
+[tool.ruff.lint.isort]
+known-first-party = ["errorworks"]
+
+[tool.ruff.lint.flake8-bugbear]
+extend-immutable-calls = ["typer.Option", "typer.Argument"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*.py" = ["T20"]
+
+# === Mypy Configuration ===
+[tool.mypy]
+python_version = "3.12"
+strict = true
+warn_return_any = true
+warn_unused_configs = true
+warn_unreachable = true
+show_error_codes = true
+plugins = ["pydantic.mypy"]
+warn_unused_ignores = true
+
+[[tool.mypy.overrides]]
+module = [
+    "mcp.*",
+    "hypothesis.*",
+]
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "tests.*"
+disallow_untyped_decorators = false
+disallow_untyped_defs = false
+disallow_incomplete_defs = false
+check_untyped_defs = false
+
+# === Pytest Configuration ===
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+addopts = [
+    "-ra",
+    "--strict-markers",
+    "--strict-config",
+]
+markers = [
+    "slow: marks tests as slow",
+    "stress: marks tests as stress tests",
+    "chaosllm: Configure ChaosLLM server for the test",
+    "chaosweb: Configure ChaosWeb server for the test",
+    "integration: marks tests as integration tests",
+]
+
+# === Coverage Configuration ===
+[tool.coverage.run]
+source = ["src/errorworks"]
+branch = true
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "if TYPE_CHECKING:",
+    "raise NotImplementedError",
+    "@abstractmethod",
+]

errorworks-0.1.0/src/errorworks/__init__.py

@@ -0,0 +1,3 @@
+"""Composable chaos-testing servers for LLM and web scraping pipelines."""
+
+__version__ = "0.1.0"

errorworks-0.1.0/src/errorworks/engine/__init__.py

@@ -0,0 +1,53 @@
+"""ChaosEngine: Shared utilities for chaos testing servers.
+
+Provides composition-based building blocks used by ChaosLLM, ChaosWeb, and
+future chaos plugins (ChaosFile, ChaosSQL, etc.):
+
+- InjectionEngine: Burst state machine + priority/weighted error selection
+- MetricsStore: Thread-safe SQLite with schema-driven DDL
+- LatencySimulator: Configurable artificial latency
+- Config loading: deep_merge, preset loading, YAML precedence
+
+Each chaos plugin *composes* these utilities rather than inheriting from
+base classes, avoiding covariant return type friction and HTTP-leakage
+into non-HTTP domains.
+"""
+
+from errorworks.engine.config_loader import (
+    deep_merge,
+    list_presets,
+    load_preset,
+)
+from errorworks.engine.injection_engine import InjectionEngine
+from errorworks.engine.latency import LatencySimulator
+from errorworks.engine.metrics_store import MetricsStore
+from errorworks.engine.types import (
+    DANGEROUS_BIND_HOSTS,
+    BurstConfig,
+    ErrorSpec,
+    LatencyConfig,
+    MetricsConfig,
+    MetricsSchema,
+    SelectionMode,
+    ServerConfig,
+)
+from errorworks.engine.vocabulary import ENGLISH_VOCABULARY, LOREM_VOCABULARY
+
+__all__ = [
+    "DANGEROUS_BIND_HOSTS",
+    "ENGLISH_VOCABULARY",
+    "LOREM_VOCABULARY",
+    "BurstConfig",
+    "ErrorSpec",
+    "InjectionEngine",
+    "LatencyConfig",
+    "LatencySimulator",
+    "MetricsConfig",
+    "MetricsSchema",
+    "MetricsStore",
+    "SelectionMode",
+    "ServerConfig",
+    "deep_merge",
+    "list_presets",
+    "load_preset",
+]

errorworks-0.1.0/src/errorworks/engine/admin.py

@@ -0,0 +1,116 @@
+"""Shared admin endpoint logic for chaos servers.
+
+Provides free functions for admin authentication and endpoint handling,
+composed by both ChaosLLM and ChaosWeb servers via the ChaosServer protocol.
+"""
+
+from __future__ import annotations
+
+import hmac
+import json
+import sqlite3
+from typing import Any, Protocol, runtime_checkable
+
+import pydantic
+import structlog
+from starlette.requests import Request
+from starlette.responses import JSONResponse
+
+logger = structlog.get_logger(__name__)
+
+
+@runtime_checkable
+class ChaosServer(Protocol):
+    """Minimal interface for admin endpoint handlers.
+
+    Both ChaosLLMServer and ChaosWebServer satisfy this protocol.
+    """
+
+    def get_admin_token(self) -> str: ...
+    def get_current_config(self) -> dict[str, Any]: ...
+    def update_config(self, updates: dict[str, Any]) -> None: ...
+    def reset(self) -> str: ...
+    def export_metrics(self) -> dict[str, Any]: ...
+    def get_stats(self) -> dict[str, Any]: ...
+
+
+def check_admin_auth(request: Request, token: str) -> JSONResponse | None:
+    """Check admin authentication.
+
+    Returns None if auth passes, or a 401/403 JSONResponse if it fails.
+    """
+    auth_header = request.headers.get("Authorization", "")
+    if not auth_header.startswith("Bearer "):
+        return JSONResponse(
+            {"error": {"type": "authentication_error", "message": "Missing Authorization: Bearer <token> header"}},
+            status_code=401,
+        )
+    if not hmac.compare_digest(auth_header[7:], token):
+        return JSONResponse(
+            {"error": {"type": "authorization_error", "message": "Invalid admin token"}},
+            status_code=403,
+        )
+    return None
+
+
+async def handle_admin_config(request: Request, server: ChaosServer) -> JSONResponse:
+    """Handle GET/POST /admin/config."""
+    if (denied := check_admin_auth(request, server.get_admin_token())) is not None:
+        return denied
+    if request.method == "GET":
+        return JSONResponse(server.get_current_config())
+    try:
+        body = await request.json()
+    except (json.JSONDecodeError, ValueError, UnicodeDecodeError):
+        return JSONResponse(
+            {"error": {"type": "invalid_request_error", "message": "Request body must be valid JSON"}},
+            status_code=400,
+        )
+    try:
+        server.update_config(body)
+    except (ValueError, TypeError, pydantic.ValidationError) as e:
+        return JSONResponse(
+            {"error": {"type": "validation_error", "message": str(e)}},
+            status_code=422,
+        )
+    return JSONResponse({"status": "updated", "config": server.get_current_config()})
+
+
+async def handle_admin_stats(request: Request, server: ChaosServer) -> JSONResponse:
+    """Handle GET /admin/stats."""
+    if (denied := check_admin_auth(request, server.get_admin_token())) is not None:
+        return denied
+    try:
+        return JSONResponse(server.get_stats())
+    except sqlite3.Error as e:
+        return JSONResponse(
+            {"error": {"type": "database_error", "message": f"Failed to retrieve stats: {e}"}},
+            status_code=503,
+        )
+
+
+async def handle_admin_reset(request: Request, server: ChaosServer) -> JSONResponse:
+    """Handle POST /admin/reset."""
+    if (denied := check_admin_auth(request, server.get_admin_token())) is not None:
+        return denied
+    try:
+        new_run_id = server.reset()
+    except sqlite3.Error as e:
+        return JSONResponse(
+            {"error": {"type": "database_error", "message": f"Failed to reset metrics: {e}"}},
+            status_code=503,
+        )
+    return JSONResponse({"status": "reset", "new_run_id": new_run_id})
+
+
+async def handle_admin_export(request: Request, server: ChaosServer) -> JSONResponse:
+    """Handle GET /admin/export."""
+    if (denied := check_admin_auth(request, server.get_admin_token())) is not None:
+        return denied
+    try:
+        return JSONResponse(server.export_metrics())
+    except sqlite3.Error as e:
+        return JSONResponse(
+            {"error": {"type": "database_error", "message": f"Failed to export metrics: {e}"}},
+            status_code=503,
+        )

errorworks-0.1.0/src/errorworks/engine/cli.py

@@ -0,0 +1,37 @@
+"""Unified CLI for ChaosEngine testing servers.
+
+Aggregates ChaosLLM and ChaosWeb under a single ``chaosengine`` command:
+
+    chaosengine llm serve --preset=gentle
+    chaosengine llm presets
+    chaosengine web serve --preset=stress_scraping
+    chaosengine web presets
+
+Standalone entry points (``chaosllm``, ``chaosweb``) continue to work
+unchanged — this CLI simply mounts the same Typer apps as sub-commands.
+"""
+
+from __future__ import annotations
+
+import typer
+
+from errorworks.llm.cli import app as llm_app
+from errorworks.web.cli import app as web_app
+
+app = typer.Typer(
+    name="chaosengine",
+    help="ChaosEngine: Unified chaos testing server management.",
+    no_args_is_help=True,
+)
+
+app.add_typer(llm_app, name="llm", help="ChaosLLM: Fake LLM server for load testing and fault injection.")
+app.add_typer(web_app, name="web", help="ChaosWeb: Fake web server for scraping pipeline resilience testing.")
+
+
+def main() -> None:
+    """Entry point for chaosengine CLI."""
+    app()
+
+
+if __name__ == "__main__":
+    main()