proofrun 0.1.1__tar.gz → 0.2.0__tar.gz

proofrun-0.2.0/.dockerignore

@@ -0,0 +1,13 @@
+.git
+.github
+.venv
+.pytest_cache
+.ruff_cache
+.coverage
+htmlcov
+dist
+**/__pycache__
+tests
+docs
+deploy
+FUTURE.md

{proofrun-0.1.1 → proofrun-0.2.0}/.github/workflows/ci.yaml

@@ -16,7 +16,7 @@ permissions:
 jobs:
   quality:
     name: Quality & Tests
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
 
     strategy:
       fail-fast: false
@@ -27,15 +27,15 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v7
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Set up uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v7
         with:
           enable-cache: true
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -46,29 +46,25 @@ jobs:
       - name: Run code quality & coverage checks
         run: make check
 
-      - name: Run integration scenarios
-        run: |
-          make run-profile
-          make run-auth
-          make run-chain
-          make run-crud
+      - name: Run integration scenarios (local demo API, no network)
+        run: make integration
 
   build:
     name: Build Package
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     needs: quality
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v7
         with:
           python-version: "3.11"
 
       - name: Set up uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v7
         with:
           enable-cache: true
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -84,3 +80,25 @@ jobs:
           ls -lh dist/
           test -f dist/*.whl
           test -f dist/*.tar.gz
+
+      - name: Install the built packages in a clean environment
+        run: make dist-smoke
+
+  docker:
+    name: Docker Image
+    runs-on: ubuntu-24.04
+    needs: quality
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Build image
+        run: docker build --tag proofrun:ci .
+
+      - name: Smoke test image
+        run: |
+          docker run --rm proofrun:ci --version
+          docker run --rm -v "$PWD/scenarios:/monitors:ro" \
+            -e DEMO_PASSWORD=demo-password \
+            proofrun:ci /monitors/resilient.yaml --validate

proofrun-0.2.0/.github/workflows/release.yaml

@@ -0,0 +1,175 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+jobs:
+  quality:
+    name: Quality & Tests
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.11"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install dependencies
+        run: make install
+
+      - name: Run code quality checks
+        run: make check
+
+      - name: Run integration scenarios (local demo API, no network)
+        run: make integration
+
+  build:
+    name: Build Package
+    runs-on: ubuntu-24.04
+    needs: quality
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.11"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Verify tag matches package version
+        run: |
+          TAG_VERSION="${GITHUB_REF_NAME#v}"
+          PKG_VERSION="$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2)"
+          echo "tag: ${TAG_VERSION}  pyproject: ${PKG_VERSION}"
+          if [ "${TAG_VERSION}" != "${PKG_VERSION}" ]; then
+            echo "::error::Tag v${TAG_VERSION} does not match pyproject version ${PKG_VERSION}"
+            exit 1
+          fi
+
+      - name: Build distributions
+        run: make build
+
+      - name: Install the built packages in a clean environment
+        run: make dist-smoke
+
+      - name: Store distributions
+        uses: actions/upload-artifact@v7
+        with:
+          name: package-distributions
+          path: dist/
+
+  publish-to-pypi:
+    name: Publish to PyPI
+    runs-on: ubuntu-24.04
+    needs: build
+    environment: release
+    permissions:
+      id-token: write
+    steps:
+      - name: Download distributions
+        uses: actions/download-artifact@v8
+        with:
+          name: package-distributions
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  publish-docker:
+    name: Publish Docker Image
+    runs-on: ubuntu-24.04
+    needs: build
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Build image for smoke test
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          load: true
+          tags: proofrun:smoke
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Smoke test image
+        run: |
+          TAG_VERSION="${GITHUB_REF_NAME#v}"
+          OUTPUT="$(docker run --rm proofrun:smoke --version)"
+          echo "${OUTPUT}"
+          echo "${OUTPUT}" | grep -qxF "Proofrun version ${TAG_VERSION}"
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Generate image tags and labels
+        id: meta
+        uses: docker/metadata-action@v6
+        with:
+          images: ghcr.io/kishan-thanki/proofrun
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=raw,value=latest,enable=${{ !contains(github.ref_name, '-') }}
+
+      - name: Build and push (linux/amd64, linux/arm64)
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  github-release:
+    name: Create GitHub Release
+    runs-on: ubuntu-24.04
+    needs: [publish-to-pypi, publish-docker]
+    permissions:
+      contents: write
+    steps:
+      - name: Download distributions
+        uses: actions/download-artifact@v8
+        with:
+          name: package-distributions
+          path: dist/
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v3
+        with:
+          files: dist/*
+          generate_release_notes: true

{proofrun-0.1.1 → proofrun-0.2.0}/.gitignore

@@ -82,3 +82,7 @@ __pypackages__/
 
 
 FUTURE.md
+
+# downloaded archives and patches
+*.zip
+*.patch

proofrun-0.2.0/CHANGELOG.md

@@ -0,0 +1,116 @@
+# Changelog
+
+All notable changes to Proofrun are documented here. The format follows
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Proofrun is below 1.0,
+so a minor release can contain breaking changes; each one is listed under
+**Breaking changes**.
+
+## [0.2.0]
+
+### Added
+
+- **Environment variables.** `${env.NAME}` is resolved when a config file is
+  loaded, in any value or key, including `base_url`. A missing variable is a
+  configuration error that names the variable and where it is used.
+- **Retries.** `retries` and `retry_delay_seconds` (per scenario or under
+  `global`) re-run the whole scenario before a failure counts. Results show
+  `Attempts: N`. Retries also work in run-once mode.
+- **Thresholds.** `failure_threshold` and `recovery_threshold` count consecutive
+  runs before a scenario becomes `FAILING` or recovers. The defaults (`1`, `1`)
+  keep the previous behavior.
+- **Per-scenario scheduling.** Every scenario runs on its own `interval_seconds`,
+  with jitter. A slow scenario does not delay another, and a scenario never
+  overlaps with itself. `interval_seconds` can also be set under `global`.
+- **Metrics.** `--metrics-port` (and `--metrics-host`) serve
+  `/metrics` and `/health` in daemon mode. Metrics: `proofrun_scenario_up`,
+  `_runs_total`, `_failures_total`, `_duration_seconds`,
+  `_last_success_timestamp_seconds`, `_last_failure_timestamp_seconds`. The only
+  label is `scenario`. The server binds to loopback by default.
+- **`--validate`.** Checks a configuration, including environment references,
+  without sending requests.
+- **Docker image** for `linux/amd64` and `linux/arm64`, published to
+  `ghcr.io/kishan-thanki/proofrun` by the release pipeline (after the tests pass and
+  the tag matches the package version), running as a non-root user.
+- **Deployment examples:** a hardened systemd unit and environment file in
+  `deploy/systemd/`, and `deploy/docker-compose.yaml`.
+- `scenarios/resilient.yaml`, an offline example of the features above.
+- A warning when `--webhook-url` or `--metrics-port` is given in run-once mode,
+  where they have no effect.
+- Duplicate scenario names and extract variable names starting with `env.` are
+  rejected when the config is loaded.
+- PyPI metadata: keywords, classifiers, and project URLs.
+
+### Changed
+
+- The daemon is now event-driven per scenario instead of running every scenario
+  in lockstep. Configuration changes are picked up within about a second.
+- An invalid configuration file is parsed once per change and its error is
+  printed once, instead of on every cycle.
+- Configuration validation errors show the field location and message without
+  echoing the offending input, so resolved secrets cannot leak.
+- Rich markup in scenario names, step names, and error text is escaped.
+- `uvloop` is installed only where it is available (not on Windows), and Proofrun
+  falls back to the standard `asyncio` loop. `orjson`, which was never used, is no
+  longer a dependency. The minimum `pydantic` is now 2.7.
+- The source is organized into packages by concern (`cli`, `config`, `runner`,
+  `monitor`, `notify`, `metrics`, plus `health`) instead of a single `core` package.
+  Only files moved; behavior is unchanged.
+- Integration scenarios and the CI and release pipelines run against the local demo
+  API (`make integration`), with no dependency on a third-party host.
+
+### Fixed
+
+- A failed webhook delivery printed the full webhook URL, which often contains a secret
+  token, and a timeout printed nothing useful. Failures now report only the reason (an
+  HTTP status, a timeout, or a connection error) and the scenario name.
+- The daemon kept running with no valid configuration, looking alive while monitoring
+  nothing. An invalid configuration at startup now exits with code 1. An invalid edit
+  while running still keeps the last working configuration.
+- The daemon was killed abruptly by `SIGTERM`, and as PID 1 in a container it never
+  received the signal, so `docker stop` waited and then killed it. `SIGTERM` now stops
+  it cleanly.
+- `expect.status` accepted an empty list and codes outside 100 to 599, which can never
+  match a response. Both are now configuration errors.
+- `--interval` and `--metrics-host` were silently ignored in run-once mode. They now
+  produce the same warning as `--webhook-url` and `--metrics-port`.
+- A configuration with no scenarios, or a scenario with no steps, used to pass and report
+  success without making a request. Both are now configuration errors.
+- Misspelled field names were silently ignored, so the check they were meant to
+  configure never ran. Unknown fields are now rejected with their location.
+- Header mismatch errors no longer print the values of sensitive headers (names
+  containing auth, cookie, token, secret, key, password, or session), so they cannot
+  reach the console or webhook alerts.
+- Dictionary keys that interpolate to the same name now fail the step instead of
+  silently dropping one of them.
+- Scenarios that ran at the same time shared one HTTP client, so a run could clear
+  or read another run's cookies. Every run now has its own client.
+- Removed stale `proof` naming (badges, example headers and query values, a hard-coded
+  User-Agent version, and the fallback package version).
+
+### Breaking changes
+
+- **A daemon with an invalid configuration at startup now exits with code 1** instead of
+  waiting for the file to be fixed. Under systemd or Docker, a restart policy will keep
+  retrying until the file is valid.
+- **Configuration validation is strict.** A file with a misspelled or unknown field, no
+  scenarios, or a scenario without steps now fails to load. Run `proofrun --validate`
+  on your files before upgrading.
+- **The webhook environment variable is `PROOFRUN_WEBHOOK_URL`.** `PROOF_WEBHOOK_URL`
+  is no longer read, and there is no alias. If you set the old name, alerts stop
+  until you rename it.
+- **`interval_seconds` is now per scenario.** Before, all scenarios ran together on
+  the smallest interval in the file. Scenarios with different intervals now run at
+  different rates. `--interval N` still runs everything every N seconds.
+- Configuration validation error text has a new format.
+- `${env.NAME}` is resolved at load time, so a config that contained that literal text
+  needs the variable to exist (it would previously have failed at run time).
+- Python imports changed. `proofrun.core` no longer exists; for example
+  `proofrun.core.daemon.Daemon` is now `proofrun.monitor.daemon.Daemon`, and
+  `proofrun.cli.execute_config` is replaced by `run_once` in `proofrun.cli.commands`.
+  The command line is the supported interface; there is no stable Python API yet.
+
+## [0.1.1]
+
+Initial public release: YAML scenarios, multi-step requests with JSONPath
+extraction, status, latency, header, and JSON Schema assertions, run-once and
+daemon modes, webhook alerts on health changes, and hot configuration reload.

proofrun-0.2.0/Dockerfile

@@ -0,0 +1,31 @@
+# syntax=docker/dockerfile:1
+
+FROM python:3.12.15-alpine AS build
+
+RUN apk upgrade --no-cache
+
+ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_CACHE_DIR=1
+
+WORKDIR /src
+COPY pyproject.toml README.md LICENSE ./
+COPY src ./src
+
+RUN python -m venv /opt/venv \
+    && /opt/venv/bin/pip install .
+
+FROM python:3.12.15-alpine
+
+COPY --from=build /opt/venv /opt/venv
+
+RUN apk upgrade --no-cache \
+    && adduser -S -D -H -u 10001 -s /bin/false proofrun
+
+ENV PATH="/opt/venv/bin:${PATH}" \
+    PYTHONUNBUFFERED=1
+
+USER proofrun
+WORKDIR /monitors
+
+ENTRYPOINT ["proofrun"]
+CMD ["--help"]

proofrun-0.2.0/Makefile

@@ -0,0 +1,89 @@
+PYTHON ?= python3
+
+.PHONY: install test coverage coverage-html lint format format-check typecheck check \
+	serve integration build dist-smoke clean full
+
+full: clean install check build
+
+install:
+	uv sync --group dev
+
+test:
+	uv run pytest -q
+
+coverage:
+	uv run pytest -q --cov=proofrun --cov-branch \
+		--cov-report=term-missing \
+		--cov-fail-under=100
+
+coverage-html:
+	uv run pytest -q --cov=proofrun --cov-branch \
+		--cov-report=html
+	@echo "Coverage report generated: htmlcov/index.html"
+
+lint:
+	uv run ruff check src/ tests/
+
+format:
+	uv run ruff format src/ tests/
+
+format-check:
+	uv run ruff format --check src/ tests/
+	uv run ruff check src/ tests/
+
+typecheck:
+	uv run pyright
+
+check: lint format-check typecheck coverage
+
+serve:
+	$(PYTHON) scenarios/server.py
+
+integration:
+	@$(PYTHON) scenarios/server.py & \
+	SERVER_PID=$$!; \
+	trap 'kill $$SERVER_PID 2>/dev/null || true' EXIT; \
+	up=0; \
+	for i in $$(seq 1 50); do \
+		$(PYTHON) -c "import urllib.request as u; u.urlopen('http://127.0.0.1:8099/health')" \
+			>/dev/null 2>&1 && up=1 && break; \
+		sleep 0.2; \
+	done; \
+	if [ $$up -ne 1 ]; then \
+		echo "The demo API did not start on 127.0.0.1:8099. Is '$(PYTHON)' installed and the port free?"; \
+		echo "Try: make integration PYTHON=/path/to/python3"; \
+		exit 1; \
+	fi; \
+	export DEMO_PASSWORD=demo-password; \
+	status=0; \
+	for name in profile auth chain crud resilient; do \
+		echo "==> scenarios/$$name.yaml"; \
+		uv run proofrun scenarios/$$name.yaml --validate >/dev/null || status=1; \
+		uv run proofrun scenarios/$$name.yaml || status=1; \
+	done; \
+	exit $$status
+
+dist-smoke:
+	@set -e; tmp=$$(mktemp -d); trap 'rm -rf "$$tmp"' EXIT; \
+	for pkg in dist/*.whl dist/*.tar.gz; do \
+		echo "==> installing $$pkg in a clean environment"; \
+		$(PYTHON) -m venv "$$tmp/venv"; \
+		"$$tmp/venv/bin/pip" install --quiet "$$pkg"; \
+		"$$tmp/venv/bin/proofrun" --version; \
+		DEMO_PASSWORD=smoke "$$tmp/venv/bin/proofrun" scenarios/resilient.yaml --validate >/dev/null; \
+		rm -rf "$$tmp/venv"; \
+	done
+
+build:
+	uv build
+
+clean:
+	rm -rf dist/ \
+		.ruff_cache/ \
+		.pytest_cache/ \
+		.pyright_cache/ \
+		.coverage \
+		.coverage.* \
+		htmlcov/ \
+		__pycache__/ \
+		*.egg-info/