netdryrun 0.3.2__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/ISSUE_TEMPLATE/feedback.yml +15 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/workflows/ci.yml +7 -1
- netdryrun-0.3.3/.github/workflows/mcp-in-runner.yml +38 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/workflows/published-action.yml +3 -3
- netdryrun-0.3.3/.gitlab-ci.yml +43 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/AGENTS.md +1 -1
- {netdryrun-0.3.2 → netdryrun-0.3.3}/CHANGELOG.md +38 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/PKG-INFO +5 -5
- {netdryrun-0.3.2 → netdryrun-0.3.3}/README.md +3 -3
- {netdryrun-0.3.2 → netdryrun-0.3.3}/action.yml +40 -16
- {netdryrun-0.3.2 → netdryrun-0.3.3}/advisories/0.3.0-false-pass/README.md +3 -0
- netdryrun-0.3.3/advisories/0.3.2-rejected-config/README.md +40 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/render.py +30 -5
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/run.py +23 -5
- netdryrun-0.3.3/corpus/v2/AMENDMENTS.md +13 -0
- netdryrun-0.3.3/corpus/v2/NONDETERMINISTIC.yml +26 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/RESULTS.md +15 -7
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-demo/AGENT-RUN.md +1 -1
- netdryrun-0.3.3/examples/agent-eval/PREREGISTRATION-OPEN.md +48 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/README.md +22 -0
- netdryrun-0.3.3/examples/agent-eval/RESULTS.md +156 -0
- netdryrun-0.3.3/examples/agent-eval/open_agent.py +267 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/run_eval.py +29 -6
- netdryrun-0.3.3/examples/agent-eval/stuck.py +94 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/batfish-pipeline/github-workflow.yml +1 -1
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/sonic-frr/topology.clab.yml +4 -1
- {netdryrun-0.3.2 → netdryrun-0.3.3}/pyproject.toml +3 -3
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.2/GATE.md +25 -6
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.2/accepted.json +2 -1
- netdryrun-0.3.3/releases/0.3.2/gate/after-publish.json +161 -0
- netdryrun-0.3.3/releases/0.3.3/GATE.md +473 -0
- netdryrun-0.3.3/releases/0.3.3/accepted.json +4025 -0
- netdryrun-0.3.3/releases/0.3.3/corpus-v1-results.json +1223 -0
- netdryrun-0.3.3/releases/0.3.3/corpus-v2-results.json +740 -0
- netdryrun-0.3.3/releases/0.3.3/gate/live.json +542 -0
- netdryrun-0.3.3/releases/0.3.3/gate/mutation.json +63 -0
- netdryrun-0.3.3/releases/0.3.3/gate/static.json +163 -0
- netdryrun-0.3.3/releases/0.3.3/gate/unit.json +94 -0
- netdryrun-0.3.3/releases/0.3.3/mutants.md +12093 -0
- netdryrun-0.3.3/releases/0.3.3/review.md +256 -0
- netdryrun-0.3.3/scripts/capture_fixtures.sh +224 -0
- netdryrun-0.3.3/scripts/check_verdict.py +60 -0
- netdryrun-0.3.3/scripts/compat_matrix.py +193 -0
- netdryrun-0.3.3/scripts/fixture_shape.py +92 -0
- netdryrun-0.3.3/scripts/get_sonic_vs.sh +17 -0
- netdryrun-0.3.3/scripts/mcp_in_runner.py +121 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/scripts/release_gate.py +163 -11
- {netdryrun-0.3.2 → netdryrun-0.3.3}/scripts/sync_facts.py +51 -7
- {netdryrun-0.3.2 → netdryrun-0.3.3}/server.json +3 -3
- {netdryrun-0.3.2 → netdryrun-0.3.3}/src/netdryrun/__init__.py +1 -1
- netdryrun-0.3.3/src/netdryrun/agent_result.py +302 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/src/netdryrun/checks.py +10 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/src/netdryrun/cli.py +34 -5
- netdryrun-0.3.3/src/netdryrun/comment.py +133 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/src/netdryrun/config.py +6 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/src/netdryrun/drivers/base.py +6 -1
- {netdryrun-0.3.2 → netdryrun-0.3.3}/src/netdryrun/drivers/eos.py +13 -2
- {netdryrun-0.3.2 → netdryrun-0.3.3}/src/netdryrun/drivers/frr.py +172 -4
- {netdryrun-0.3.2 → netdryrun-0.3.3}/src/netdryrun/drivers/srlinux.py +11 -5
- {netdryrun-0.3.2 → netdryrun-0.3.3}/src/netdryrun/lab.py +13 -2
- netdryrun-0.3.3/src/netdryrun/mcp_server.py +374 -0
- netdryrun-0.3.3/src/netdryrun/record.py +230 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/src/netdryrun/runner.py +51 -0
- netdryrun-0.3.3/templates/README.md +119 -0
- netdryrun-0.3.3/templates/netdryrun.yml +183 -0
- netdryrun-0.3.3/tests/compat_verified.json +27 -0
- netdryrun-0.3.3/tests/fixtures/frr_boot_log_clean.txt +28 -0
- netdryrun-0.3.3/tests/fixtures/frr_boot_log_rejected.conf +29 -0
- netdryrun-0.3.3/tests/fixtures/frr_boot_log_rejected.txt +47 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/sonic_bgp_neighbors.txt +25 -28
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/sonic_route.txt +4 -4
- netdryrun-0.3.3/tests/test_action_comment.py +103 -0
- netdryrun-0.3.3/tests/test_check_verdict.py +38 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_cli.py +12 -0
- netdryrun-0.3.3/tests/test_cli_verdict.py +145 -0
- netdryrun-0.3.3/tests/test_comment.py +187 -0
- netdryrun-0.3.3/tests/test_compat_matrix.py +54 -0
- netdryrun-0.3.3/tests/test_corpus_nondeterministic.py +56 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_docs_claims.py +1 -4
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_driver_fuzz.py +3 -0
- netdryrun-0.3.3/tests/test_drivers_audit_033.py +415 -0
- netdryrun-0.3.3/tests/test_frr_config_load.py +584 -0
- netdryrun-0.3.3/tests/test_gitlab_component.py +195 -0
- netdryrun-0.3.3/tests/test_listing_text.py +20 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_mcp.py +20 -0
- netdryrun-0.3.3/tests/test_mcp_async.py +217 -0
- netdryrun-0.3.3/tests/test_mcp_in_runner.py +27 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_mcp_result_schema.py +3 -6
- netdryrun-0.3.3/tests/test_record.py +233 -0
- netdryrun-0.3.3/tests/test_release_gate_mutants.py +202 -0
- netdryrun-0.3.3/tests/test_release_gate_retired.py +55 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_runner_robustness.py +3 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_security.py +3 -3
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_verdict_drivers.py +5 -5
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_verdict_runner.py +18 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/ai-agents.md +63 -3
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/batfish.md +11 -7
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus.md +15 -7
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/faq.md +6 -6
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/getting-started.md +1 -1
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/github-action.md +17 -6
- netdryrun-0.3.3/website/docs/guides/agent-pr.md +69 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/guides/frr-ci.md +1 -1
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/guides/sonic-ci.md +3 -1
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/guides/your-lab.md +6 -4
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/how-its-built.md +1 -1
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/index.md +2 -2
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/llms-full.txt +304 -56
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/llms.txt +5 -2
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/recipes.md +1 -1
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/reference/cli.md +9 -2
- netdryrun-0.3.3/website/docs/reference/compatibility.md +70 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/reference/config.md +16 -4
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/schemas/mcp-result-v1.json +18 -3
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/trust.md +3 -2
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/tutorial.md +1 -1
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/when-to-use-what.md +6 -6
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/mkdocs.yml +2 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/overrides/main.html +1 -1
- netdryrun-0.3.2/corpus/v2/AMENDMENTS.md +0 -11
- netdryrun-0.3.2/examples/agent-eval/RESULTS.md +0 -77
- netdryrun-0.3.2/scripts/eos_capture_fixtures.sh +0 -19
- netdryrun-0.3.2/scripts/get_sonic_vs.sh +0 -10
- netdryrun-0.3.2/src/netdryrun/mcp_server.py +0 -486
- netdryrun-0.3.2/src/netdryrun/record.py +0 -111
- netdryrun-0.3.2/tests/test_record.py +0 -26
- netdryrun-0.3.2/tests/test_release_gate_mutants.py +0 -45
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.devcontainer/devcontainer.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.devcontainer/hint.sh +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.devcontainer/setup.sh +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.gitattributes +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/ISSUE_TEMPLATE/bug.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/ISSUE_TEMPLATE/corpus-case.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/assets/netdryrun-agent-demo.gif +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/assets/netdryrun-demo.gif +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/assets/netdryrun-wordmark-navy-tm.svg +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/assets/netdryrun-wordmark-navy.svg +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/assets/netdryrun-wordmark-white-tm.svg +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/assets/netdryrun-wordmark-white.svg +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/dependabot.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/workflows/codeql.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/workflows/mcp-registry.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/workflows/pages.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/workflows/release-gate.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/workflows/release.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/workflows/reproduce-corpus.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.github/workflows/scorecard.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/.gitignore +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/CONTRIBUTING.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/LICENSE +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/RELEASING.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/SECURITY.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/TRADEMARKS.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/advisories/0.3.0-false-pass/1-misspelled-top-key.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/advisories/0.3.0-false-pass/2-quoted-boolean.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/advisories/0.3.0-false-pass/3-unknown-afi.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/advisories/0.3.0-false-pass/4-unknown-protocol.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/advisories/0.3.0-false-pass/5-duplicate-key.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/advisories/0.3.0-false-pass/6-min-zero.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/advisories/0.3.0-false-pass/7-vrf-all.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/AMENDMENTS.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/INTENT.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/RESULTS.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/base/configs/ce1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/base/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/base/configs/pe1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/base/configs/pe2.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/base/configs/peer1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/base/configs/up1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/base/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/base/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/01-no-change.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/02-med-to-upstream.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/03-prepend-to-peer.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/04-tag-community-upstream.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/05-description-only.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/06-customer-adds-prefix.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/07-tighten-max-prefix.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/08-upstream-timers.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/09-customer-localpref.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/10-transit-export-filter-removed.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/11-peer-export-filter-removed.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/12-ibgp-send-community-dropped.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/13-upstream-asn-typo.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/14-peer-session-shutdown.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/15-customer-filter-too-strict.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/16-mass-withdrawal.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/17-default-leaked-to-peer.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/18-redistribute-connected.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/19-nexthop-self-removed.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/20-ibgp-shutdown.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/21-customer-max-prefix-removed.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/22-customer-leaks-third-party.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/23-fat-finger-third-party-origin.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/24-customer-blackholed.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/25-aggregate-typo.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/26-route-map-order-bug.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/27-local-asn-typo.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/28-customer-import-policy-removed.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/29-graceful-restart.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/cases/30-peer-import-filter-removed.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/results/results.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/specs/baseline-ping.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/specs/baseline-sessions.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/specs/intent.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/tools/check_edits.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/tools/gen_cases.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/INTENT.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/SITE.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/base/configs/ce1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/base/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/base/configs/pe1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/base/configs/pe2.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/base/configs/peer1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/base/configs/up1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/base/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/base/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/01-no-change.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/02-redistribute-connected-pe2.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/03-test-prefix-left-in-config.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/04-redundant-aggregate-typo.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/05-customer-max-prefix-removed.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/06-peer-import-filter-permit-all.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/07-customer-adds-prefixes.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/08-customer-announces-aggregate.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/09-ibgp-send-community-line-removed.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/10-ibgp-standard-communities-disabled.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/11-customer-max-prefix-warning-only.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/12-customer-max-prefix-raised.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/13-upstream-routes-tagged-as-customer.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/14-customer-gets-default-only.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/15-upstream-default-filtered.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/16-customer-session-peer-group.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/17-customer-region-community.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/cases/18-aggregate-discard-route-removed.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/results/results.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/specs/baseline-ping.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/specs/baseline-sessions.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/corpus/v2/specs/intent.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/docs/DESIGN.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-demo/.gitignore +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-demo/README.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-demo/TICKET.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-demo/changes/attempt-1-open-filters.diff +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-demo/changes/attempt-2-prefix-lists.diff +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-demo/configs/cust1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-demo/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-demo/configs/edge1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-demo/configs/peer1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-demo/configs/up1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-demo/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-demo/demo.sh +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-demo/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-demo/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/.gitignore +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/PREREGISTRATION.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/grade_rerun.sh +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T01-new-block/grader.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T01-new-block/ticket/TICKET.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T01-new-block/ticket/configs/cust1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T01-new-block/ticket/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T01-new-block/ticket/configs/edge1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T01-new-block/ticket/configs/peer1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T01-new-block/ticket/configs/up1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T01-new-block/ticket/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T01-new-block/ticket/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T01-new-block/ticket/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T02-transit-only/grader.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T02-transit-only/ticket/TICKET.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T02-transit-only/ticket/configs/cust1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T02-transit-only/ticket/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T02-transit-only/ticket/configs/edge1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T02-transit-only/ticket/configs/peer1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T02-transit-only/ticket/configs/up1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T02-transit-only/ticket/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T02-transit-only/ticket/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T02-transit-only/ticket/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T03-review-open-filters/grader.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T03-review-open-filters/ticket/PROPOSED.diff +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T03-review-open-filters/ticket/TICKET.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T03-review-open-filters/ticket/configs/cust1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T03-review-open-filters/ticket/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T03-review-open-filters/ticket/configs/edge1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T03-review-open-filters/ticket/configs/peer1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T03-review-open-filters/ticket/configs/up1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T03-review-open-filters/ticket/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T03-review-open-filters/ticket/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T03-review-open-filters/ticket/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T04-review-community/grader.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T04-review-community/ticket/PROPOSED.diff +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T04-review-community/ticket/TICKET.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T04-review-community/ticket/configs/cust1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T04-review-community/ticket/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T04-review-community/ticket/configs/edge1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T04-review-community/ticket/configs/peer1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T04-review-community/ticket/configs/up1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T04-review-community/ticket/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T04-review-community/ticket/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T04-review-community/ticket/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T05-review-safe/grader.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T05-review-safe/ticket/PROPOSED.diff +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T05-review-safe/ticket/TICKET.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T05-review-safe/ticket/configs/cust1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T05-review-safe/ticket/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T05-review-safe/ticket/configs/edge1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T05-review-safe/ticket/configs/peer1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T05-review-safe/ticket/configs/up1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T05-review-safe/ticket/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T05-review-safe/ticket/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T05-review-safe/ticket/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T06-peer-wants-transit/grader.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T06-peer-wants-transit/ticket/TICKET.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T06-peer-wants-transit/ticket/configs/cust1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T06-peer-wants-transit/ticket/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T06-peer-wants-transit/ticket/configs/edge1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T06-peer-wants-transit/ticket/configs/peer1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T06-peer-wants-transit/ticket/configs/up1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T06-peer-wants-transit/ticket/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T06-peer-wants-transit/ticket/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T06-peer-wants-transit/ticket/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T07-aggregate/grader.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T07-aggregate/ticket/TICKET.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T07-aggregate/ticket/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T07-aggregate/ticket/configs/r1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T07-aggregate/ticket/configs/r2.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T07-aggregate/ticket/configs/r3.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T07-aggregate/ticket/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T07-aggregate/ticket/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T07-aggregate/ticket/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T08-review-localpref/grader.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T08-review-localpref/ticket/PROPOSED.diff +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T08-review-localpref/ticket/TICKET.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T08-review-localpref/ticket/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T08-review-localpref/ticket/configs/r1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T08-review-localpref/ticket/configs/r2.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T08-review-localpref/ticket/configs/r3.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T08-review-localpref/ticket/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T08-review-localpref/ticket/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T08-review-localpref/ticket/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T09-review-redistribute/grader.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T09-review-redistribute/ticket/PROPOSED.diff +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T09-review-redistribute/ticket/TICKET.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T09-review-redistribute/ticket/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T09-review-redistribute/ticket/configs/r1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T09-review-redistribute/ticket/configs/r2.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T09-review-redistribute/ticket/configs/r3.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T09-review-redistribute/ticket/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T09-review-redistribute/ticket/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T09-review-redistribute/ticket/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T10-broken-lab/grader.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T10-broken-lab/ticket/TICKET.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T10-broken-lab/ticket/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T10-broken-lab/ticket/configs/r1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T10-broken-lab/ticket/configs/r2.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T10-broken-lab/ticket/configs/r3.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T10-broken-lab/ticket/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T10-broken-lab/ticket/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/agent-eval/tickets/T10-broken-lab/ticket/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/arista-frr/configs/ceos1.cfg +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/arista-frr/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/arista-frr/configs/frr1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/arista-frr/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/arista-frr/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/arista-frr/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/batfish-pipeline/README.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/batfish-pipeline/batfish_check.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/batfish-pipeline/changes/default-leak.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/batfish-pipeline/changes/no-policy.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/batfish-pipeline/changes/route-map-typo.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/batfish-pipeline/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/batfish-pipeline/configs/r1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/batfish-pipeline/configs/r2.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/batfish-pipeline/configs/r3.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/batfish-pipeline/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/batfish-pipeline/frr_parse_probe.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/batfish-pipeline/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/batfish-pipeline/pipeline.sh +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/batfish-pipeline/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/frr-bgp/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/frr-bgp/configs/r1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/frr-bgp/configs/r2.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/frr-bgp/configs/r3.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/frr-bgp/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/frr-bgp/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/frr-bgp/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/frr-bgp-broken/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/frr-bgp-broken/r2-bad.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/frr-bgp-broken/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/sonic-frr/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/sonic-frr/configs/frr1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/sonic-frr/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/sonic-frr/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/srl-frr/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/srl-frr/configs/frr1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/srl-frr/configs/srl1.cli +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/srl-frr/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/srl-frr/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/srl-frr/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/srl-hidden-leak/README.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/srl-hidden-leak/configs/daemons +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/srl-hidden-leak/configs/frr1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/srl-hidden-leak/configs/srl1.cli +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/srl-hidden-leak/configs/vtysh.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/srl-hidden-leak/leaks/frr1.conf +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/srl-hidden-leak/leaks/srl1.cli +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/srl-hidden-leak/netdryrun.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/examples/srl-hidden-leak/topology.clab.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/glama.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.1/GATE.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.1/accepted.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.1/corpus-v1-results.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.1/corpus-v2-results.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.1/gate/after-publish.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.1/gate/live.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.1/gate/mutation.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.1/gate/static.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.1/gate/unit.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.1/mutants.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.1/review.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.2/corpus-v1-results.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.2/corpus-v2-results.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.2/gate/live.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.2/gate/mutation.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.2/gate/static.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.2/gate/unit.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.2/mutants.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/releases/0.3.2/review.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/scripts/corpus_case12_verify.sh +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/scripts/dev.sh +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/scripts/e2e.sh +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/scripts/eos_explore.sh +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/scripts/eos_leak_test.sh +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/scripts/mcp_handshake.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/scripts/pack_verify.sh +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/scripts/sonic_config.sh +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/scripts/sonic_explore.sh +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/scripts/sonic_record_test.sh +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/scripts/srl_leak_test.sh +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/scripts/verify-pypi.sh +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/skills/netdryrun/SKILL.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/src/netdryrun/__main__.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/src/netdryrun/drivers/__init__.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/src/netdryrun/drivers/sonic.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/src/netdryrun/faults.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/src/netdryrun/report.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/eos_bgp_paths.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/eos_bgp_paths_empty.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/eos_bgp_paths_local.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/eos_bgp_paths_noexport.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/eos_bgp_route.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/eos_bgp_summary.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/eos_leak_bgp.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/eos_leak_route.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/eos_ping.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/eos_route.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/frr_bgp_neighbors.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/frr_bgp_paths.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/frr_bgp_paths_local.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/frr_bgp_paths_noexport.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/frr_bgp_paths_v6.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/frr_bgp_table_header.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/frr_ping.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/frr_route.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/report-real-failed-bgpd-down.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/report-real-failed-leak.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/report-real-passed.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/sonic_bgp_paths.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/sonic_bgp_paths_empty.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/srl_attr_set_backup.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/srl_attr_set_best.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/srl_attr_set_noexport.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/srl_bgp_neighbors.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/srl_bgp_rib_paths.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/srl_bgp_rib_paths_noexport.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/srl_leak_rib.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/srl_leak_route.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/srl_neighbor_local_pref.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/srl_ping.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/srl_rib_empty.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/fixtures/srl_route.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/README.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0001-advisory-1-misspelled-top-key.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0002-advisory-2-quoted-boolean.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0003-advisory-3-unknown-afi.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0004-advisory-4-unknown-protocol.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0005-advisory-5-duplicate-key.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0006-advisory-6-min-zero.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0007-advisory-7-vrf-all.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0008-frr-vrf-typo-reads-as-no-routes.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0009-eos-ipv6-summary-error-reads-as-zero.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0010-srl-evpn-count-reads-zero.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0011-frr-leak-received-not-installed.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0012-frr-unknown-vrf-session-down.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0013-received-not-selected-state.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0014-runner-findings-pinned.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0015-action-green-on-contradicting-report.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0016-eos-leak-received-not-selected.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0017-srl-leak-received-not-selected.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0018-ipv6-scope-injects-cli-lines.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0019-agent-contract-findings-pinned.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/redteam/RT-0020-no-bgp-neighbors-at-the-cap.yml +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_advisory_fixtures.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_batfish_example.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_bgp_route.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_checks.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_config.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_config_properties.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_config_strict.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_driver_false_pass.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_driver_matrix.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_drivers.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_faults.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_inconclusive.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_invariants.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_load_bearing_keys.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_redteam.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_release_gate_cleanup.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_release_gate_commits.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_release_gate_render.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_report.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_report_schema.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_surfaces_agree.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_verdict_checks.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/tests/test_verdict_config.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/build_llms_full.py +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/.well-known/security.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/2a082c23a5eb0424e4b8be22182c5462.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/CNAME +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/assets/favicon-32.png +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/assets/favicon.ico +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/assets/favicon.svg +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/assets/netdryrun-mark-180.png +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/assets/netdryrun-mark-192.png +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/assets/netdryrun-mark-512.png +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/assets/netdryrun-mark.svg +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/assets/netdryrun-social-1280x640.png +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/assets/netdryrun-wordmark-navy.svg +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/assets/netdryrun-wordmark-white-tm.svg +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/assets/netdryrun-wordmark-white.svg +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/contact.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/01-no-change.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/02-med-to-upstream.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/03-prepend-to-peer.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/04-tag-community-upstream.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/05-description-only.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/06-customer-adds-prefix.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/07-tighten-max-prefix.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/08-upstream-timers.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/09-customer-localpref.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/10-transit-export-filter-removed.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/11-peer-export-filter-removed.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/12-ibgp-send-community-dropped.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/13-upstream-asn-typo.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/14-peer-session-shutdown.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/15-customer-filter-too-strict.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/16-mass-withdrawal.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/17-default-leaked-to-peer.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/18-redistribute-connected.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/19-nexthop-self-removed.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/20-ibgp-shutdown.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/21-customer-max-prefix-removed.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/22-customer-leaks-third-party.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/23-fat-finger-third-party-origin.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/24-customer-blackholed.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/25-aggregate-typo.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/26-route-map-order-bug.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/27-local-asn-typo.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/28-customer-import-policy-removed.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/29-graceful-restart.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/corpus/30-peer-import-filter-removed.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/design-partners.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/examples.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/favicon.ico +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/legal-notice.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/posts/ping-missed-the-leak.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/privacy.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/robots.txt +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/schemas/report-v1.json +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/spec/verdict-v1.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/stylesheets/brand.css +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/support.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/docs/troubleshooting.md +0 -0
- {netdryrun-0.3.2 → netdryrun-0.3.3}/website/hooks/structured_data.py +0 -0
|
@@ -61,6 +61,21 @@ body:
|
|
|
61
61
|
- Yes, I'll run it by hand before changes
|
|
62
62
|
- Maybe, if something changes (say what below)
|
|
63
63
|
- No
|
|
64
|
+
- type: input
|
|
65
|
+
id: keep_price
|
|
66
|
+
attributes:
|
|
67
|
+
label: At what annual price would your team keep this control in its pull requests?
|
|
68
|
+
description: Optional. A number per year for your team, in any currency, or a range. A rough answer is fine. This issue is public; if you'd rather say it privately, email hello@netdryrun.com instead and leave this blank.
|
|
69
|
+
- type: input
|
|
70
|
+
id: too_expensive
|
|
71
|
+
attributes:
|
|
72
|
+
label: At what annual price would it be too expensive to consider?
|
|
73
|
+
description: Optional.
|
|
74
|
+
- type: input
|
|
75
|
+
id: too_cheap
|
|
76
|
+
attributes:
|
|
77
|
+
label: At what annual price would it be so cheap you'd doubt it?
|
|
78
|
+
description: Optional.
|
|
64
79
|
- type: textarea
|
|
65
80
|
id: friction
|
|
66
81
|
attributes:
|
|
@@ -37,11 +37,15 @@ jobs:
|
|
|
37
37
|
example: [frr-bgp, srl-frr]
|
|
38
38
|
steps:
|
|
39
39
|
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
|
40
|
-
-
|
|
40
|
+
- id: ndr
|
|
41
|
+
uses: ./
|
|
41
42
|
with:
|
|
42
43
|
path: examples/${{ matrix.example }}
|
|
43
44
|
artifact-name: report-${{ matrix.example }}
|
|
44
45
|
pr-comment: ${{ matrix.example == 'frr-bgp' }}
|
|
46
|
+
- name: The agent result agrees with the run
|
|
47
|
+
if: always() && steps.ndr.outputs.status != ''
|
|
48
|
+
run: python scripts/check_verdict.py netdryrun-verdict.json "${{ steps.ndr.outputs.status }}"
|
|
45
49
|
|
|
46
50
|
# Proves the tool catches a real mistake: this run MUST fail.
|
|
47
51
|
catches-bad-change:
|
|
@@ -60,6 +64,8 @@ jobs:
|
|
|
60
64
|
echo "status=${{ steps.ndr.outputs.status }} failed=${{ steps.ndr.outputs.failed }}"
|
|
61
65
|
test "${{ steps.ndr.outputs.status }}" = "failed"
|
|
62
66
|
test "${{ steps.ndr.outputs.failed }}" -ge 1
|
|
67
|
+
- name: The agent result says the same, with the failures and the files tested
|
|
68
|
+
run: python scripts/check_verdict.py netdryrun-verdict.json failed
|
|
63
69
|
|
|
64
70
|
# The agent demo: Netdryrun MUST reject attempt 1 on the leak guards only (it opens the filters)
|
|
65
71
|
# and accept attempt 2 (two prefix-list lines). demo.sh exits non-zero otherwise.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# 0.3.3 spike (d): the MCP server inside a GitHub-hosted runner, driven by a scripted MCP client
|
|
2
|
+
# (scripts/mcp_in_runner.py). Runs on pushes to the 0.3.3 branch that touch the server or the spike.
|
|
3
|
+
name: MCP in runner (spike)
|
|
4
|
+
|
|
5
|
+
on:
|
|
6
|
+
push:
|
|
7
|
+
branches: [ndr/agent-runtime]
|
|
8
|
+
paths:
|
|
9
|
+
- "src/netdryrun/**"
|
|
10
|
+
- "scripts/mcp_in_runner.py"
|
|
11
|
+
- ".github/workflows/mcp-in-runner.yml"
|
|
12
|
+
workflow_dispatch:
|
|
13
|
+
|
|
14
|
+
permissions:
|
|
15
|
+
contents: read
|
|
16
|
+
|
|
17
|
+
jobs:
|
|
18
|
+
mcp-in-runner:
|
|
19
|
+
runs-on: ubuntu-latest
|
|
20
|
+
timeout-minutes: 25
|
|
21
|
+
steps:
|
|
22
|
+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
|
23
|
+
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
24
|
+
with:
|
|
25
|
+
python-version: "3.12"
|
|
26
|
+
- name: Install containerlab and Netdryrun with the MCP extra
|
|
27
|
+
run: |
|
|
28
|
+
bash -c "$(curl -sL https://get.containerlab.dev)" -- -v 0.81.1
|
|
29
|
+
python -m pip install --quiet ".[mcp]" jsonschema
|
|
30
|
+
- name: Drive the MCP server with a scripted client
|
|
31
|
+
run: python scripts/mcp_in_runner.py --out "$RUNNER_TEMP/mcp-in-runner.json"
|
|
32
|
+
- name: Upload the spike result
|
|
33
|
+
if: always()
|
|
34
|
+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
35
|
+
with:
|
|
36
|
+
name: mcp-in-runner
|
|
37
|
+
path: ${{ runner.temp }}/mcp-in-runner.json
|
|
38
|
+
if-no-files-found: ignore
|
|
@@ -16,7 +16,7 @@ env:
|
|
|
16
16
|
# The release under test. Pinned by tag on purpose: this must be exactly what the docs tell
|
|
17
17
|
# users to write. Bump it together with the `uses:` refs below after each release; the
|
|
18
18
|
# `install` job fails when PyPI has a newer release than this.
|
|
19
|
-
RELEASE: "0.3.
|
|
19
|
+
RELEASE: "0.3.2"
|
|
20
20
|
|
|
21
21
|
jobs:
|
|
22
22
|
action-passes:
|
|
@@ -24,7 +24,7 @@ jobs:
|
|
|
24
24
|
steps:
|
|
25
25
|
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
|
26
26
|
# Pinned by tag on purpose (not by SHA, not ./): this is the line from the docs.
|
|
27
|
-
- uses: netdryrun/netdryrun@v0.3.
|
|
27
|
+
- uses: netdryrun/netdryrun@v0.3.2
|
|
28
28
|
id: ndr
|
|
29
29
|
with:
|
|
30
30
|
path: examples/frr-bgp
|
|
@@ -46,7 +46,7 @@ jobs:
|
|
|
46
46
|
steps:
|
|
47
47
|
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
|
48
48
|
# Pinned by tag on purpose (not by SHA, not ./): this is the line from the docs.
|
|
49
|
-
- uses: netdryrun/netdryrun@v0.3.
|
|
49
|
+
- uses: netdryrun/netdryrun@v0.3.2
|
|
50
50
|
id: ndr
|
|
51
51
|
continue-on-error: true
|
|
52
52
|
with:
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Pipeline for a GitLab copy of this repository (GitHub stays the source). It tests the CI/CD
|
|
2
|
+
# component in templates/ on the examples and, on a semantic-version tag, creates the release that
|
|
3
|
+
# publishes the component to the GitLab CI/CD Catalog. Needs a runner that allows privileged jobs:
|
|
4
|
+
# gitlab.com's default hosted Linux runners do.
|
|
5
|
+
stages: [test, verify, release]
|
|
6
|
+
|
|
7
|
+
include:
|
|
8
|
+
# The component at this commit, with Netdryrun from this commit (not the release on PyPI)
|
|
9
|
+
- component: $CI_SERVER_FQDN/$CI_PROJECT_PATH/netdryrun@$CI_COMMIT_SHA
|
|
10
|
+
inputs:
|
|
11
|
+
job-name: netdryrun-frr-bgp
|
|
12
|
+
path: examples/frr-bgp
|
|
13
|
+
netdryrun-package: "."
|
|
14
|
+
- component: $CI_SERVER_FQDN/$CI_PROJECT_PATH/netdryrun@$CI_COMMIT_SHA
|
|
15
|
+
inputs:
|
|
16
|
+
job-name: netdryrun-frr-bgp-broken
|
|
17
|
+
path: examples/frr-bgp-broken
|
|
18
|
+
netdryrun-package: "."
|
|
19
|
+
|
|
20
|
+
# The broken example leaks routes: Netdryrun must catch it (exit code 1). Any other result fails.
|
|
21
|
+
netdryrun-frr-bgp-broken:
|
|
22
|
+
allow_failure:
|
|
23
|
+
exit_codes: [1]
|
|
24
|
+
|
|
25
|
+
frr-bgp-broken-must-fail:
|
|
26
|
+
stage: verify
|
|
27
|
+
image: busybox:1.37
|
|
28
|
+
needs: [netdryrun-frr-bgp-broken]
|
|
29
|
+
script:
|
|
30
|
+
- echo "netdryrun-frr-bgp-broken status=$NETDRYRUN_STATUS failed=$NETDRYRUN_FAILED"
|
|
31
|
+
- test "$NETDRYRUN_STATUS" = failed && test "$NETDRYRUN_FAILED" -gt 0
|
|
32
|
+
|
|
33
|
+
# The Catalog only lists releases made by the release keyword in a pipeline, from tags like 0.4.0
|
|
34
|
+
release:
|
|
35
|
+
stage: release
|
|
36
|
+
image: registry.gitlab.com/gitlab-org/cli:latest
|
|
37
|
+
rules:
|
|
38
|
+
- if: $CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+$/
|
|
39
|
+
script:
|
|
40
|
+
- echo "Releasing the Netdryrun component $CI_COMMIT_TAG"
|
|
41
|
+
release:
|
|
42
|
+
tag_name: $CI_COMMIT_TAG
|
|
43
|
+
description: "Netdryrun $CI_COMMIT_TAG. Changelog: https://github.com/netdryrun/netdryrun/blob/main/CHANGELOG.md"
|
|
@@ -28,6 +28,6 @@ A configured session that is down always counts as `failed`, never `inconclusive
|
|
|
28
28
|
- Netdryrun never touches production devices. Only run topologies you were given or created in the lab folder.
|
|
29
29
|
|
|
30
30
|
## MCP
|
|
31
|
-
`netdryrun mcp` starts an MCP server (stdio) with the tools <!-- facts:mcp_tools -->`dry_run`, `run_checks`, `record_baseline`, `validate` and `check_types`<!-- /facts:mcp_tools -->. Each result has a separate `verdict` field: `CHECKS PASSED (only what the checks cover)`, `DO NOT APPLY THIS CHANGE`, or `INCONCLUSIVE: the lab did not reach a testable state; this says nothing about your change. Do not apply it until a run passes.` `CHECKS PASSED (only what the checks cover)` means only that every check in the file passed in the lab; it says nothing about behavior no check covers. The three verdict strings are frozen from 0.3.2 (0.3.1 said `SAFE TO PROCEED` for a pass); decide on `status`, not the text. From 0.3.2, a failed result lists each failed check under `failures` with its `reason`, an `evidence` excerpt and one read-only `next_diagnostic` command for its node (run it on a lab kept with `keep_lab: true`). From 0.3.2, `tested` lists a sha256 of `netdryrun.yml`, the topology and every config file the run used, read before it started: the verdict applies to those exact files, so run again after any edit. Result schema: https://netdryrun.com/schemas/mcp-result-v1.json. The rules as an installable agent skill: `skills/netdryrun/SKILL.md`. Add it with `claude mcp add netdryrun -- netdryrun mcp`, or run it with `uvx netdryrun mcp`.
|
|
31
|
+
`netdryrun mcp` starts an MCP server (stdio) with the tools <!-- facts:mcp_tools -->`dry_run`, `start_run`, `get_result`, `run_checks`, `record_baseline`, `validate` and `check_types`<!-- /facts:mcp_tools -->. Each result has a separate `verdict` field: `CHECKS PASSED (only what the checks cover)`, `DO NOT APPLY THIS CHANGE`, or `INCONCLUSIVE: the lab did not reach a testable state; this says nothing about your change. Do not apply it until a run passes.` `CHECKS PASSED (only what the checks cover)` means only that every check in the file passed in the lab; it says nothing about behavior no check covers. The three verdict strings are frozen from 0.3.2 (0.3.1 said `SAFE TO PROCEED` for a pass); decide on `status`, not the text. From 0.3.2, a failed result lists each failed check under `failures` with its `reason`, an `evidence` excerpt and one read-only `next_diagnostic` command for its node (run it on a lab kept with `keep_lab: true`). From 0.3.2, `tested` lists a sha256 of `netdryrun.yml`, the topology and every config file the run used, read before it started: the verdict applies to those exact files, so run again after any edit. Result schema: https://netdryrun.com/schemas/mcp-result-v1.json. The rules as an installable agent skill: `skills/netdryrun/SKILL.md`. Add it with `claude mcp add netdryrun -- netdryrun mcp`, or run it with `uvx netdryrun mcp`.
|
|
32
32
|
|
|
33
33
|
Docs: https://netdryrun.com/ai-agents/ · Full docs in one file: https://netdryrun.com/llms-full.txt
|
|
@@ -1,5 +1,43 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.3.3 (2026-10-30)
|
|
4
|
+
|
|
5
|
+
The agent runtime: an agent with no local Docker gets a verdict through a pull request, MCP calls answer within seconds, and `record` writes baselines that survive normal growth.
|
|
6
|
+
|
|
7
|
+
### Changed
|
|
8
|
+
|
|
9
|
+
- **Verdict Corpus v2: case 18 is nondeterministic in this lab, and the v2 score is 16 of 17 deterministic cases, not 17/18.** The 0.3.3 release gate found that `18-aggregate-discard-route-removed` gives different verdicts on the same build: FRR's BGP import check sometimes withholds the aggregate and sometimes announces it (three reruns: unsafe, unsafe, safe). The earlier note that 0.3.1 fixed this case was wrong, and is kept, struck through, with the correction. The case is listed in `corpus/v2/NONDETERMINISTIC.yml` with every verdict seen, left out of every score, and run three times by the release gate, which records it and never blocks on it. The case file and its pre-registered expected verdict are unchanged; a replacement without the race goes into a later corpus version.
|
|
10
|
+
- **The release gate's "no harmful change approved" line now works.** It looked for cases expected `harmful`, a word no corpus case uses (they say `unsafe`), so it could never fail. It is now pinned on the committed results.
|
|
11
|
+
- **The MCP Registry description and the PyPI summary** now start with the same searchable sentence: "Validate BGP and routing changes in a real containerlab lab: pass/fail verdict with evidence."
|
|
12
|
+
- **A run that deployed from files edited while it was in progress is an error.** The agent result (MCP `dry_run`, `start_run`, `run --verdict`) hashes the files again at the end; if any changed, the lab may have run either version, so the result is `status: error`, `DO NOT APPLY THIS CHANGE`, with the files in `tested.changed_during_run`: run again. Found by the 0.3.3 cold review.
|
|
13
|
+
- **The PR comment and MR note: one per lab, with the commit, from the expected author.** The marker names the lab's `path`, so two labs on one PR don't overwrite each other's header. The header names the head commit, so a header from an older push can be told apart, and a run that wrote no report rewrites it as `error`. Only a comment by `comment-author` (default `github-actions[bot]`; on GitLab, the token's own user) is updated, so a look-alike comment posted by someone else is ignored. Found by the 0.3.3 cold review.
|
|
14
|
+
- **The MCP server destroys the labs it started when the client disconnects** during a `start_run`, and a run whose worker stops on anything at all ends as `status: error`, never stuck at `running`.
|
|
15
|
+
- **The PR comment can't carry a look-alike header.** Device output in the summary can no longer open an HTML comment, so the only `netdryrun-verdict` line is the real one; the summary is cut to keep the header within GitHub's comment limit.
|
|
16
|
+
- **A directory committed at a report path is removed before the run** (Action and GitLab component), so it can't make the report write fail and turn a failed run into `error`.
|
|
17
|
+
- **`netdryrun record` writes prefix-count ranges by default:** today's count as `min`, and `max` with 20% headroom (at least 2), instead of an exact count that broke on normal growth. A neighbor that sends nothing stays `exact: 0`. `--exact-prefixes` keeps the old behaviour; `--min-prefixes` records minimums as before, except that a count of 0 is now `exact: 0` (0.3.2 wrote `min: 0`, which `validate` refuses). The MCP `record_baseline` tool does the same.
|
|
18
|
+
- **A truncated baseline is loud.** When a node has more BGP routes than `record` writes (50 per node and address family), the file starts with an `# INCOMPLETE BASELINE` line naming the node and the count, and the line is printed on stderr.
|
|
19
|
+
|
|
20
|
+
### Fixes
|
|
21
|
+
|
|
22
|
+
- **A link-local `next_hop` in a `bgp_route` check is refused.** Devices report it without its interface and two sessions can share one, so a path through the other session matched. Found by the 0.3.3 delta review.
|
|
23
|
+
- **A run on an FRR lab that skipped a config line is an error, never a pass.** FRR loads its config line by line at boot and skips a line it rejects (a typo such as `default-orignate`, an incomplete command); 0.3.2 ran the checks on the lab without it, so a run could pass on a change that never loaded. After the checks, Netdryrun now reads every FRR node's boot log, and any rejected line makes the run `error` (exit 2), naming the node, line and reason. A daemon can complain about a line and still apply it (bgpd answers `bgp graceful-restart` with "Failure to communicate" on FRR 10.4.1). So such a line doesn't count when line N of the node's `frr.conf` is that line and that daemon's own running config (`vtysh -d <daemon>`) has it under the same headers at every depth (`router bgp`, VRF, address family, `vni`), at least as many times as the file does, and has the line's whole block as written. A refused header line (`vni 10`, `address-family ...`) always counts. Every other kind of rejection always counts, and so does any line when the running config or `frr.conf` can't be read. A boot log that can't be read, or that shows no sign of a config load, is an error too. Nodes whose driver can't be detected are read as well, and only the current boot counts. Advisory: `advisories/0.3.2-rejected-config`. Not yet checked on Arista cEOS, SR Linux or SONiC startup config (config applied with `setup:` commands is): a documented limit, fix scheduled for 0.4.
|
|
24
|
+
- **A path with no MED attribute no longer passes `med: 0`** (FRR, EOS, SR Linux). Its MED was read as 0, but with `bgp bestpath med missing-as-worst` a missing MED is the worst, not 0. The MED of such a path is now unknown, so any `med` check fails on it and says `MED is not set`. Found by a Grok review.
|
|
25
|
+
- **A link-local neighbor shared by several sessions is never evaluated** (FRR, SR Linux, EOS). Neighbors such as `fe80::1%eth1` and `fe80::1%eth2` share one key, so one session's state or prefix count answered for the other: a down session could hide behind an up one, a `state: down` check could pass while one was up, and a `max` count could be read from the wrong session. A `bgp_session` or `bgp_prefixes` check on such a key now says it is ambiguous and is not evaluated (never a pass); check a global address instead. `record` leaves such a neighbor out with an `INCOMPLETE BASELINE` line. Found by the 0.3.3 cross-platform and delta reviews.
|
|
26
|
+
- **Every SR Linux BGP route type counts as BGP** (`bgp-evpn`, `bgp-vpn`, `bgp-label`, ...), so a `protocol: bgp` check, and a `present: false` guard, see a default route imported through EVPN or L3VPN.
|
|
27
|
+
- **EOS `Idle(Admin)` reads as `idle`** (EOS prints no space before the reason), so `state: idle` matches an admin-down peer.
|
|
28
|
+
- **`record` says when a link has no address where it is read** (SONiC keeps it on `EthernetN`, SR Linux on `e1-Y.0`), with an `INCOMPLETE BASELINE` line, instead of writing no guard silently.
|
|
29
|
+
- **A route check on FRR never passes on a graceful-restart stale path.** While a peer restarts, bgpd keeps its paths marked stale until it re-sends them or the stale timer ends; a `route` or `bgp_route` check that sees one now reads it as not settled (not evaluated, never a pass), whether it expects the route present or absent. Found by the 0.3.3 cross-platform review.
|
|
30
|
+
- **Known limits stated**, with the mitigation and a fix scheduled for 0.4, in the advisory, the config and `record` references and the compatibility matrix: on SR Linux, EOS and SONiC a graceful-restart stale path still counts as present; SR Linux prefix counts include routes dropped by loop detection; `record` snapshots the default VRF only.
|
|
31
|
+
|
|
32
|
+
### Added
|
|
33
|
+
|
|
34
|
+
- **A GitLab CI/CD component, not listed yet** (`templates/netdryrun.yml`, docs in `templates/README.md`): the GitHub Action's flow for GitLab, with the agent result kept as an artifact and an optional merge request note with the same fixed header. It is unlisted until it has run in a real GitLab pipeline; it has run locally with gitlab-ci-local, and the note only against a stub of the GitLab API.
|
|
35
|
+
- **`record` adds the standard guards by default** (and `record_baseline`): on every BGP node, no default route via BGP and no link subnet of the topology via BGP. Each guard is run on the recording lab first; one the lab already fails (a default an edge router receives from upstream, selected or not) is left out, with a comment in the file saying why. A sample that errors is retried; a guard that never gets a real answer is left out with an `INCOMPLETE BASELINE` line, never silently. A prefix-count range has room for one leaked default route, so the guard is what catches it (found by the 0.3.3 multi-vendor audit). `--no-guards` (`guards: false`) leaves them out. A route read that fails, or more link subnets than are guarded, is an `INCOMPLETE BASELINE` line.
|
|
36
|
+
- **Each `start_run` writes its own evidence file**, `netdryrun-report-<run_id>.json`, so several runs of one folder don't share the last one's report.
|
|
37
|
+
- **Exact MCP setup for Cursor, VS Code (Copilot agent mode), Windsurf and Codex CLI** on the AI agents page: the file, its location and the block for each, checked against each client's documentation on 2026-10-28 and linked to it. VS Code's key is `servers`, not `mcpServers`; Codex CLI's 60-second default tool timeout is shorter than a dry run, so the page raises it or points to `start_run`.
|
|
38
|
+
- **The pull-request path for agents without Docker.** `netdryrun run --verdict FILE` (and `check`) writes the agent result, the same `netdryrun.mcp-result/v1` object the MCP tools return, with the files tested. The GitHub Action writes it (`verdict-report`, default `netdryrun-verdict.json`) and uploads it with the reports, and its PR comment now starts with a fixed header: a `<!-- netdryrun-verdict: {...} -->` line with the status, verdict and counts as JSON, then the verdict in words. The header is held to the run's checked exit code: a missing or contradicting result reads `error`. New guide: [AI agents without Docker](https://netdryrun.com/guides/agent-pr/).
|
|
39
|
+
- **An asynchronous MCP shape.** `start_run` returns at once with a `run_id` and `status: running` (no verdict), and `get_result` polls it until the run finishes; the finished result is exactly `dry_run`'s, with its `run_id` (or a setup error, `status: error`, if the run stopped on an internal error). A dry run takes about 60–120 s, longer than some clients wait for one tool call (Codex CLI's default per-tool timeout is 60 s). A config error or a path outside `NETDRYRUN_MCP_ROOT` is answered at once; a run that stops on an internal error ends as `status: error`, never stuck at `running`. Runs live in the server process. The pending shape, `run_pending`, is in the result schema. `dry_run` is unchanged.
|
|
40
|
+
|
|
3
41
|
## 0.3.2 (2026-10-28)
|
|
4
42
|
|
|
5
43
|
The agent contract: what an AI agent needs to choose Netdryrun, read its result and act on it safely. Tool names and parameters are unchanged.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: netdryrun
|
|
3
|
-
Version: 0.3.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 0.3.3
|
|
4
|
+
Summary: Validate BGP and routing changes in a real containerlab lab: pass/fail verdict with evidence. Finds the route leaks that ping and 'BGP up' checks miss. CLI, GitHub Action and MCP server.
|
|
5
5
|
Project-URL: Homepage, https://netdryrun.com
|
|
6
6
|
Project-URL: Documentation, https://netdryrun.com/getting-started/
|
|
7
7
|
Project-URL: Changelog, https://github.com/netdryrun/netdryrun/blob/main/CHANGELOG.md
|
|
@@ -75,7 +75,7 @@ FAILED: 5 passed, 1 failed (6 checks)
|
|
|
75
75
|
That run caught a leaked default route that was *received but not selected*. Ping and session checks alone would miss this example: every session is up and every address answers. (A static analyzer such as Batfish also catches this particular leak; the [Batfish page](https://netdryrun.com/batfish/) shows one that it misses and a lab catches.)
|
|
76
76
|
|
|
77
77
|
<!-- facts:readme_status -->
|
|
78
|
-
> Status: **v0.3.
|
|
78
|
+
> Status: **v0.3.3.** Runs on Linux and WSL2, with drivers for FRR, Nokia SR Linux, SONiC (docker-sonic-vs) and Arista EOS (cEOS, bring your own image). On the [verdict corpus](https://netdryrun.com/corpus/) v1 (30 realistic BGP changes: 19 harmful, 11 safe), Netdryrun judged 24/30 correctly, ping-only checks 20/30 and all-sessions-up checks 15/30. Netdryrun approved 5 of the 19 harmful changes (ping-only checks: 10 of 19) and blocked 1 of the 11 safe ones. On v2 (18 cases written after v1 to target v1's misses): Netdryrun 16 of 17 deterministic cases correct, ping-only 9/17, all-sessions-up 7/17; case 18 is nondeterministic in this lab and excluded from the score (listed with its reruns). Expected verdicts were committed before any run, on one FRR topology: a comparison of testing methods, not a production reliability estimate. Method, every miss and a one-command rerun are on the corpus page. Provenance: both ran on Netdryrun 0.3.1.
|
|
79
79
|
<!-- /facts:readme_status -->
|
|
80
80
|
|
|
81
81
|
## Why
|
|
@@ -197,7 +197,7 @@ jobs:
|
|
|
197
197
|
runs-on: ubuntu-latest
|
|
198
198
|
steps:
|
|
199
199
|
- uses: actions/checkout@v4
|
|
200
|
-
- uses: netdryrun/netdryrun@v0.3.
|
|
200
|
+
- uses: netdryrun/netdryrun@v0.3.3
|
|
201
201
|
with:
|
|
202
202
|
path: network/ # folder containing netdryrun.yml
|
|
203
203
|
```
|
|
@@ -258,7 +258,7 @@ What Netdryrun adds: positive checks wait for convergence, negative checks run o
|
|
|
258
258
|
|
|
259
259
|
- Lab routers are not your production routers. A pass on FRR, SR Linux, SONiC VS or Arista cEOS is strong evidence for protocol logic and policy intent, but not proof of hardware, scale or version-specific behavior. For Arista, bring your own licensed cEOS image to test real vendor code.
|
|
260
260
|
- Checks cover IPv4/IPv6 unicast BGP (sessions, prefix counts, best-path attributes), routes and ping, with link-down, node-isolation and route-injection (FRR only) failure scenarios. Route checks work for routes from any protocol today (`protocol: ospf`, `isis`, `static`, `connected`, ...); OSPF/IS-IS *adjacency* checks and path/traceroute are coming.
|
|
261
|
-
- No Cisco or Juniper driver yet
|
|
261
|
+
- No Cisco or Juniper driver in a release yet. Their lab images need a vendor account and can't be redistributed, so their drivers will be bring-your-own-image, like cEOS; Juniper and Cisco NX-OS are next on the [roadmap](https://netdryrun.com/faq/#why-not-cisco-or-juniper).
|
|
262
262
|
|
|
263
263
|
How results are checked and released, with evidence for each claim: [How it's built](https://netdryrun.com/how-its-built/).
|
|
264
264
|
|
|
@@ -32,7 +32,7 @@ FAILED: 5 passed, 1 failed (6 checks)
|
|
|
32
32
|
That run caught a leaked default route that was *received but not selected*. Ping and session checks alone would miss this example: every session is up and every address answers. (A static analyzer such as Batfish also catches this particular leak; the [Batfish page](https://netdryrun.com/batfish/) shows one that it misses and a lab catches.)
|
|
33
33
|
|
|
34
34
|
<!-- facts:readme_status -->
|
|
35
|
-
> Status: **v0.3.
|
|
35
|
+
> Status: **v0.3.3.** Runs on Linux and WSL2, with drivers for FRR, Nokia SR Linux, SONiC (docker-sonic-vs) and Arista EOS (cEOS, bring your own image). On the [verdict corpus](https://netdryrun.com/corpus/) v1 (30 realistic BGP changes: 19 harmful, 11 safe), Netdryrun judged 24/30 correctly, ping-only checks 20/30 and all-sessions-up checks 15/30. Netdryrun approved 5 of the 19 harmful changes (ping-only checks: 10 of 19) and blocked 1 of the 11 safe ones. On v2 (18 cases written after v1 to target v1's misses): Netdryrun 16 of 17 deterministic cases correct, ping-only 9/17, all-sessions-up 7/17; case 18 is nondeterministic in this lab and excluded from the score (listed with its reruns). Expected verdicts were committed before any run, on one FRR topology: a comparison of testing methods, not a production reliability estimate. Method, every miss and a one-command rerun are on the corpus page. Provenance: both ran on Netdryrun 0.3.1.
|
|
36
36
|
<!-- /facts:readme_status -->
|
|
37
37
|
|
|
38
38
|
## Why
|
|
@@ -154,7 +154,7 @@ jobs:
|
|
|
154
154
|
runs-on: ubuntu-latest
|
|
155
155
|
steps:
|
|
156
156
|
- uses: actions/checkout@v4
|
|
157
|
-
- uses: netdryrun/netdryrun@v0.3.
|
|
157
|
+
- uses: netdryrun/netdryrun@v0.3.3
|
|
158
158
|
with:
|
|
159
159
|
path: network/ # folder containing netdryrun.yml
|
|
160
160
|
```
|
|
@@ -215,7 +215,7 @@ What Netdryrun adds: positive checks wait for convergence, negative checks run o
|
|
|
215
215
|
|
|
216
216
|
- Lab routers are not your production routers. A pass on FRR, SR Linux, SONiC VS or Arista cEOS is strong evidence for protocol logic and policy intent, but not proof of hardware, scale or version-specific behavior. For Arista, bring your own licensed cEOS image to test real vendor code.
|
|
217
217
|
- Checks cover IPv4/IPv6 unicast BGP (sessions, prefix counts, best-path attributes), routes and ping, with link-down, node-isolation and route-injection (FRR only) failure scenarios. Route checks work for routes from any protocol today (`protocol: ospf`, `isis`, `static`, `connected`, ...); OSPF/IS-IS *adjacency* checks and path/traceroute are coming.
|
|
218
|
-
- No Cisco or Juniper driver yet
|
|
218
|
+
- No Cisco or Juniper driver in a release yet. Their lab images need a vendor account and can't be redistributed, so their drivers will be bring-your-own-image, like cEOS; Juniper and Cisco NX-OS are next on the [roadmap](https://netdryrun.com/faq/#why-not-cisco-or-juniper).
|
|
219
219
|
|
|
220
220
|
How results are checked and released, with evidence for each claim: [How it's built](https://netdryrun.com/how-its-built/).
|
|
221
221
|
|
|
@@ -15,6 +15,9 @@ inputs:
|
|
|
15
15
|
json-report:
|
|
16
16
|
description: Path for the JSON evidence report
|
|
17
17
|
default: netdryrun-report.json
|
|
18
|
+
verdict-report:
|
|
19
|
+
description: "Path for the agent result: status, verdict, failures and the sha256 of every file tested, as JSON in the shape the MCP tools return (netdryrun.mcp-result/v1). Uploaded with the reports"
|
|
20
|
+
default: netdryrun-verdict.json
|
|
18
21
|
junit-report:
|
|
19
22
|
description: Path for the JUnit XML report
|
|
20
23
|
default: netdryrun-junit.xml
|
|
@@ -33,6 +36,9 @@ inputs:
|
|
|
33
36
|
pr-comment:
|
|
34
37
|
description: "On pull requests, post (and keep updating) one comment with the verdict and evidence. Needs permissions pull-requests: write"
|
|
35
38
|
default: "false"
|
|
39
|
+
comment-author:
|
|
40
|
+
description: "The account that writes the PR comment: only its comment is updated, so a comment someone else posted with the same marker is ignored. The default is the account behind the default github-token; set it when you pass another token"
|
|
41
|
+
default: github-actions[bot]
|
|
36
42
|
github-token:
|
|
37
43
|
description: Token for the PR comment
|
|
38
44
|
default: ${{ github.token }}
|
|
@@ -97,19 +103,29 @@ runs:
|
|
|
97
103
|
NDR_PATH: ${{ inputs.path }}
|
|
98
104
|
NDR_JSON: ${{ inputs.json-report }}
|
|
99
105
|
NDR_JUNIT: ${{ inputs.junit-report }}
|
|
106
|
+
NDR_VERDICT: ${{ inputs.verdict-report }}
|
|
100
107
|
NDR_SETTLE: ${{ inputs.settle }}
|
|
101
108
|
NDR_HOLD: ${{ inputs.hold }}
|
|
102
109
|
run: |
|
|
103
110
|
set +e
|
|
104
111
|
# Reports left in the workspace (committed in the PR, or from an earlier run on a self-hosted
|
|
105
112
|
# runner) must never be read, posted or uploaded as this run's result
|
|
106
|
-
|
|
113
|
+
# A directory at a report path (committed in the PR) would make the report write fail and turn a
|
|
114
|
+
# failed run into "error": remove it too, but never the workspace or anything above it.
|
|
115
|
+
for f in "$NDR_JSON" "$NDR_JUNIT" "$NDR_VERDICT" netdryrun-summary.md netdryrun-comment.json; do
|
|
116
|
+
if [ -d "$f" ] && [ ! -L "$f" ] && [ "$(realpath -- "$f")" != "$(realpath .)" ] \
|
|
117
|
+
&& case "$(realpath .)/" in "$(realpath -- "$f")/"*) false;; *) true;; esac; then
|
|
118
|
+
sudo rm -rf -- "$f"
|
|
119
|
+
else
|
|
120
|
+
sudo rm -f -- "$f"
|
|
121
|
+
fi
|
|
122
|
+
done
|
|
107
123
|
# containerlab needs root on GitHub-hosted runners. Pass only what Netdryrun needs (not the
|
|
108
124
|
# job's tokens, not the runner user's home). This only narrows the environment: a lab runs
|
|
109
125
|
# privileged containers, so run untrusted topologies only in jobs without secrets.
|
|
110
126
|
sudo env -i "PATH=$PATH" "HOME=/root" "GITHUB_STEP_SUMMARY=$GITHUB_STEP_SUMMARY" \
|
|
111
127
|
netdryrun run "$NDR_PATH" \
|
|
112
|
-
--json "$NDR_JSON" --junit "$NDR_JUNIT" --markdown netdryrun-summary.md --settle "$NDR_SETTLE" --hold "$NDR_HOLD"
|
|
128
|
+
--json "$NDR_JSON" --junit "$NDR_JUNIT" --verdict "$NDR_VERDICT" --markdown netdryrun-summary.md --settle "$NDR_SETTLE" --hold "$NDR_HOLD"
|
|
113
129
|
echo "exit_code=$?" >> "$GITHUB_OUTPUT"
|
|
114
130
|
|
|
115
131
|
- name: Read result
|
|
@@ -147,6 +163,7 @@ runs:
|
|
|
147
163
|
path: |
|
|
148
164
|
${{ inputs.json-report }}
|
|
149
165
|
${{ inputs.junit-report }}
|
|
166
|
+
${{ inputs.verdict-report }}
|
|
150
167
|
if-no-files-found: ignore
|
|
151
168
|
retention-days: 14
|
|
152
169
|
|
|
@@ -155,21 +172,28 @@ runs:
|
|
|
155
172
|
shell: bash
|
|
156
173
|
env:
|
|
157
174
|
GH_TOKEN: ${{ inputs.github-token }}
|
|
158
|
-
|
|
159
|
-
|
|
175
|
+
NDR_COMMENT_API: ${{ github.api_url }}
|
|
176
|
+
NDR_REPO: ${{ github.repository }}
|
|
177
|
+
NDR_PR: ${{ github.event.pull_request.number }}
|
|
178
|
+
NDR_COMMIT: ${{ github.event.pull_request.head.sha }}
|
|
179
|
+
NDR_LABEL: ${{ inputs.path }}
|
|
180
|
+
NDR_AUTHOR: ${{ inputs.comment-author }}
|
|
181
|
+
NDR_VERDICT: ${{ inputs.verdict-report }}
|
|
182
|
+
NDR_SUMMARY: netdryrun-summary.md
|
|
183
|
+
NDR_STATUS: ${{ steps.result.outputs.status }}
|
|
160
184
|
run: |
|
|
161
|
-
#
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
python -
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
185
|
+
# One comment per lab, updated in place: a fixed header (status, verdict, counts and the PR head commit,
|
|
186
|
+
# held to the run's checked status), then the summary. A run that wrote no report still rewrites the
|
|
187
|
+
# header, as error. A comment problem is only a warning: it never turns a test result red.
|
|
188
|
+
python - <<'EOF'
|
|
189
|
+
import os, sys
|
|
190
|
+
try:
|
|
191
|
+
from netdryrun.comment import main_github
|
|
192
|
+
except ImportError as e:
|
|
193
|
+
print(f"::warning title=Netdryrun::could not comment on the PR (Netdryrun is not installed: {e})")
|
|
194
|
+
sys.exit(0)
|
|
195
|
+
sys.exit(main_github(os.environ))
|
|
196
|
+
EOF
|
|
173
197
|
|
|
174
198
|
- name: Upload evidence to Netdryrun Cloud
|
|
175
199
|
id: cloud
|
|
@@ -38,6 +38,9 @@ These need a running lab to show, so the proof is in the test suite, which runs
|
|
|
38
38
|
|
|
39
39
|
## Limits that remain in 0.3.1
|
|
40
40
|
- **A node that never reports BGP neighbors.** In 0.3.1 its negative and attribute checks waited up to their timeout, then still ran, and could pass. **Fixed in 0.3.2:** when the file has a BGP check on that node (`bgp_session`, `bgp_prefixes`, `bgp_route`, or a `route` with `protocol: bgp`), those checks fail as "not evaluated" at the cap. A node with no BGP check anywhere in the file still goes ahead with a warning in each result, since it may not run BGP at all.
|
|
41
|
+
- **Graceful restart on SR Linux, EOS and SONiC** (found by the 0.3.3 cross-platform review): while a peer restarts with graceful restart, its routes stay in the table marked stale until it re-sends them or the stale timer ends, and on these platforms the BGP checks count a stale path as present and best. A route check can pass on a route that is about to go. On FRR, Netdryrun reads the stale flag: a route or `bgp_route` check never passes on a stale path (0.3.3). Mitigation: leave graceful restart off in the lab, or give the checks a `timeout` above the stale timer. Fix scheduled: 0.4 (read the stale flag on the other platforms, from a live graceful-restart capture).
|
|
42
|
+
- **SR Linux prefix counts** (found by the 0.3.3 cross-platform review, seen on a live SR Linux 26.7.2): a `bgp_prefixes` count on SR Linux includes routes dropped by BGP loop detection, which are not usable, so a `min` or `exact` check can pass with fewer usable prefixes. Mitigation: pair the count with `route` checks for the prefixes that matter. Fix scheduled: 0.4 (count usable routes only).
|
|
43
|
+
- **Startup config on Arista cEOS, Nokia SR Linux and SONiC**: Netdryrun doesn't yet check that every line of a startup config loaded, so a line the NOS rejected leaves the lab running without it. Config applied with `setup:` commands is checked. Mitigation: for each changed line, add a positive check that holds only if the line loaded. Fix scheduled: 0.4. (FRR is checked from 0.3.3: see [the 0.3.2 advisory](../0.3.2-rejected-config/README.md).)
|
|
41
44
|
- **A slow peer** (still a limit in 0.3.2). A neighbor with a long advertisement interval can deliver a route after the gate released a negative check. Raise `--settle` and `--hold` above your slowest advertisement interval. Planned for 0.4: read each peer's configured advertisement interval and extend the settle window automatically.
|
|
42
45
|
- **`afi: evpn`** is refused on Arista EOS and Nokia SR Linux (their drivers can't read an EVPN count). On FRR and SONiC the count is read from the router's BGP summary; a family the neighbor doesn't carry counts as 0 received prefixes.
|
|
43
46
|
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Advisory: Netdryrun 0.3.2 could pass on an FRR lab that skipped a config line
|
|
2
|
+
|
|
3
|
+
**Affected:** netdryrun 0.3.2 and earlier, and the `netdryrun/netdryrun@v0.3.2` Action, on labs with FRR nodes. **Fixed in:** 0.3.3. **Found by:** our own 0.4 work, which reproduced it on a live lab, before any outside user reported it.
|
|
4
|
+
|
|
5
|
+
## What happened
|
|
6
|
+
|
|
7
|
+
FRR loads its configuration line by line when the container boots. A line it rejects is skipped, and the router keeps running without it. Examples are a typo such as `neighbor 10.0.12.2 default-orignate`, an incomplete command, or a command this FRR release doesn't have. The only trace is a line in the container's log.
|
|
8
|
+
|
|
9
|
+
Netdryrun 0.3.2 didn't read that log. The checks ran on a lab without the changed line, so they tested a network the change never reached, and a run could pass. A check that the change must not break something passes trivially when the change isn't there.
|
|
10
|
+
|
|
11
|
+
## What 0.3.3 does
|
|
12
|
+
|
|
13
|
+
After the checks, Netdryrun reads the boot log of every FRR node in the lab, including nodes no check names. If FRR rejected any line, the run is an **error (exit 2)**, never a pass. The message names the node, the line number, the line and the reason:
|
|
14
|
+
|
|
15
|
+
```text
|
|
16
|
+
FRR rejected configuration lines at boot, so the lab ran without them and the checks did not test the change as
|
|
17
|
+
written: r1: line 21: neighbor 10.0.12.2 default-orignate (Unknown command); line 22: neighbor 10.0.12.2
|
|
18
|
+
maximum-prefix (Command incomplete); line 23: neighbor 10.9.9.9 route-map ALLOW-ALL in (rejected by bgpd). Fix the
|
|
19
|
+
config and run again.
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
A daemon can complain about a line and still apply it (bgpd answers `bgp graceful-restart` with "Failure to communicate" on FRR 10.4.1). So such a line doesn't count when line N of the node's `frr.conf` is that line and that daemon's own running config (`vtysh -d <daemon>`) has it under the same headers at every depth (`router bgp`, VRF, address family, `vni`), at least as many times as the file does, and has the line's whole block as written. A refused header line (`vni 10`, `address-family ...`) always counts. Every other kind of rejection always counts, and so does any line when the running config or `frr.conf` can't be read. (Without that, 0.3.3's own release gate saw a false error on Verdict Corpus case 29.) A boot log that can't be read is an error too. Config applied with `setup:` commands already failed the run in 0.3.2, because `vtysh -c` exits 1 on a rejected line.
|
|
23
|
+
|
|
24
|
+
## Proof
|
|
25
|
+
|
|
26
|
+
The tests run on every commit. They are built from real `docker logs` output of `quay.io/frrouting/frr:10.4.1`, started as containerlab starts it, with the three bad lines above:
|
|
27
|
+
|
|
28
|
+
- `tests/test_frr_config_load.py`: the rejected lines are read from the real boot log; a clean boot still passes; a rejected line on any FRR node makes the run an error, with or without `--deploy`; an unreadable log is an error; a rejected `setup:` command stops the run.
|
|
29
|
+
- Fixtures: `tests/fixtures/frr_boot_log_rejected.txt`, `frr_boot_log_rejected.conf`, `frr_boot_log_clean.txt`.
|
|
30
|
+
|
|
31
|
+
To reproduce, add a misspelled line to a router's `frr.conf` in any FRR lab, for example `examples/frr-bgp`, and run the same checks. 0.3.2 reports what the checks saw. 0.3.3 stops with exit 2 and names the line.
|
|
32
|
+
|
|
33
|
+
## Limits that remain in 0.3.3
|
|
34
|
+
|
|
35
|
+
- **Arista cEOS, Nokia SR Linux and SONiC startup config:** Netdryrun doesn't yet check that every line of a startup config loaded. Config applied with `setup:` commands is checked (a command that fails stops the run). Mitigation: for each changed line, add a positive check that holds only if the line loaded, for example the route or attribute the line creates. Fix scheduled: 0.4 (verify the applied config on every platform).
|
|
36
|
+
- The other limits are listed in the [0.3.0 advisory](../0.3.0-false-pass/README.md#limits-that-remain-in-031).
|
|
37
|
+
|
|
38
|
+
## Found another way it can pass without looking?
|
|
39
|
+
|
|
40
|
+
[Open a corpus case](https://github.com/netdryrun/netdryrun/issues/new?template=corpus-case.yml): it becomes a permanent must-fail test, credited to you.
|
|
@@ -37,8 +37,16 @@ def outcome(expected: str, verdict: str) -> str:
|
|
|
37
37
|
return "false pass" if expected == "unsafe" else "false block"
|
|
38
38
|
|
|
39
39
|
|
|
40
|
-
def
|
|
40
|
+
def nondeterministic(corpus: Path = HERE) -> dict:
|
|
41
|
+
"""Cases marked in NONDETERMINISTIC.yml after the run: their verdict follows a race in the lab. Not scored."""
|
|
42
|
+
path = corpus / "NONDETERMINISTIC.yml"
|
|
43
|
+
return (yaml.safe_load(path.read_text(encoding="utf-8")) or {}) if path.exists() else {}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def stats(rows: list[dict], skip=()) -> dict:
|
|
47
|
+
"""Scores over the deterministic cases: the IDs in `skip` (nondeterministic) are left out."""
|
|
41
48
|
out = {}
|
|
49
|
+
rows = [r for r in rows if r["id"] not in skip]
|
|
42
50
|
unsafe = [r for r in rows if r["expected"] == "unsafe"]
|
|
43
51
|
for s in LABELS:
|
|
44
52
|
oc = [outcome(r["expected"], r["strategies"][s]["verdict"]) for r in rows]
|
|
@@ -53,14 +61,17 @@ def stats(rows: list[dict]) -> dict:
|
|
|
53
61
|
|
|
54
62
|
def results_markdown(data: dict, link=None, h: str = "##", corpus: Path = HERE, amendments: Path = AMENDMENTS) -> str:
|
|
55
63
|
rows, env = data["cases"], data["environment"]
|
|
56
|
-
|
|
64
|
+
nd = nondeterministic(corpus)
|
|
65
|
+
st = stats(rows, nd)
|
|
57
66
|
rel = corpus.relative_to(HERE.parent).as_posix()
|
|
58
67
|
lines = [
|
|
59
68
|
f"Generated {env['generated']} with netdryrun {env['netdryrun']}, containerlab {env['containerlab']}, "
|
|
60
69
|
f"FRR {env['frr_image']}, commit `{env.get('commit', '?')}`. Rerun everything with `python corpus/run.py{env.get('rerun_args', '')}`.", "",
|
|
61
70
|
"**Conflict of interest:** the Netdryrun maintainers wrote this corpus and the runner. Expected verdicts were "
|
|
62
71
|
f"committed before any run (see the git history of `{rel}/cases/`). Losses are kept and shown at the same size as the catches.", "",
|
|
63
|
-
f"{h} Summary ({st['total']} cases: {st['unsafe']} harmful, {st['safe']} safe
|
|
72
|
+
(f"{h} Summary ({st['total']} deterministic cases: {st['unsafe']} harmful, {st['safe']} safe; "
|
|
73
|
+
f"{len(nd)} nondeterministic, not scored)" if nd else
|
|
74
|
+
f"{h} Summary ({st['total']} cases: {st['unsafe']} harmful, {st['safe']} safe)"), "",
|
|
64
75
|
"| Strategy | Correct | Harmful changes caught | Approved a bad change | Blocked a good change | Errors |",
|
|
65
76
|
"|---|---|---|---|---|---|",
|
|
66
77
|
]
|
|
@@ -72,9 +83,23 @@ def results_markdown(data: dict, link=None, h: str = "##", corpus: Path = HERE,
|
|
|
72
83
|
for r in rows:
|
|
73
84
|
cells = [MARK[outcome(r["expected"], r["strategies"][s]["verdict"])] for s in LABELS]
|
|
74
85
|
flagged = "; ".join(r["strategies"]["netdryrun"]["failed_checks"]) or "nothing"
|
|
86
|
+
if r["id"] in nd:
|
|
87
|
+
cells = ["🎲 not scored"] * len(LABELS)
|
|
88
|
+
flagged = "nondeterministic in this lab: see below"
|
|
75
89
|
name = f"[{r['title']}]({link(r['id'])})" if link else r["title"]
|
|
76
90
|
lines.append(f"| `{r['id']}` {name} | {r['category']} | {r['expected']} | {' | '.join(cells)} | {flagged} |")
|
|
77
|
-
losses = [r for r in rows if
|
|
91
|
+
losses = [r for r in rows if r["id"] not in nd
|
|
92
|
+
and outcome(r["expected"], r["strategies"]["netdryrun"]["verdict"]) != "correct"]
|
|
93
|
+
if nd:
|
|
94
|
+
lines += ["", f"{h} Nondeterministic cases (not scored)", "",
|
|
95
|
+
"Same build, same machine, same case, different verdicts: the verdict follows a race in the lab, not "
|
|
96
|
+
"the change. These cases are left out of every score above and listed here with every Netdryrun "
|
|
97
|
+
f"verdict seen. The case files and expected verdicts are unchanged; the list is in `{rel}/NONDETERMINISTIC.yml`.", ""]
|
|
98
|
+
for cid, x in nd.items():
|
|
99
|
+
row = next((r for r in rows if r["id"] == cid), {"title": cid, "expected": "?"})
|
|
100
|
+
seen = "; ".join(f"{v['when']}, {v['run']}: **{v['verdict']}**" for v in x["netdryrun_verdicts"])
|
|
101
|
+
lines.append(f"- **`{cid}` {row['title']}** (expected {row['expected']}). {x['why'].strip()} "
|
|
102
|
+
f"Netdryrun verdicts: {seen}.")
|
|
78
103
|
lines += ["", f"{h} Netdryrun's losses", ""]
|
|
79
104
|
if not losses:
|
|
80
105
|
lines.append("None in this run.")
|
|
@@ -244,7 +269,7 @@ def render_v2() -> None:
|
|
|
244
269
|
body = body.replace("`corpus/v2/cases/`", f"[`corpus/v2/cases/`]({REPO}/tree/main/corpus/v2/cases)")
|
|
245
270
|
head = SITE_PAGE.read_text(encoding="utf-8").split(V2_MARKER)[0].rstrip("\n")
|
|
246
271
|
SITE_PAGE.write_text(head + "\n\n" + V2_MARKER + "\n\n" + intro + "\n\n" + body, encoding="utf-8", newline="\n")
|
|
247
|
-
st = stats(data["cases"])
|
|
272
|
+
st = stats(data["cases"], nondeterministic(V2))
|
|
248
273
|
print(f"wrote corpus/v2/RESULTS.md and the v2 section of {SITE_PAGE.name}")
|
|
249
274
|
print(json.dumps({s: st[s] for s in LABELS} | {"total": st["total"], "unsafe": st["unsafe"]}, indent=1))
|
|
250
275
|
|
|
@@ -37,12 +37,19 @@ from netdryrun.runner import run
|
|
|
37
37
|
HERE = Path(__file__).resolve().parent
|
|
38
38
|
CORPORA = {"v1": HERE, "v2": HERE / "v2"} # v1 lives at the top of corpus/ and is frozen
|
|
39
39
|
STRATEGIES = {"ping": "baseline-ping.yml", "sessions": "baseline-sessions.yml", "netdryrun": "intent.yml"}
|
|
40
|
+
NONDETERMINISTIC_RUNS = 3 # a case whose verdict follows a race in the lab runs this many times (NONDETERMINISTIC.yml)
|
|
40
41
|
|
|
41
42
|
|
|
42
43
|
class CaseError(ValueError):
|
|
43
44
|
pass
|
|
44
45
|
|
|
45
46
|
|
|
47
|
+
def nondeterministic(corpus: Path = HERE) -> dict:
|
|
48
|
+
"""Case IDs marked in NONDETERMINISTIC.yml after the run: excluded from the score, run several times."""
|
|
49
|
+
path = corpus / "NONDETERMINISTIC.yml"
|
|
50
|
+
return (yaml.safe_load(path.read_text(encoding="utf-8")) or {}) if path.exists() else {}
|
|
51
|
+
|
|
52
|
+
|
|
46
53
|
def load_cases(prefix: str | None, corpus: Path = HERE) -> list[dict]:
|
|
47
54
|
cases = []
|
|
48
55
|
for path in sorted((corpus / "cases").glob("*.yml")):
|
|
@@ -131,16 +138,19 @@ def write_results(rows: list[dict], out: Path, env: dict, corpus: Path = HERE) -
|
|
|
131
138
|
"| Strategy | Correct | False passes (bad change approved) | False blocks (good change rejected) | Errors |",
|
|
132
139
|
"|---|---|---|---|---|",
|
|
133
140
|
]
|
|
141
|
+
scored = [r for r in rows if not r.get("nondeterministic")]
|
|
134
142
|
for s in names:
|
|
135
|
-
oc = [outcome(r["expected"], r["strategies"][s]["verdict"]) for r in
|
|
136
|
-
lines.append(f"| {s} | {oc.count('correct')}/{len(
|
|
143
|
+
oc = [outcome(r["expected"], r["strategies"][s]["verdict"]) for r in scored]
|
|
144
|
+
lines.append(f"| {s} | {oc.count('correct')}/{len(scored)} | {oc.count('false pass')} | {oc.count('false block')} | {oc.count('error')} |")
|
|
137
145
|
lines += ["", "## Cases", "", "| Case | Category | Expected | Ping | Sessions | Netdryrun | What Netdryrun flagged |", "|---|---|---|---|---|---|---|"]
|
|
138
146
|
mark = {"correct": "✅", "false pass": "❌ false pass", "false block": "⚠️ false block", "error": "💥 error"}
|
|
139
147
|
for r in rows:
|
|
140
148
|
cells = [mark[outcome(r["expected"], r["strategies"][s]["verdict"])] for s in names]
|
|
149
|
+
if r.get("nondeterministic"):
|
|
150
|
+
cells = ["🎲 not scored"] * len(names)
|
|
141
151
|
flagged = "; ".join(r["strategies"]["netdryrun"]["failed_checks"]) or "nothing"
|
|
142
152
|
lines.append(f"| `{r['id']}` {r['title']} | {r['category']} | {r['expected']} | {' | '.join(cells)} | {flagged} |")
|
|
143
|
-
losses = [r for r in
|
|
153
|
+
losses = [r for r in scored if outcome(r["expected"], r["strategies"]["netdryrun"]["verdict"]) != "correct"]
|
|
144
154
|
lines += ["", "## Netdryrun losses", ""]
|
|
145
155
|
if not losses:
|
|
146
156
|
lines.append("None in this run.")
|
|
@@ -178,17 +188,25 @@ def main() -> int:
|
|
|
178
188
|
if args.corpus != "v1":
|
|
179
189
|
env["corpus"] = args.corpus
|
|
180
190
|
env["rerun_args"] = f" --corpus {args.corpus}"
|
|
191
|
+
nd = nondeterministic(corpus)
|
|
181
192
|
rows = []
|
|
182
193
|
for case in cases:
|
|
183
194
|
print(f"== {case['id']}: {case['title']} (expected {case['expected']})", flush=True)
|
|
184
195
|
try:
|
|
185
|
-
|
|
196
|
+
n = NONDETERMINISTIC_RUNS if case["id"] in nd else 1
|
|
197
|
+
# --keep keeps only the last run's lab: an earlier one left running would collide with the next deploy
|
|
198
|
+
runs = [evaluate(case, root, args.keep and i == n - 1, corpus) for i in range(n)]
|
|
199
|
+
row = runs[0]
|
|
200
|
+
if n > 1: # the first run stays the row's verdict; every run is kept
|
|
201
|
+
row["nondeterministic"] = True
|
|
202
|
+
row["runs"] = [{s: r["strategies"][s]["verdict"] for s in STRATEGIES} for r in runs]
|
|
186
203
|
except CaseError as e:
|
|
187
204
|
print(f" CASE ERROR: {e}", flush=True)
|
|
188
205
|
return 2
|
|
189
206
|
rows.append(row)
|
|
190
207
|
print(" " + " ".join(f"{s}={row['strategies'][s]['verdict']}" for s in STRATEGIES)
|
|
191
|
-
+ f" ({row['seconds']}s)",
|
|
208
|
+
+ f" ({row['seconds']}s)" + (f" nondeterministic, runs: {row['runs']}" if row.get("runs") else ""),
|
|
209
|
+
flush=True)
|
|
192
210
|
if not args.case:
|
|
193
211
|
write_results(rows, corpus / "results", env, corpus)
|
|
194
212
|
print(f"wrote {corpus.relative_to(HERE.parent).as_posix()}/RESULTS.md and results/results.json")
|