agentcheck-ai 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. agentcheck_ai-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml +49 -0
  2. agentcheck_ai-0.1.0/.github/workflows/agentcheck-example.yml +101 -0
  3. agentcheck_ai-0.1.0/.github/workflows/ci.yml +341 -0
  4. agentcheck_ai-0.1.0/.github/workflows/release.yml +133 -0
  5. agentcheck_ai-0.1.0/AGENTS.md +95 -0
  6. agentcheck_ai-0.1.0/CONTRIBUTING.md +129 -0
  7. agentcheck_ai-0.1.0/LICENSE +21 -0
  8. agentcheck_ai-0.1.0/MANIFEST.in +17 -0
  9. agentcheck_ai-0.1.0/PKG-INFO +415 -0
  10. agentcheck_ai-0.1.0/README.md +368 -0
  11. agentcheck_ai-0.1.0/SECURITY.md +63 -0
  12. agentcheck_ai-0.1.0/agentcheck/__init__.py +13 -0
  13. agentcheck_ai-0.1.0/agentcheck/__main__.py +4 -0
  14. agentcheck_ai-0.1.0/agentcheck/adapters/__init__.py +43 -0
  15. agentcheck_ai-0.1.0/agentcheck/adapters/base.py +365 -0
  16. agentcheck_ai-0.1.0/agentcheck/adapters/controlled_model.py +194 -0
  17. agentcheck_ai-0.1.0/agentcheck/adapters/custom.py +1530 -0
  18. agentcheck_ai-0.1.0/agentcheck/adapters/openai_agents.py +2878 -0
  19. agentcheck_ai-0.1.0/agentcheck/adapters/openai_handoff_effects.py +454 -0
  20. agentcheck_ai-0.1.0/agentcheck/adapters/pydantic_ai.py +1529 -0
  21. agentcheck_ai-0.1.0/agentcheck/adapters/pydantic_ai_controlled.py +140 -0
  22. agentcheck_ai-0.1.0/agentcheck/analyze/__init__.py +5 -0
  23. agentcheck_ai-0.1.0/agentcheck/analyze/failures.py +227 -0
  24. agentcheck_ai-0.1.0/agentcheck/application.py +1000 -0
  25. agentcheck_ai-0.1.0/agentcheck/artifacts.py +147 -0
  26. agentcheck_ai-0.1.0/agentcheck/baseline/__init__.py +27 -0
  27. agentcheck_ai-0.1.0/agentcheck/baseline/build.py +124 -0
  28. agentcheck_ai-0.1.0/agentcheck/baseline/compare.py +301 -0
  29. agentcheck_ai-0.1.0/agentcheck/baseline/contract.py +218 -0
  30. agentcheck_ai-0.1.0/agentcheck/baseline/load.py +107 -0
  31. agentcheck_ai-0.1.0/agentcheck/baseline/service.py +154 -0
  32. agentcheck_ai-0.1.0/agentcheck/cli.py +1497 -0
  33. agentcheck_ai-0.1.0/agentcheck/config.py +377 -0
  34. agentcheck_ai-0.1.0/agentcheck/custom.py +100 -0
  35. agentcheck_ai-0.1.0/agentcheck/domain/__init__.py +174 -0
  36. agentcheck_ai-0.1.0/agentcheck/domain/agent_spec.py +225 -0
  37. agentcheck_ai-0.1.0/agentcheck/domain/base.py +88 -0
  38. agentcheck_ai-0.1.0/agentcheck/domain/evaluation.py +140 -0
  39. agentcheck_ai-0.1.0/agentcheck/domain/finding.py +95 -0
  40. agentcheck_ai-0.1.0/agentcheck/domain/run.py +261 -0
  41. agentcheck_ai-0.1.0/agentcheck/domain/scenario.py +342 -0
  42. agentcheck_ai-0.1.0/agentcheck/domain/verdict.py +14 -0
  43. agentcheck_ai-0.1.0/agentcheck/errors.py +29 -0
  44. agentcheck_ai-0.1.0/agentcheck/evaluate/__init__.py +5 -0
  45. agentcheck_ai-0.1.0/agentcheck/evaluate/engine.py +1141 -0
  46. agentcheck_ai-0.1.0/agentcheck/fixtures/__init__.py +29 -0
  47. agentcheck_ai-0.1.0/agentcheck/fixtures/loader.py +255 -0
  48. agentcheck_ai-0.1.0/agentcheck/fixtures/pack.py +87 -0
  49. agentcheck_ai-0.1.0/agentcheck/generate/__init__.py +131 -0
  50. agentcheck_ai-0.1.0/agentcheck/generate/boundaries.py +1579 -0
  51. agentcheck_ai-0.1.0/agentcheck/generate/lint.py +579 -0
  52. agentcheck_ai-0.1.0/agentcheck/generate/mutations.py +787 -0
  53. agentcheck_ai-0.1.0/agentcheck/generate/realization.py +394 -0
  54. agentcheck_ai-0.1.0/agentcheck/generate/selection.py +495 -0
  55. agentcheck_ai-0.1.0/agentcheck/generate/suite.py +860 -0
  56. agentcheck_ai-0.1.0/agentcheck/generate/templates.py +455 -0
  57. agentcheck_ai-0.1.0/agentcheck/initialize.py +119 -0
  58. agentcheck_ai-0.1.0/agentcheck/inspect/__init__.py +29 -0
  59. agentcheck_ai-0.1.0/agentcheck/inspect/capabilities.py +879 -0
  60. agentcheck_ai-0.1.0/agentcheck/inspect/extractor.py +596 -0
  61. agentcheck_ai-0.1.0/agentcheck/policies/__init__.py +48 -0
  62. agentcheck_ai-0.1.0/agentcheck/policies/builtins.py +114 -0
  63. agentcheck_ai-0.1.0/agentcheck/policies/derived.py +145 -0
  64. agentcheck_ai-0.1.0/agentcheck/policies/loader.py +375 -0
  65. agentcheck_ai-0.1.0/agentcheck/policies/pack.py +63 -0
  66. agentcheck_ai-0.1.0/agentcheck/privacy.py +129 -0
  67. agentcheck_ai-0.1.0/agentcheck/redaction.py +239 -0
  68. agentcheck_ai-0.1.0/agentcheck/replay/__init__.py +58 -0
  69. agentcheck_ai-0.1.0/agentcheck/replay/bind.py +194 -0
  70. agentcheck_ai-0.1.0/agentcheck/replay/fileset.py +440 -0
  71. agentcheck_ai-0.1.0/agentcheck/replay/load.py +134 -0
  72. agentcheck_ai-0.1.0/agentcheck/replay/manifest.py +321 -0
  73. agentcheck_ai-0.1.0/agentcheck/report/__init__.py +12 -0
  74. agentcheck_ai-0.1.0/agentcheck/report/load.py +507 -0
  75. agentcheck_ai-0.1.0/agentcheck/report/render.py +390 -0
  76. agentcheck_ai-0.1.0/agentcheck/review/__init__.py +25 -0
  77. agentcheck_ai-0.1.0/agentcheck/review/contract.py +129 -0
  78. agentcheck_ai-0.1.0/agentcheck/review/service.py +134 -0
  79. agentcheck_ai-0.1.0/agentcheck/review/store.py +213 -0
  80. agentcheck_ai-0.1.0/agentcheck/runner/__init__.py +47 -0
  81. agentcheck_ai-0.1.0/agentcheck/runner/budgets.py +187 -0
  82. agentcheck_ai-0.1.0/agentcheck/runner/network_guard.py +284 -0
  83. agentcheck_ai-0.1.0/agentcheck/runner/orchestrator.py +687 -0
  84. agentcheck_ai-0.1.0/agentcheck/runner/tool_gateway.py +911 -0
  85. agentcheck_ai-0.1.0/agentcheck/runner/worker.py +389 -0
  86. agentcheck_ai-0.1.0/agentcheck/runner/world.py +310 -0
  87. agentcheck_ai-0.1.0/agentcheck/schema_safety.py +57 -0
  88. agentcheck_ai-0.1.0/agentcheck/shrink/__init__.py +48 -0
  89. agentcheck_ai-0.1.0/agentcheck/shrink/candidates.py +135 -0
  90. agentcheck_ai-0.1.0/agentcheck/shrink/complexity.py +97 -0
  91. agentcheck_ai-0.1.0/agentcheck/shrink/result.py +179 -0
  92. agentcheck_ai-0.1.0/agentcheck/shrink/search.py +292 -0
  93. agentcheck_ai-0.1.0/agentcheck/shrink/signature.py +137 -0
  94. agentcheck_ai-0.1.0/agentcheck/store/__init__.py +37 -0
  95. agentcheck_ai-0.1.0/agentcheck/store/schema.py +78 -0
  96. agentcheck_ai-0.1.0/agentcheck/store/sqlite.py +565 -0
  97. agentcheck_ai-0.1.0/agentcheck_ai.egg-info/PKG-INFO +415 -0
  98. agentcheck_ai-0.1.0/agentcheck_ai.egg-info/SOURCES.txt +187 -0
  99. agentcheck_ai-0.1.0/agentcheck_ai.egg-info/dependency_links.txt +1 -0
  100. agentcheck_ai-0.1.0/agentcheck_ai.egg-info/entry_points.txt +2 -0
  101. agentcheck_ai-0.1.0/agentcheck_ai.egg-info/requires.txt +26 -0
  102. agentcheck_ai-0.1.0/agentcheck_ai.egg-info/top_level.txt +1 -0
  103. agentcheck_ai-0.1.0/docs/ci-trust-model.md +199 -0
  104. agentcheck_ai-0.1.0/docs/custom-agents.md +221 -0
  105. agentcheck_ai-0.1.0/docs/development-history.md +116 -0
  106. agentcheck_ai-0.1.0/docs/pydantic-ai.md +196 -0
  107. agentcheck_ai-0.1.0/docs/validation-evidence.md +130 -0
  108. agentcheck_ai-0.1.0/examples/evaluation/account_agent/README.md +37 -0
  109. agentcheck_ai-0.1.0/examples/evaluation/account_agent/agent.py +403 -0
  110. agentcheck_ai-0.1.0/examples/evaluation/account_agent/agentcheck.json +14 -0
  111. agentcheck_ai-0.1.0/examples/evaluation/custom_agent/README.md +37 -0
  112. agentcheck_ai-0.1.0/examples/evaluation/custom_agent/agent.py +102 -0
  113. agentcheck_ai-0.1.0/examples/evaluation/custom_agent/agentcheck.json +15 -0
  114. agentcheck_ai-0.1.0/examples/evaluation/handoff_router/README.md +37 -0
  115. agentcheck_ai-0.1.0/examples/evaluation/handoff_router/__init__.py +1 -0
  116. agentcheck_ai-0.1.0/examples/evaluation/handoff_router/agent.py +230 -0
  117. agentcheck_ai-0.1.0/examples/evaluation/handoff_router/agentcheck.json +14 -0
  118. agentcheck_ai-0.1.0/examples/evaluation/pydantic_agent/README.md +71 -0
  119. agentcheck_ai-0.1.0/examples/evaluation/pydantic_agent/agent.py +60 -0
  120. agentcheck_ai-0.1.0/examples/evaluation/pydantic_agent/agentcheck-fixtures.json +12 -0
  121. agentcheck_ai-0.1.0/examples/evaluation/pydantic_agent/agentcheck.json +14 -0
  122. agentcheck_ai-0.1.0/pyproject.toml +106 -0
  123. agentcheck_ai-0.1.0/scripts/check_distribution.py +85 -0
  124. agentcheck_ai-0.1.0/scripts/check_interpreter_integrity.py +109 -0
  125. agentcheck_ai-0.1.0/scripts/check_no_secrets.py +72 -0
  126. agentcheck_ai-0.1.0/scripts/check_workflow_safety.py +333 -0
  127. agentcheck_ai-0.1.0/setup.cfg +4 -0
  128. agentcheck_ai-0.1.0/tests/agentcheck/conftest.py +71 -0
  129. agentcheck_ai-0.1.0/tests/agentcheck/test_allowlist_dns.py +203 -0
  130. agentcheck_ai-0.1.0/tests/agentcheck/test_analysis.py +45 -0
  131. agentcheck_ai-0.1.0/tests/agentcheck/test_application.py +76 -0
  132. agentcheck_ai-0.1.0/tests/agentcheck/test_artifacts.py +87 -0
  133. agentcheck_ai-0.1.0/tests/agentcheck/test_baseline.py +1105 -0
  134. agentcheck_ai-0.1.0/tests/agentcheck/test_boundaries.py +756 -0
  135. agentcheck_ai-0.1.0/tests/agentcheck/test_capabilities.py +610 -0
  136. agentcheck_ai-0.1.0/tests/agentcheck/test_cli_diagnostics.py +200 -0
  137. agentcheck_ai-0.1.0/tests/agentcheck/test_cli_e2e.py +461 -0
  138. agentcheck_ai-0.1.0/tests/agentcheck/test_compat_manifest.py +90 -0
  139. agentcheck_ai-0.1.0/tests/agentcheck/test_config.py +166 -0
  140. agentcheck_ai-0.1.0/tests/agentcheck/test_confirmation_variant_cases.py +324 -0
  141. agentcheck_ai-0.1.0/tests/agentcheck/test_controlled_model.py +154 -0
  142. agentcheck_ai-0.1.0/tests/agentcheck/test_custom_agent_adapter.py +1739 -0
  143. agentcheck_ai-0.1.0/tests/agentcheck/test_custom_agent_contract.py +245 -0
  144. agentcheck_ai-0.1.0/tests/agentcheck/test_custom_agent_ux.py +807 -0
  145. agentcheck_ai-0.1.0/tests/agentcheck/test_derived_policies.py +153 -0
  146. agentcheck_ai-0.1.0/tests/agentcheck/test_domain.py +617 -0
  147. agentcheck_ai-0.1.0/tests/agentcheck/test_evaluation.py +371 -0
  148. agentcheck_ai-0.1.0/tests/agentcheck/test_example_account_agent.py +166 -0
  149. agentcheck_ai-0.1.0/tests/agentcheck/test_example_custom_agent.py +234 -0
  150. agentcheck_ai-0.1.0/tests/agentcheck/test_example_handoff_agent.py +415 -0
  151. agentcheck_ai-0.1.0/tests/agentcheck/test_example_pydantic_agent.py +148 -0
  152. agentcheck_ai-0.1.0/tests/agentcheck/test_generate.py +794 -0
  153. agentcheck_ai-0.1.0/tests/agentcheck/test_generated_suite_budget.py +112 -0
  154. agentcheck_ai-0.1.0/tests/agentcheck/test_handoff_callbacks.py +446 -0
  155. agentcheck_ai-0.1.0/tests/agentcheck/test_handoffs.py +848 -0
  156. agentcheck_ai-0.1.0/tests/agentcheck/test_init.py +402 -0
  157. agentcheck_ai-0.1.0/tests/agentcheck/test_interactive_scenarios.py +1476 -0
  158. agentcheck_ai-0.1.0/tests/agentcheck/test_invocation_identity.py +76 -0
  159. agentcheck_ai-0.1.0/tests/agentcheck/test_lint.py +249 -0
  160. agentcheck_ai-0.1.0/tests/agentcheck/test_mutations.py +572 -0
  161. agentcheck_ai-0.1.0/tests/agentcheck/test_network_containment.py +389 -0
  162. agentcheck_ai-0.1.0/tests/agentcheck/test_openai_adapter.py +823 -0
  163. agentcheck_ai-0.1.0/tests/agentcheck/test_outcome_variant_cases.py +220 -0
  164. agentcheck_ai-0.1.0/tests/agentcheck/test_output_schema_and_tool_identity.py +231 -0
  165. agentcheck_ai-0.1.0/tests/agentcheck/test_package_boundary.py +85 -0
  166. agentcheck_ai-0.1.0/tests/agentcheck/test_package_loader.py +304 -0
  167. agentcheck_ai-0.1.0/tests/agentcheck/test_policy_packs.py +456 -0
  168. agentcheck_ai-0.1.0/tests/agentcheck/test_positive_path_cases.py +351 -0
  169. agentcheck_ai-0.1.0/tests/agentcheck/test_preflight_diagnostics.py +472 -0
  170. agentcheck_ai-0.1.0/tests/agentcheck/test_prerequisite_fixtures.py +499 -0
  171. agentcheck_ai-0.1.0/tests/agentcheck/test_pydantic_ai_adapter.py +842 -0
  172. agentcheck_ai-0.1.0/tests/agentcheck/test_realization.py +346 -0
  173. agentcheck_ai-0.1.0/tests/agentcheck/test_replay.py +611 -0
  174. agentcheck_ai-0.1.0/tests/agentcheck/test_report.py +656 -0
  175. agentcheck_ai-0.1.0/tests/agentcheck/test_representative_inputs.py +426 -0
  176. agentcheck_ai-0.1.0/tests/agentcheck/test_review.py +540 -0
  177. agentcheck_ai-0.1.0/tests/agentcheck/test_schema_safety.py +50 -0
  178. agentcheck_ai-0.1.0/tests/agentcheck/test_scoped_provider_access.py +174 -0
  179. agentcheck_ai-0.1.0/tests/agentcheck/test_selection.py +366 -0
  180. agentcheck_ai-0.1.0/tests/agentcheck/test_shrink.py +1164 -0
  181. agentcheck_ai-0.1.0/tests/agentcheck/test_source_fileset.py +586 -0
  182. agentcheck_ai-0.1.0/tests/agentcheck/test_sqlite_store.py +375 -0
  183. agentcheck_ai-0.1.0/tests/agentcheck/test_suite_compatibility.py +278 -0
  184. agentcheck_ai-0.1.0/tests/agentcheck/test_target_loading.py +420 -0
  185. agentcheck_ai-0.1.0/tests/agentcheck/test_templates.py +19 -0
  186. agentcheck_ai-0.1.0/tests/agentcheck/test_worker_process.py +796 -0
  187. agentcheck_ai-0.1.0/tests/agentcheck/test_workflow_safety.py +102 -0
  188. agentcheck_ai-0.1.0/tests/agentcheck/test_world_gateway.py +572 -0
  189. agentcheck_ai-0.1.0/tests/compat_manifest.py +145 -0
@@ -0,0 +1,49 @@
1
+ name: Bug report
2
+ description: Something AgentCheck did that it should not have, or did not do that it should
3
+ labels: ["bug"]
4
+ body:
5
+ - type: markdown
6
+ attributes:
7
+ value: |
8
+ Please do not paste API keys, provider payloads, or customer data.
9
+ Artifacts under `.agentcheck/` can contain prompts, model output and
10
+ absolute paths — read before pasting.
11
+
12
+ For a security vulnerability, do not open an issue. See SECURITY.md.
13
+ - type: textarea
14
+ id: what
15
+ attributes:
16
+ label: What happened
17
+ description: What you expected, and what you got instead.
18
+ validations:
19
+ required: true
20
+ - type: dropdown
21
+ id: framework
22
+ attributes:
23
+ label: Framework adapter
24
+ options: ["OpenAI Agents SDK", "PydanticAI", "Neither / not applicable"]
25
+ validations:
26
+ required: true
27
+ - type: input
28
+ id: versions
29
+ attributes:
30
+ label: Versions
31
+ description: AgentCheck version, framework SDK version, Python version.
32
+ placeholder: "agentcheck 0.1.0, openai-agents 0.20.0, Python 3.12"
33
+ validations:
34
+ required: true
35
+ - type: dropdown
36
+ id: verdict
37
+ attributes:
38
+ label: Verdict involved
39
+ description: INFRA_ERROR usually means the harness failed, not your agent.
40
+ options: ["PASS", "FAIL", "INCONCLUSIVE", "INFRA_ERROR", "No verdict / command failed"]
41
+ validations:
42
+ required: true
43
+ - type: textarea
44
+ id: repro
45
+ attributes:
46
+ label: Reproduction
47
+ description: The command you ran and, if possible, a minimal target.
48
+ validations:
49
+ required: true
@@ -0,0 +1,101 @@
1
+ # Example AgentCheck regression gate.
2
+ #
3
+ # Copy this workflow into a repository that has an AgentCheck target, then:
4
+ # 1. Set AGENTCHECK_TARGET to that directory (often ".").
5
+ # 2. Commit a trusted baseline produced by an explicit command, not by a
6
+ # failing CI run:
7
+ # agentcheck test "$AGENTCHECK_TARGET" --no-store
8
+ # agentcheck baseline create "$AGENTCHECK_TARGET" --latest --out agentcheck-baseline.json
9
+ # 3. Enable pull_request/push triggers if you want this on every change.
10
+ #
11
+ # This workflow does not call a hosted AgentCheck service, does not use the
12
+ # GitHub API, and does not supply provider credentials. It fails only on new
13
+ # or changed authoritative FAIL identities, not on historical failures and
14
+ # not on INCONCLUSIVE.
15
+ #
16
+ # This repository is not itself an AgentCheck target at its root. The default
17
+ # below points at the bundled deterministic example. There is no committed
18
+ # example baseline here; workflow_dispatch will exit 2 until you create one
19
+ # locally or in a consuming project.
20
+
21
+ # Runner note: this workflow is deliberately GitHub-hosted, unlike ci.yml.
22
+ # It is a template meant to be copied into a project that uses AgentCheck, so
23
+ # pointing it at a self-hosted label would hand every reader a workflow that
24
+ # runs on exactly one machine. It is workflow_dispatch only, so it costs
25
+ # nothing until someone triggers it. See docs/ci-trust-model.md.
26
+
27
+ name: AgentCheck
28
+
29
+ on:
30
+ workflow_dispatch:
31
+
32
+ permissions:
33
+ contents: read
34
+
35
+ concurrency:
36
+ group: ${{ github.workflow }}-${{ github.ref }}
37
+ cancel-in-progress: true
38
+
39
+ env:
40
+ PIP_DISABLE_PIP_VERSION_CHECK: "1"
41
+ PYTHONDONTWRITEBYTECODE: "1"
42
+ AGENTCHECK_TARGET: examples/evaluation/account_agent
43
+ AGENTCHECK_BASELINE: agentcheck-baseline.json
44
+
45
+ jobs:
46
+ regression-gate:
47
+ name: Compare against a trusted baseline
48
+ runs-on: ubuntu-latest
49
+ timeout-minutes: 20
50
+
51
+ steps:
52
+ - name: Checkout
53
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
54
+ with:
55
+ persist-credentials: false
56
+
57
+ - name: Set up Python
58
+ uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
59
+ with:
60
+ python-version: "3.12"
61
+ cache: pip
62
+ cache-dependency-path: pyproject.toml
63
+
64
+ - name: Install AgentCheck
65
+ run: |
66
+ python -m pip install --upgrade "pip==26.1.2" "setuptools==83.0.0"
67
+ python -m pip install ".[openai-agents]"
68
+
69
+ - name: Run deterministic AgentCheck tests
70
+ run: |
71
+ set +e
72
+ agentcheck test "$AGENTCHECK_TARGET" --no-store --run-id "ci-${{ github.run_id }}"
73
+ status=$?
74
+ set -e
75
+ # Historical FAILs (1) and INCONCLUSIVE (3) are not release blockers.
76
+ # The baseline check decides whether any FAIL is new. Infrastructure (2)
77
+ # still fails the job.
78
+ if [ "$status" -eq 2 ] || [ "$status" -eq 130 ]; then
79
+ exit "$status"
80
+ fi
81
+ if [ "$status" -gt 3 ]; then
82
+ exit "$status"
83
+ fi
84
+
85
+ - name: Require a committed baseline
86
+ run: |
87
+ baseline="$AGENTCHECK_TARGET/$AGENTCHECK_BASELINE"
88
+ if [ ! -f "$baseline" ]; then
89
+ echo "No trusted baseline at $baseline"
90
+ echo "Create one explicitly with:"
91
+ echo " agentcheck baseline create \"$AGENTCHECK_TARGET\" --latest --out $AGENTCHECK_BASELINE"
92
+ echo "A failing test run is never an implicit baseline."
93
+ exit 2
94
+ fi
95
+
96
+ - name: Fail on new authoritative regressions
97
+ run: |
98
+ agentcheck baseline check "$AGENTCHECK_TARGET" \
99
+ --baseline "$AGENTCHECK_BASELINE" \
100
+ --run-id "ci-${{ github.run_id }}" \
101
+ --json
@@ -0,0 +1,341 @@
1
+ # Normal AgentCheck CI.
2
+ #
3
+ # ============================================================================
4
+ # READ BEFORE MAKING THIS REPOSITORY PUBLIC
5
+ # ============================================================================
6
+ # Every job below runs on `agentcheck-local`, a self-hosted runner that is a
7
+ # maintainer's own workstation. That is acceptable *only* while this repository
8
+ # is private and every branch in it is written by the owner.
9
+ #
10
+ # The moment this repository becomes public, anyone can open a pull request
11
+ # containing arbitrary code. Running that on this runner is remote code
12
+ # execution on a personal machine with its filesystem, its network, and its
13
+ # credentials.
14
+ #
15
+ # The `if:` guard on each job is what stands in the way: it admits pushes and
16
+ # same-repository pull requests only, so a fork's pull request is skipped
17
+ # rather than executed. Going public therefore does not silently expose the
18
+ # runner -- but it does leave external contributors with no CI at all, which is
19
+ # not a workable open-source project.
20
+ #
21
+ # So before publication, the tests and checks jobs must move back to
22
+ # GitHub-hosted or isolated ephemeral runners. `docs/ci-trust-model.md` records
23
+ # the full argument and the exact change. scripts/check_workflow_safety.py
24
+ # proves the guard holds in the meantime, and it runs as part of this workflow.
25
+ # ============================================================================
26
+ #
27
+ # No job here needs a secret, and none makes a provider call.
28
+
29
+ name: CI
30
+
31
+ on:
32
+ push:
33
+ branches: [main]
34
+ pull_request:
35
+ branches: [main]
36
+
37
+ permissions:
38
+ contents: read
39
+
40
+ concurrency:
41
+ group: ${{ github.workflow }}-${{ github.ref }}
42
+ cancel-in-progress: true
43
+
44
+ env:
45
+ PIP_DISABLE_PIP_VERSION_CHECK: "1"
46
+ PYTHONDONTWRITEBYTECODE: "1"
47
+ # The interpreter that runs the whole suite. Every other supported version
48
+ # runs the cross-version suite from tests/compat_manifest.py.
49
+ PRIMARY_PYTHON: "3.12"
50
+
51
+ jobs:
52
+ tests:
53
+ name: Tests (Python ${{ matrix.python-version }})
54
+ # Carried by every job independently: a job-level `if` is not inherited, so
55
+ # a single unguarded job would be enough to run a fork's code on this
56
+ # machine.
57
+ #
58
+ # An allowlist, not `event_name != 'pull_request'`. A deny-list admits any
59
+ # trigger added to `on:` later -- a workflow_dispatch from a fork's branch,
60
+ # a future pull_request_target -- without anyone noticing. Naming the two
61
+ # trusted contexts means a new trigger has to be added here deliberately
62
+ # before it can reach this runner.
63
+ if: >-
64
+ github.event_name == 'push'
65
+ || (github.event_name == 'pull_request'
66
+ && github.event.pull_request.head.repo.full_name == github.repository)
67
+ runs-on: [self-hosted, Linux, X64]
68
+ # The suite measured 20:00 at -n 2 on this machine. 40 leaves real margin
69
+ # for a workstation's own background load without letting a hung job hold
70
+ # the single runner all afternoon.
71
+ timeout-minutes: 40
72
+ strategy:
73
+ fail-fast: false
74
+ matrix:
75
+ # One runner means these queue serially, so every entry is wall-clock a
76
+ # pull request waits. On a pull request all three supported versions run;
77
+ # after merge only the primary does, because the merge-race this second
78
+ # run exists to catch would show up on any interpreter. See the Test step.
79
+ python-version: ${{ github.event_name == 'push' && fromJSON('["3.12"]') || fromJSON('["3.10", "3.11", "3.12"]') }}
80
+
81
+ env:
82
+ # A self-hosted runner inherits the service environment of a machine that
83
+ # holds real credentials, so every variable this package reads is pinned
84
+ # to an inert value. AgentCheck's environment surface is small: a grep for
85
+ # os.environ/os.getenv across the package and tests finds only the name
86
+ # below, plus OPENAI_API_KEY.
87
+ #
88
+ # Deliberately absent: OPENAI_API_KEY. agentcheck/inspect/extractor.py
89
+ # branches on whether that name is *present*, and `FOO: ""` makes it
90
+ # present. Pinning it here would change what those tests observe. The
91
+ # runner environment is verified credential-free by a step below instead.
92
+ AGENTCHECK_ALLOWED_TEST: ""
93
+
94
+ steps:
95
+ - name: Checkout
96
+ # The default clean: true runs git clean -ffdx, which clears the
97
+ # editable-install egg-info, __pycache__, and anything else a previous
98
+ # job left in this persistent workspace.
99
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
100
+ with:
101
+ persist-credentials: false
102
+
103
+ - name: Install uv
104
+ # Not actions/setup-python. The interpreters it downloads record the
105
+ # absolute path of their own libpython as
106
+ # /opt/hostedtoolcache/Python/<version>/x64/lib, which exists only on a
107
+ # GitHub-hosted VM. This runner keeps its tool cache elsewhere, so the
108
+ # loader substitutes the host's libpython and the job ends up running a
109
+ # standard library and a core from different releases. uv's
110
+ # interpreters resolve libpython relative to the executable, so they
111
+ # stay correct wherever they are unpacked.
112
+ uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
113
+ with:
114
+ version: "0.9.18"
115
+ # This runner keeps uv's cache on local disk between jobs, so a
116
+ # GitHub cache round-trip would be pure overhead.
117
+ enable-cache: false
118
+
119
+ - name: Verify no provider credential reached this job
120
+ # The suite asserts on how AgentCheck behaves when a provider
121
+ # credential is absent, and extractor.py takes a different branch when
122
+ # OPENAI_API_KEY is present. A key exported into the runner service
123
+ # would not fail loudly -- it would quietly change what several tests
124
+ # observe. Names only; no value is read or printed.
125
+ run: |
126
+ for name in OPENAI_API_KEY ANTHROPIC_API_KEY GOOGLE_API_KEY; do
127
+ if printenv "$name" >/dev/null 2>&1; then
128
+ echo "$name is present in this job's environment; the suite expects it absent" >&2
129
+ exit 1
130
+ fi
131
+ done
132
+ echo "PASS: no provider credential is visible to this job"
133
+
134
+ - name: Create an isolated virtual environment
135
+ # only-managed rather than whatever Python the host publishes: the job
136
+ # should test the version the matrix names, not a host build that
137
+ # merely shares its minor number. RUNNER_TEMP is recreated per job.
138
+ run: |
139
+ uv venv --python "${{ matrix.python-version }}" \
140
+ --python-preference only-managed "$RUNNER_TEMP/venv"
141
+ echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH"
142
+
143
+ - name: Verify interpreter integrity
144
+ # Seconds, before anything expensive. A mismatched interpreter does not
145
+ # fail honestly: it segfaults the worker subprocesses and arrives as a
146
+ # wall of behavioural failures with no diagnostics -- exactly the
147
+ # infrastructure-as-product-failure confusion AgentCheck exists to
148
+ # prevent.
149
+ run: python scripts/check_interpreter_integrity.py "${{ matrix.python-version }}"
150
+
151
+ - name: Install
152
+ run: uv pip install --python "$RUNNER_TEMP/venv/bin/python" -e ".[dev]"
153
+
154
+ - name: Compile Python sources
155
+ run: python -m compileall -q agentcheck scripts examples tests
156
+
157
+ - name: Test
158
+ # -n 2, measured rather than guessed, and not a knob to turn up.
159
+ #
160
+ # Every scenario runs in its own child process under a 10s wall-clock
161
+ # budget. That budget is a product default and the suite's ability to
162
+ # notice a starved scenario depends on it, so it is never raised to buy
163
+ # CI headroom. pytest-xdist workers each spawn their own scenario
164
+ # children, so the worker count decides whether those children get
165
+ # their budget. On this 14-core machine -n 2 has a green history; -n 3
166
+ # and -n 4 were benchmarked and starved scenarios, turning real
167
+ # verdicts into worker_timeout.
168
+ #
169
+ # The primary interpreter runs everything. The others run the
170
+ # cross-version suite, whose contents and rationale live in
171
+ # tests/compat_manifest.py and are enforced by
172
+ # tests/agentcheck/test_compat_manifest.py -- the list is read from the
173
+ # manifest here rather than duplicated, so the two cannot drift.
174
+ #
175
+ # The package has no sys.version_info branching, so what differs between
176
+ # interpreters is where it meets one: recursive TypeAliasType models,
177
+ # subprocess workers, import machinery, SDK internals and the CLI entry
178
+ # point. Those all run on every version. Pure logic over already-built
179
+ # objects runs once.
180
+ run: |
181
+ if [ "${{ matrix.python-version }}" = "$PRIMARY_PYTHON" ]; then
182
+ python -m pytest -p no:cacheprovider tests -q -n 2
183
+ else
184
+ mapfile -t compat < <(python tests/compat_manifest.py)
185
+ echo "cross-version suite: ${#compat[@]} files"
186
+ python -m pytest -p no:cacheprovider "${compat[@]}" -q -n 2
187
+ fi
188
+
189
+ checks:
190
+ # One job, not five. There is a single self-hosted runner, so every extra
191
+ # job is another slot in a serial queue: as separate jobs these checks cost
192
+ # a few minutes of work but five job setups and five checkouts, all waiting
193
+ # behind each other. Merged into one job they stay individually named
194
+ # steps, so each result is still visible, and `!cancelled()` keeps a
195
+ # failure in one check from hiding the outcome of the rest.
196
+ name: Quality, packaging, extras, and runner trust
197
+ if: >-
198
+ github.event_name == 'push'
199
+ || (github.event_name == 'pull_request'
200
+ && github.event.pull_request.head.repo.full_name == github.repository)
201
+ runs-on: [self-hosted, Linux, X64]
202
+ timeout-minutes: 30
203
+
204
+ env:
205
+ AGENTCHECK_ALLOWED_TEST: ""
206
+
207
+ steps:
208
+ - name: Checkout
209
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
210
+ with:
211
+ persist-credentials: false
212
+
213
+ - name: Install uv
214
+ uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
215
+ with:
216
+ version: "0.9.18"
217
+ enable-cache: false
218
+
219
+ - name: Verify no provider credential reached this job
220
+ run: |
221
+ for name in OPENAI_API_KEY ANTHROPIC_API_KEY GOOGLE_API_KEY; do
222
+ if printenv "$name" >/dev/null 2>&1; then
223
+ echo "$name is present in this job's environment" >&2
224
+ exit 1
225
+ fi
226
+ done
227
+ echo "PASS: no provider credential is visible to this job"
228
+
229
+ - name: Create isolated virtual environments
230
+ run: |
231
+ uv venv --python 3.12 --python-preference only-managed "$RUNNER_TEMP/venv"
232
+ uv venv --python 3.12 --python-preference only-managed "$RUNNER_TEMP/base-venv"
233
+ uv venv --python 3.12 --python-preference only-managed "$RUNNER_TEMP/openai-venv"
234
+ uv venv --python 3.12 --python-preference only-managed "$RUNNER_TEMP/pydantic-venv"
235
+ uv venv --python 3.12 --python-preference only-managed "$RUNNER_TEMP/wheel-venv"
236
+ echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH"
237
+
238
+ - name: Verify interpreter integrity
239
+ run: python scripts/check_interpreter_integrity.py "3.12"
240
+
241
+ - name: Install
242
+ run: uv pip install --python "$RUNNER_TEMP/venv/bin/python" -e ".[dev]"
243
+
244
+ - name: Verify no untrusted context can reach this runner
245
+ # Every job here runs on a personal machine, and a job-level `if` is not
246
+ # inherited, so the guards are the only thing between a fork's branch
247
+ # and this host. They are checked by evaluating them against synthetic
248
+ # push, same-repo, fork, pull_request_target, workflow_run,
249
+ # workflow_dispatch, and issue_comment contexts rather than by matching
250
+ # their text, because the mistakes worth catching are logical ones.
251
+ if: ${{ !cancelled() }}
252
+ run: python scripts/check_workflow_safety.py
253
+
254
+ - name: Ruff
255
+ if: ${{ !cancelled() }}
256
+ run: python -m ruff check agentcheck tests scripts
257
+
258
+ - name: Mypy
259
+ if: ${{ !cancelled() }}
260
+ run: python -m mypy agentcheck
261
+
262
+ - name: Secret scan
263
+ if: ${{ !cancelled() }}
264
+ run: python scripts/check_no_secrets.py
265
+
266
+ - name: Base install has no framework
267
+ # A base install must import and run with neither framework present.
268
+ if: ${{ !cancelled() }}
269
+ run: |
270
+ uv pip install --python "$RUNNER_TEMP/base-venv/bin/python" .
271
+ "$RUNNER_TEMP/base-venv/bin/python" - <<'PY'
272
+ import importlib.util, subprocess
273
+ for framework in ("agents", "pydantic_ai"):
274
+ assert importlib.util.find_spec(framework) is None, (
275
+ f"{framework} must not be a base dependency"
276
+ )
277
+ import agentcheck
278
+ print("base import ok:", agentcheck.__version__)
279
+ PY
280
+ "$RUNNER_TEMP/base-venv/bin/agentcheck" --help > /dev/null
281
+
282
+ - name: A missing extra is actionable, not a traceback
283
+ if: ${{ !cancelled() }}
284
+ run: |
285
+ "$RUNNER_TEMP/base-venv/bin/python" - <<'PY'
286
+ from agentcheck.adapters import AdapterDependencyError
287
+ from agentcheck.adapters.openai_agents import _require_sdk as need_openai
288
+ from agentcheck.adapters.pydantic_ai import _require_sdk as need_pydantic
289
+
290
+ for require in (need_openai, need_pydantic):
291
+ try:
292
+ require()
293
+ except AdapterDependencyError as exc:
294
+ assert "pip install" in str(exc), exc
295
+ print("ok:", exc)
296
+ else:
297
+ raise AssertionError(f"{require} should have refused")
298
+ PY
299
+
300
+ - name: Extra openai-agents in isolation
301
+ # Installed alone, so a leak from the other extra shows up here rather
302
+ # than being masked by a combined environment.
303
+ if: ${{ !cancelled() }}
304
+ run: |
305
+ uv pip install --python "$RUNNER_TEMP/openai-venv/bin/python" ".[openai-agents]" pytest
306
+ "$RUNNER_TEMP/openai-venv/bin/python" -c "
307
+ import importlib.util
308
+ assert importlib.util.find_spec('agents') is not None
309
+ assert importlib.util.find_spec('pydantic_ai') is None, 'pydantic-ai leaked'
310
+ "
311
+ "$RUNNER_TEMP/openai-venv/bin/python" -m pytest tests/agentcheck/test_openai_adapter.py -q
312
+
313
+ - name: Extra pydantic-ai in isolation
314
+ if: ${{ !cancelled() }}
315
+ run: |
316
+ uv pip install --python "$RUNNER_TEMP/pydantic-venv/bin/python" ".[pydantic-ai]" pytest
317
+ "$RUNNER_TEMP/pydantic-venv/bin/python" -c "
318
+ import importlib.util
319
+ assert importlib.util.find_spec('pydantic_ai') is not None
320
+ assert importlib.util.find_spec('agents') is None, 'openai-agents leaked'
321
+ "
322
+ "$RUNNER_TEMP/pydantic-venv/bin/python" -m pytest tests/agentcheck/test_pydantic_ai_adapter.py -q
323
+
324
+ - name: Build wheel and sdist
325
+ if: ${{ !cancelled() }}
326
+ run: python -m build --outdir "$RUNNER_TEMP/dist"
327
+
328
+ - name: Audit artifact contents
329
+ if: ${{ !cancelled() }}
330
+ run: |
331
+ rm -rf dist && cp -r "$RUNNER_TEMP/dist" dist
332
+ python scripts/check_distribution.py
333
+
334
+ - name: Wheel install smoke test
335
+ if: ${{ !cancelled() }}
336
+ run: |
337
+ uv pip install --python "$RUNNER_TEMP/wheel-venv/bin/python" "$RUNNER_TEMP"/dist/*.whl
338
+ cd "$RUNNER_TEMP"
339
+ "$RUNNER_TEMP/wheel-venv/bin/python" -c "import agentcheck; print(agentcheck.__version__)"
340
+ "$RUNNER_TEMP/wheel-venv/bin/agentcheck" --help > /dev/null
341
+ echo "wheel install smoke passed"
@@ -0,0 +1,133 @@
1
+ # PyPI Trusted Publishing. This workflow intentionally has no manual trigger:
2
+ # publishing starts only when a non-prerelease GitHub Release is published.
3
+ #
4
+ # Before the first release, create the `pypi` GitHub Environment with a required
5
+ # reviewer (where the repository plan supports it), then register this exact
6
+ # workflow and environment as the pending publisher for `agentcheck-ai` on PyPI.
7
+
8
+ name: Publish to PyPI
9
+
10
+ on:
11
+ release:
12
+ types: [published]
13
+
14
+ permissions:
15
+ contents: read
16
+
17
+ concurrency:
18
+ group: pypi-${{ github.event.release.tag_name }}
19
+ cancel-in-progress: false
20
+
21
+ jobs:
22
+ build:
23
+ name: Build and validate distributions
24
+ if: ${{ github.event.release.prerelease == false }}
25
+ runs-on: ubuntu-latest
26
+ timeout-minutes: 15
27
+ permissions:
28
+ contents: read
29
+
30
+ steps:
31
+ - name: Checkout the published release tag
32
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
33
+ with:
34
+ ref: ${{ github.ref }}
35
+ fetch-depth: 0
36
+ persist-credentials: false
37
+
38
+ - name: Set up Python
39
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
40
+ with:
41
+ python-version: "3.12"
42
+
43
+ - name: Verify release source and version
44
+ env:
45
+ DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
46
+ RELEASE_SHA: ${{ github.sha }}
47
+ RELEASE_TAG: ${{ github.event.release.tag_name }}
48
+ run: |
49
+ python - <<'PY'
50
+ import os
51
+ import pathlib
52
+ import tomllib
53
+
54
+ metadata = tomllib.loads(pathlib.Path("pyproject.toml").read_text(encoding="utf-8"))
55
+ version = metadata["project"]["version"]
56
+ expected_tag = f"v{version}"
57
+ release_tag = os.environ["RELEASE_TAG"]
58
+ if release_tag != expected_tag:
59
+ raise SystemExit(
60
+ f"release tag {release_tag!r} does not match package version {version!r}; "
61
+ f"expected {expected_tag!r}"
62
+ )
63
+ print(f"release tag matches agentcheck-ai {version}")
64
+ PY
65
+
66
+ if [ "$(git rev-parse HEAD)" != "$RELEASE_SHA" ]; then
67
+ echo "checked-out source does not match the published release tag" >&2
68
+ exit 1
69
+ fi
70
+
71
+ if ! git merge-base --is-ancestor HEAD "refs/remotes/origin/$DEFAULT_BRANCH"; then
72
+ echo "published release tag is not reachable from $DEFAULT_BRANCH" >&2
73
+ exit 1
74
+ fi
75
+
76
+ - name: Install build validation tools
77
+ run: python -m pip install "build==1.5.0" "twine==7.0.0"
78
+
79
+ - name: Build wheel and source distribution
80
+ run: python -m build
81
+
82
+ - name: Validate distributions
83
+ run: |
84
+ python -m twine check --strict dist/*
85
+ python scripts/check_distribution.py
86
+ python - <<'PY'
87
+ import pathlib
88
+ import tomllib
89
+
90
+ metadata = tomllib.loads(pathlib.Path("pyproject.toml").read_text(encoding="utf-8"))
91
+ version = metadata["project"]["version"]
92
+ expected = {
93
+ f"agentcheck_ai-{version}-py3-none-any.whl",
94
+ f"agentcheck_ai-{version}.tar.gz",
95
+ }
96
+ actual = {path.name for path in pathlib.Path("dist").iterdir() if path.is_file()}
97
+ if actual != expected:
98
+ raise SystemExit(
99
+ f"release artifact set mismatch: expected {sorted(expected)!r}, "
100
+ f"found {sorted(actual)!r}"
101
+ )
102
+ print(f"validated release artifacts: {sorted(actual)!r}")
103
+ PY
104
+
105
+ - name: Upload validated distributions
106
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
107
+ with:
108
+ name: python-package-distributions
109
+ path: dist/
110
+ if-no-files-found: error
111
+ retention-days: 1
112
+
113
+ publish:
114
+ name: Publish distributions to PyPI
115
+ if: ${{ github.event.release.prerelease == false }}
116
+ needs: build
117
+ runs-on: ubuntu-latest
118
+ timeout-minutes: 10
119
+ environment:
120
+ name: pypi
121
+ url: https://pypi.org/p/agentcheck-ai
122
+ permissions:
123
+ id-token: write
124
+
125
+ steps:
126
+ - name: Download validated distributions
127
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
128
+ with:
129
+ name: python-package-distributions
130
+ path: dist/
131
+
132
+ - name: Publish distributions to PyPI
133
+ uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2