certora-prover-cli 0.2.4__tar.gz → 0.2.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/PKG-INFO +2 -1
  2. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/pyproject.toml +1 -0
  3. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/_version.py +2 -2
  4. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/auth.py +9 -17
  5. certora_prover_cli-0.2.6/src/prover_output_utility/budget.py +118 -0
  6. certora_prover_cli-0.2.6/src/prover_output_utility/counterexample.py +126 -0
  7. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/models.py +17 -0
  8. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/prover_cli.py +149 -3
  9. certora_prover_cli-0.2.6/src/prover_output_utility/verdicts.py +139 -0
  10. certora_prover_cli-0.2.6/src/prover_output_utility/wait.py +65 -0
  11. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/.gitignore +0 -0
  12. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/LICENSE +0 -0
  13. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/README.md +0 -0
  14. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/__init__.py +0 -0
  15. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/__init__.py +0 -0
  16. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/aws_data_fetcher.py +0 -0
  17. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/base_data_fetcher.py +0 -0
  18. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/cache.py +0 -0
  19. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/data_fetcher.py +0 -0
  20. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/local_data_fetcher.py +0 -0
  21. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/main.py +0 -0
  22. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/tree_parser.py +0 -0
  23. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/url_utils.py +0 -0
  24. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/aws_auth.py +0 -0
  25. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/breadcrumb.py +0 -0
  26. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/exceptions.py +0 -0
  27. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/group_summary.py +0 -0
  28. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/job_report.py +0 -0
  29. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/parsers.py +0 -0
  30. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/prover_api.py +0 -0
  31. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/py.typed +0 -0
  32. {certora_prover_cli-0.2.4 → certora_prover_cli-0.2.6}/src/prover_output_utility/sighash.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: certora-prover-cli
3
- Version: 0.2.4
3
+ Version: 0.2.6
4
4
  Summary: A Python API and CLI for parsing outputs from the Certora Prover
5
5
  Project-URL: Homepage, https://github.com/Certora/ProverCLI
6
6
  Project-URL: Repository, https://github.com/Certora/ProverCLI
@@ -21,6 +21,7 @@ Requires-Dist: requests>=2.28.0
21
21
  Requires-Dist: urllib3>=1.26.0
22
22
  Provides-Extra: dev
23
23
  Requires-Dist: build>=1.0; extra == 'dev'
24
+ Requires-Dist: pytest>=8.0; extra == 'dev'
24
25
  Requires-Dist: reuse>=5.0; extra == 'dev'
25
26
  Description-Content-Type: text/markdown
26
27
 
@@ -37,6 +37,7 @@ dependencies = [
37
37
  [project.optional-dependencies]
38
38
  dev = [
39
39
  "build>=1.0",
40
+ "pytest>=8.0",
40
41
  "reuse>=5.0",
41
42
  ]
42
43
 
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.2.4'
22
- __version_tuple__ = version_tuple = (0, 2, 4)
21
+ __version__ = version = '0.2.6'
22
+ __version_tuple__ = version_tuple = (0, 2, 6)
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -83,28 +83,20 @@ except ImportError:
83
83
  SERVERS = None
84
84
 
85
85
 
86
- # Environment variables holding an organization API key, in the order they are
87
- # consulted. The env-scoped name comes first so a machine configured for one
88
- # environment cannot be pointed at another by a stray generic value.
89
- _API_KEY_ENV_VARS: Dict[LoginEnv, tuple[str, ...]] = {
90
- "prod": ("CERTORA_API_KEY_PROD", "CERTORA_API_KEY"),
91
- "stg": ("CERTORA_API_KEY_STG", "CERTORA_API_KEY"),
92
- "dev": ("CERTORA_API_KEY_DEV", "CERTORA_API_KEY"),
93
- }
86
+ #: The environment variable holding the Certora key, the one certora-cli reads,
87
+ #: so one value serves submitting a job and reading its results.
88
+ KEY_ENV_VAR = "CERTORAKEY"
94
89
 
95
90
 
96
91
  def organization_api_key() -> str | None:
97
- """The organization API key for the current login env, if one is set.
92
+ """The organization API key, if one is set.
98
93
 
99
- This is the key issued on the organization's API page, not ``CERTORAKEY``.
100
- The API accepts it on the ``Authorization`` header and exchanges it for a
101
- short-lived token itself, so a caller that has one needs no browser login.
94
+ This is the key issued on the organization's API page, the same one
95
+ certora-cli submits jobs with. The API accepts it on the ``Authorization``
96
+ header and exchanges it for a short-lived token itself, so a caller that
97
+ has one needs no browser login.
102
98
  """
103
- for name in _API_KEY_ENV_VARS[resolve_login_env()]:
104
- value = (os.environ.get(name) or "").strip()
105
- if value:
106
- return value
107
- return None
99
+ return (os.environ.get(KEY_ENV_VAR) or "").strip() or None
108
100
 
109
101
 
110
102
  class ProverAuth:
@@ -0,0 +1,118 @@
1
+ # SPDX-FileCopyrightText: 2026 Certora Ltd.
2
+ #
3
+ # SPDX-License-Identifier: GPL-3.0-only
4
+
5
+ """
6
+ How many prover jobs and prover minutes a group of cloud jobs has used, against a budget.
7
+
8
+ The jobs are the ones the data API lists for the group (``certoraRun --group_id``), read in one
9
+ request. A finished job counts its runtime on the server, rounded up to whole minutes. A job not
10
+ yet finished counts ``reserve`` minutes or its time so far, whichever is larger: never less than it
11
+ has used, but less than it will use if it runs past ``reserve``. The use is also broken down by
12
+ the first word of each job's ``msg``. A job submitted moments ago may not be listed yet.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import math
18
+ from dataclasses import dataclass, field
19
+ from datetime import datetime, timedelta, timezone
20
+ from typing import Any, Dict, List, Optional
21
+
22
+
23
+ @dataclass
24
+ class Usage:
25
+ jobs: int = 0
26
+ minutes: int = 0
27
+ running: int = 0
28
+
29
+
30
+ @dataclass
31
+ class BudgetUsage:
32
+ """Budget, use, what is left, use per ``msg`` prefix, and whether ``more_jobs`` more fit."""
33
+
34
+ group_id: str
35
+ max_jobs: int
36
+ max_minutes: int
37
+ reserve_minutes: int
38
+ used: Usage
39
+ more_jobs: int = 0
40
+ by_msg: Dict[str, Usage] = field(default_factory=dict)
41
+
42
+ @property
43
+ def left_jobs(self) -> int:
44
+ return self.max_jobs - self.used.jobs
45
+
46
+ @property
47
+ def left_minutes(self) -> int:
48
+ return self.max_minutes - self.used.minutes
49
+
50
+ @property
51
+ def fits(self) -> bool:
52
+ return self.more_jobs <= self.left_jobs and self.more_jobs * self.reserve_minutes <= self.left_minutes
53
+
54
+ def to_dict(self) -> Dict[str, Any]:
55
+ return {
56
+ "group_id": self.group_id,
57
+ "budget": {"jobs": self.max_jobs, "minutes": self.max_minutes},
58
+ "reserve_minutes": self.reserve_minutes,
59
+ "used": vars(self.used),
60
+ "left": {"jobs": self.left_jobs, "minutes": self.left_minutes},
61
+ "more_jobs": self.more_jobs,
62
+ "fits": self.fits,
63
+ "by_msg": {prefix: vars(u) for prefix, u in self.by_msg.items()},
64
+ }
65
+
66
+
67
+ def _time(value: Optional[str]) -> Optional[datetime]:
68
+ return datetime.fromisoformat(value.replace("Z", "+00:00")) if value else None
69
+
70
+
71
+ def job_minutes(job: Dict[str, Any], reserve: int, now: datetime) -> tuple[int, bool]:
72
+ """A listed job's minutes, and whether it has finished."""
73
+ start, finish = _time(job.get("start_time")), _time(job.get("finish_time"))
74
+ if finish:
75
+ return (math.ceil((finish - start).total_seconds() / 60) if start else 0), True
76
+ elapsed = math.ceil((now - start).total_seconds() / 60) if start else 0
77
+ return max(elapsed, reserve), False
78
+
79
+
80
+ def budget_usage(
81
+ api: Any,
82
+ group_id: str,
83
+ max_jobs: int,
84
+ max_minutes: int,
85
+ reserve: int = 30,
86
+ more_jobs: int = 0,
87
+ days_back: int = 365,
88
+ ) -> BudgetUsage:
89
+ now = datetime.now(timezone.utc)
90
+ jobs: List[Dict[str, Any]] = api.data_fetcher.fetch_group_jobs(
91
+ group_id, (now - timedelta(days=days_back)).isoformat(), now.isoformat()
92
+ )
93
+ usage = BudgetUsage(group_id, max_jobs, max_minutes, reserve, Usage(), more_jobs)
94
+ for job in jobs:
95
+ minutes, final = job_minutes(job, reserve, now)
96
+ prefix = (job.get("msg") or "").split(" ", 1)[0] or "(no msg)"
97
+ for u in (usage.used, usage.by_msg.setdefault(prefix, Usage())):
98
+ u.jobs += 1
99
+ u.minutes += minutes
100
+ u.running += not final
101
+ usage.by_msg = dict(sorted(usage.by_msg.items(), key=lambda kv: -kv[1].minutes))
102
+ return usage
103
+
104
+
105
+ def format_budget_text(u: BudgetUsage) -> str:
106
+ lines = [
107
+ f"group {u.group_id}",
108
+ f"budget {u.max_jobs} jobs, {u.max_minutes} min",
109
+ f"used {u.used.jobs} jobs, {u.used.minutes} min "
110
+ f"({u.used.running} running, counted at {u.reserve_minutes} min or their time so far)",
111
+ f"left {u.left_jobs} jobs, {u.left_minutes} min",
112
+ ]
113
+ if u.more_jobs:
114
+ verdict = "fit" if u.fits else "do NOT fit"
115
+ lines.append(f"{u.more_jobs} more jobs (up to {u.more_jobs * u.reserve_minutes} min) {verdict}")
116
+ for prefix, p in u.by_msg.items():
117
+ lines.append(f" {prefix:44} {p.jobs:4} jobs {p.minutes:6} min {p.running:3} running")
118
+ return "\n".join(lines)
@@ -0,0 +1,126 @@
1
+ # SPDX-FileCopyrightText: 2026 Certora Ltd.
2
+ #
3
+ # SPDX-License-Identifier: GPL-3.0-only
4
+
5
+ """
6
+ Render a violated rule's counterexample as compact XML.
7
+
8
+ A rule's raw output (``prover-cli --calltrace rule_output_<k>.json``) holds the failing assertion,
9
+ its source location, and the call trace as a tree of frames. The rendering keeps the frames a
10
+ reader needs, substitutes each frame's values into its message, and leaves out the frames every
11
+ trace of a chain carries.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import re
17
+ from dataclasses import dataclass
18
+ from typing import Any, Dict, Optional
19
+
20
+ #: The prover writes a frame's message as a format template: literal text with ``'{n}'`` standing in
21
+ #: for the n-th of the message's ``arguments``.
22
+ _PLACEHOLDER = re.compile(r"'\{(\d+)\}'")
23
+
24
+
25
+ def _frame(message: Dict[str, Any]) -> str:
26
+ """A frame's identity: its message template with every value replaced by ``{}``.
27
+
28
+ A counterexample's values live in ``arguments``, never in ``text``, so frames emitted from the
29
+ same site share this string whatever values they took.
30
+ """
31
+ return _PLACEHOLDER.sub("{}", message.get("text", ""))
32
+
33
+
34
+ def _render_message(message: Dict[str, Any]) -> str:
35
+ """The message template with this counterexample's values substituted for its placeholders."""
36
+ arguments = message.get("arguments") or []
37
+
38
+ def value(placeholder: re.Match[str]) -> str:
39
+ index = int(placeholder.group(1))
40
+ return f"'{arguments[index].get('value')}'" if index < len(arguments) else placeholder.group(0)
41
+
42
+ return _PLACEHOLDER.sub(value, message.get("text", ""))
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class TraceShape:
47
+ """Which call-trace frames are worth rendering for one chain.
48
+
49
+ Frames are named by their message template (``{}`` where a value goes). ``dropped`` removes a
50
+ frame and everything under it. ``elided`` keeps the frame and replaces its subtree with a count.
51
+ """
52
+
53
+ dropped: frozenset[str] = frozenset()
54
+ elided: frozenset[str] = frozenset()
55
+
56
+
57
+ #: Frames every chain's traces carry and no counterexample needs.
58
+ _GENERIC_NOISE = frozenset({"Setup", "Global State"})
59
+
60
+ GENERIC_TRACE = TraceShape(dropped=_GENERIC_NOISE)
61
+
62
+ EVM_TRACE = TraceShape(
63
+ dropped=_GENERIC_NOISE | {"Evaluate branch condition", "unknown loop source code"},
64
+ )
65
+
66
+ #: Solana traces are dominated by account materialization under
67
+ #: ``cvlr_deserialize_nondet_accounts``. That subtree is elided rather than dropped, so a reader can
68
+ #: tell setup happened and how much of it was hidden. ``unknown loop source code`` is kept: a
69
+ #: loop-unwinding violation nests its failure under it. rustc 1.89 and later name the allocator shim
70
+ #: ``__rustc::__rust_alloc``, so both spellings are dropped.
71
+ SOLANA_TRACE = TraceShape(
72
+ dropped=_GENERIC_NOISE | {"__rust_alloc: {}", "__rustc::__rust_alloc: {}", "CVT_alloc_slice: {}"},
73
+ elided=frozenset({"cvlr_solana::layout::cvlr_deserialize_nondet_accounts(...)"}),
74
+ )
75
+
76
+ TRACE_SHAPES: Dict[str, TraceShape] = {
77
+ "evm": EVM_TRACE,
78
+ "solana": SOLANA_TRACE,
79
+ "soroban": GENERIC_TRACE,
80
+ }
81
+
82
+
83
+ def _descendants(node: Dict[str, Any]) -> int:
84
+ return sum(1 + _descendants(child) for child in node.get("childrenList") or [])
85
+
86
+
87
+ def calltrace_to_xml(node: Dict[str, Any], shape: TraceShape) -> str:
88
+ """Render one call-trace node and the children ``shape`` keeps."""
89
+ parts = [f"<message>{_render_message(node.get('message') or {})}</message>"]
90
+ for child in node.get("childrenList") or []:
91
+ message = child.get("message") or {}
92
+ frame = _frame(message)
93
+ if frame in shape.dropped:
94
+ continue
95
+ if frame in shape.elided:
96
+ parts.append(
97
+ f"<child><message>{_render_message(message)}</message>"
98
+ f"<elided>{_descendants(child)} frames of setup</elided></child>"
99
+ )
100
+ continue
101
+ parts.append(f"<child>{calltrace_to_xml(child, shape)}</child>")
102
+ return "".join(parts)
103
+
104
+
105
+ def render_counterexample(dump: Dict[str, Any], shape: TraceShape) -> Optional[str]:
106
+ """One violated rule's counterexample, or None when its output carries no call trace.
107
+
108
+ ``dump`` is the rule's raw output file (``assertMessage``, ``jumpToDefinition``,
109
+ ``callTrace``). The result is ``<counterexample>`` with the failing ``<assert>``, its
110
+ ``<source>`` as ``file:line``, and the rendered trace. ``shape`` has no default: another
111
+ chain's shape would silently drop frames this chain's traces need.
112
+ """
113
+ trace = dump.get("callTrace")
114
+ if not isinstance(trace, dict):
115
+ return None
116
+ parts = []
117
+ assertion = dump.get("assertMessage")
118
+ if isinstance(assertion, str) and assertion:
119
+ parts.append(f"<assert>{assertion}</assert>")
120
+ location = dump.get("jumpToDefinition")
121
+ if isinstance(location, dict):
122
+ file, start = location.get("file"), location.get("start")
123
+ if isinstance(file, str) and isinstance(start, dict) and isinstance(start.get("line"), int):
124
+ parts.append(f"<source>{file}:{start['line']}</source>")
125
+ parts.append(calltrace_to_xml(trace, shape))
126
+ return "<counterexample>" + "".join(parts) + "</counterexample>"
@@ -62,6 +62,23 @@ class JobStatus(str, Enum):
62
62
  # Unknown status (fallback for unrecognized values)
63
63
  UNKNOWN = "UNKNOWN"
64
64
 
65
+ @property
66
+ def is_final(self) -> bool:
67
+ """Whether the job has stopped: its verdicts are final, or it ended without a result."""
68
+ return self in _FINAL_JOB_STATUSES
69
+
70
+
71
+ _FINAL_JOB_STATUSES = frozenset(
72
+ {
73
+ JobStatus.SUCCEEDED,
74
+ JobStatus.FAILED,
75
+ JobStatus.CANCELED,
76
+ JobStatus.HALTED,
77
+ JobStatus.SERVICE_UNAVAILABLE,
78
+ JobStatus.UPLOAD_FAILED,
79
+ }
80
+ )
81
+
65
82
 
66
83
  class NodeType(str, Enum):
67
84
  """Type of a tree view node."""
@@ -17,10 +17,17 @@ from pathlib import Path
17
17
 
18
18
  from .api import ProverOutputAPI
19
19
  from .breadcrumb import format_breadcrumbs_text
20
+ from .budget import budget_usage, format_budget_text
21
+ from .counterexample import TRACE_SHAPES
20
22
  from .exceptions import AuthenticationError, JobNotFoundError, ProverAPIError
21
23
  from .group_summary import build_group_summary, extract_group_id, format_text as format_group_text
22
24
  from .models import CalltraceInfo, CheckResult, ParsedAlert
23
25
  from .sighash import handle_sighash_command
26
+ from .verdicts import format_verdicts_text, job_verdicts
27
+ from .wait import format_wait_text, wait_for_jobs
28
+
29
+ #: Exit code of ``--budget --more-jobs K`` when K more jobs do not fit.
30
+ OVER_BUDGET = 3
24
31
 
25
32
 
26
33
  def format_check_result(check_result: CheckResult, index: int) -> str:
@@ -126,8 +133,14 @@ Examples:
126
133
  %(prog)s --job-id 12345 --full-report
127
134
  %(prog)s --job-id 12345 --full-report --format json
128
135
  %(prog)s --job-id 12345 --download-sources ./job_sources
136
+ %(prog)s --job-id 12345 --verdicts --save-report report.json
129
137
  %(prog)s --cancel 12345 67890
130
138
 
139
+ # Waiting for jobs, and a budget over a group of jobs
140
+ %(prog)s --wait 12345 "https://prover.certora.com/output/..." --timeout 90
141
+ %(prog)s --budget --group-id GROUP --max-jobs 500 --max-minutes 5000 --more-jobs 3
142
+ %(prog)s --budget --budget-file budget.json --more-jobs 3
143
+
131
144
  # Group summary
132
145
  %(prog)s --group-id b78ea54a-a924-4f05-b1a7-58b29c1585ae
133
146
  %(prog)s --group-id "https://prover.certora.com/?groupIds=b78ea54a-..." --format json
@@ -182,6 +195,13 @@ Environment Variables:
182
195
  help="List recent jobs from the data API. "
183
196
  "Use --days-back, --limit, --my-jobs-only to refine.",
184
197
  )
198
+ job_group.add_argument(
199
+ "--wait",
200
+ nargs="+",
201
+ metavar="JOB",
202
+ help="Wait until these jobs (ids or links) have stopped, for at most --timeout seconds. "
203
+ "Prints one line per job and then ENDED or RUNNING.",
204
+ )
185
205
 
186
206
  parser.add_argument("--output", "-o", help="Output JSON file path")
187
207
  parser.add_argument("--status-only", action="store_true", help="Only get job status")
@@ -233,6 +253,65 @@ Environment Variables:
233
253
  "individually in parallel; a completed fetch is skipped on rerun."
234
254
  ),
235
255
  )
256
+ parser.add_argument(
257
+ "--verdicts",
258
+ action="store_true",
259
+ help="The job's verdicts rule by rule, from its leaf checks (one per method and per "
260
+ "assertion), with a rendered counterexample for every violation.",
261
+ )
262
+ parser.add_argument(
263
+ "--save-report",
264
+ metavar="FILE",
265
+ help="Used with --verdicts: also write the verdicts, counterexamples and full job report "
266
+ "to FILE as JSON, and still print the verdicts.",
267
+ )
268
+ parser.add_argument(
269
+ "--chain",
270
+ choices=sorted(TRACE_SHAPES),
271
+ default="evm",
272
+ help="Which frames a counterexample keeps (default: evm). Used with --verdicts.",
273
+ )
274
+ parser.add_argument(
275
+ "--timeout",
276
+ type=float,
277
+ default=90,
278
+ help="Seconds --wait waits at most (default: 90).",
279
+ )
280
+ parser.add_argument(
281
+ "--interval",
282
+ type=float,
283
+ default=30,
284
+ help="Seconds between checks while --wait waits (default: 30).",
285
+ )
286
+ parser.add_argument(
287
+ "--budget",
288
+ action="store_true",
289
+ help="Prover jobs and minutes used by the cloud jobs of a group (--group-id, or "
290
+ "group_id in --budget-file), against --max-jobs/--max-minutes or the budget file. "
291
+ f"With --more-jobs K, exits {OVER_BUDGET} when K more jobs do not fit.",
292
+ )
293
+ parser.add_argument(
294
+ "--budget-file",
295
+ metavar="FILE",
296
+ help='Used with --budget: a JSON file {"jobs": N, "minutes": M, "group_id": G}; flags '
297
+ "given on the command line take precedence.",
298
+ )
299
+ parser.add_argument("--max-jobs", type=int, help="Job budget for --budget.")
300
+ parser.add_argument("--max-minutes", type=int, help="Minute budget for --budget.")
301
+ parser.add_argument(
302
+ "--reserve-minutes",
303
+ type=int,
304
+ default=30,
305
+ help="Minutes --budget counts for a job that has not finished (or its time so far, if "
306
+ "larger), and for each of --more-jobs (default: 30).",
307
+ )
308
+ parser.add_argument(
309
+ "--more-jobs",
310
+ type=int,
311
+ default=0,
312
+ metavar="K",
313
+ help=f"Used with --budget: exit {OVER_BUDGET} unless K more jobs fit.",
314
+ )
236
315
  parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")
237
316
  parser.add_argument("--format", choices=["json", "text"], default="text", help="Output format")
238
317
  parser.add_argument(
@@ -334,8 +413,8 @@ Environment Variables:
334
413
  )
335
414
  sys.exit(1)
336
415
 
337
- # Handle --group-id (after auth check, doesn't need job input)
338
- if args.group_id:
416
+ # Handle --group-id (after auth check, doesn't need job input; --budget reads it itself)
417
+ if args.group_id and not args.budget:
339
418
  try:
340
419
  group_id = extract_group_id(args.group_id)
341
420
  api = ProverOutputAPI(use_local=False)
@@ -402,10 +481,65 @@ Environment Variables:
402
481
  print(f"❌ API error: {e}", file=sys.stderr)
403
482
  sys.exit(1)
404
483
 
484
+ # Handle --wait (after auth check)
485
+ if args.wait:
486
+ try:
487
+ api = ProverOutputAPI(use_local=False)
488
+ result = wait_for_jobs(api, args.wait, timeout=args.timeout, interval=args.interval)
489
+ if args.output:
490
+ with open(args.output, "w") as f:
491
+ json.dump(result.to_dict(), f, indent=2)
492
+ print(f"{'ENDED' if result.ended else 'RUNNING'}. Results written to {args.output}")
493
+ elif args.format == "json":
494
+ print(json.dumps(result.to_dict(), indent=2))
495
+ else:
496
+ print(format_wait_text(result))
497
+ return
498
+ except AuthenticationError as e:
499
+ print(f"❌ Authentication failed: {e}", file=sys.stderr)
500
+ sys.exit(1)
501
+
502
+ # Handle --budget (after auth check; reads the group's jobs from the data API)
503
+ if args.budget:
504
+ limits = json.loads(Path(args.budget_file).read_text()) if args.budget_file else {}
505
+ group_id = args.group_id or limits.get("group_id")
506
+ max_jobs = args.max_jobs if args.max_jobs is not None else limits.get("jobs")
507
+ max_minutes = args.max_minutes if args.max_minutes is not None else limits.get("minutes")
508
+ if not group_id or max_jobs is None or max_minutes is None:
509
+ parser.error("--budget needs a group id and both limits, from flags or --budget-file")
510
+ try:
511
+ api = ProverOutputAPI(use_local=False)
512
+ usage = budget_usage(
513
+ api,
514
+ extract_group_id(group_id),
515
+ max_jobs,
516
+ max_minutes,
517
+ reserve=args.reserve_minutes,
518
+ more_jobs=args.more_jobs,
519
+ days_back=args.days_back,
520
+ )
521
+ except AuthenticationError as e:
522
+ print(f"❌ Authentication failed: {e}", file=sys.stderr)
523
+ sys.exit(1)
524
+ except ProverAPIError as e:
525
+ print(f"❌ API error: {e}", file=sys.stderr)
526
+ sys.exit(1)
527
+ if args.output:
528
+ with open(args.output, "w") as f:
529
+ json.dump(usage.to_dict(), f, indent=2)
530
+ print(f"{usage.left_jobs} jobs, {usage.left_minutes} min left. Results written to {args.output}")
531
+ elif args.format == "json":
532
+ print(json.dumps(usage.to_dict(), indent=2))
533
+ else:
534
+ print(format_budget_text(usage))
535
+ if args.more_jobs and not usage.fits:
536
+ sys.exit(OVER_BUDGET)
537
+ return
538
+
405
539
  # Check that we have a job input (unless it's who-am-i or sighash which were handled above)
406
540
  if not args.cancel and not (args.job_id or args.job_url or args.local_path):
407
541
  parser.error(
408
- "Please provide either --job-id, --job-url, --local-path, --cancel, --group-id, --sighash, --recent-jobs, or --who-am-i"
542
+ "Please provide either --job-id, --job-url, --local-path, --cancel, --group-id, --sighash, --recent-jobs, --wait, --budget, or --who-am-i"
409
543
  )
410
544
 
411
545
  # Handle cancel case differently since it takes multiple inputs
@@ -433,6 +567,16 @@ Environment Variables:
433
567
  "result": cancel_result,
434
568
  }
435
569
 
570
+ elif args.verdicts:
571
+ verdicts = job_verdicts(api, job_input, chain=args.chain)
572
+ result = verdicts.to_dict()
573
+ if args.save_report:
574
+ with open(args.save_report, "w") as f:
575
+ json.dump(result, f, indent=1)
576
+ if not args.output and args.format != "json":
577
+ print(format_verdicts_text(verdicts))
578
+ return
579
+
436
580
  elif args.status_only:
437
581
  # Just get job status
438
582
  status = api.get_job_status(job_input)
@@ -557,6 +701,8 @@ Environment Variables:
557
701
  print(f"Found {count} alerts. Results written to {args.output}")
558
702
  elif args.full_report:
559
703
  print(f"✅ Full report retrieved. Results written to {args.output}")
704
+ elif args.verdicts:
705
+ print(f"Verdicts for {len(result['rules'])} rules. Results written to {args.output}")
560
706
  elif args.download_sources:
561
707
  print(
562
708
  f"Downloaded {result['files_count']} source files to {result['sources_dir']}. "
@@ -0,0 +1,139 @@
1
+ # SPDX-FileCopyrightText: 2026 Certora Ltd.
2
+ #
3
+ # SPDX-License-Identifier: GPL-3.0-only
4
+
5
+ """
6
+ A job's verdicts, rule by rule, with a counterexample for every violation.
7
+
8
+ The verdicts are the job's leaf checks: one per method of a parametric rule or invariant, and one
9
+ per assertion. A rule's top-level node only summarizes them. It can read ERROR while one of its
10
+ methods is VIOLATED, or VIOLATED while other methods are SANITY_FAILED, and it carries no
11
+ counterexample file, so the leaves are what to read.
12
+
13
+ A failed vacuity check (``rule_not_vacuous``, status SANITY_FAILED) is a verdict too. For a method
14
+ of a parametric rule the prover also marks the method SANITY_FAILED, but for a rule split into
15
+ sub-checks (a ``satisfy``, auto-generated assertions) the vacuity check is the only place it shows,
16
+ so it is added as a leaf of its own wherever its rule or method is not already SANITY_FAILED.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ from collections import Counter
22
+ from dataclasses import dataclass, field
23
+ from typing import Any, Dict, List, Optional
24
+
25
+ from .counterexample import TRACE_SHAPES, render_counterexample
26
+ from .job_report import JobReport
27
+ from .models import CheckResult, NodeStatus, NodeType
28
+
29
+ VACUITY_CHECK = "rule_not_vacuous"
30
+
31
+
32
+ def _failed_vacuity_checks(api: Any, job_input: str, leaves: List[CheckResult]) -> List[CheckResult]:
33
+ """The failed vacuity checks no leaf already reports as SANITY_FAILED."""
34
+ reported = {(c.rule_name, c.method_name) for c in leaves if c.status == NodeStatus.SANITY_FAILED}
35
+ return [
36
+ c
37
+ for c in api.get_all_checks(job_input, include_rule_not_vacuous=True)
38
+ if c.node_type == NodeType.SANITY
39
+ and c.assert_message == VACUITY_CHECK
40
+ and c.status == NodeStatus.SANITY_FAILED
41
+ and (c.rule_name, c.method_name) not in reported
42
+ ]
43
+
44
+
45
+ @dataclass
46
+ class RuleVerdicts:
47
+ """One rule's leaf statuses counted, and the leaves worth naming: none when every leaf is
48
+ VERIFIED; otherwise every leaf that is not, plus the VERIFIED ones that are not methods (a
49
+ rule's sub-checks, which the counts alone would leave unnamed)."""
50
+
51
+ counts: Dict[str, int] = field(default_factory=dict)
52
+ leaves: List[CheckResult] = field(default_factory=list)
53
+
54
+
55
+ @dataclass
56
+ class RenderedCounterexample:
57
+ rule: str
58
+ method: Optional[str]
59
+ output_file: str
60
+ counterexample: str
61
+
62
+
63
+ @dataclass
64
+ class JobVerdicts:
65
+ """A job's verdicts per rule, its rendered counterexamples, and its full report."""
66
+
67
+ job_id: str
68
+ report: JobReport
69
+ rules: Dict[str, RuleVerdicts] = field(default_factory=dict)
70
+ counterexamples: List[RenderedCounterexample] = field(default_factory=list)
71
+
72
+ def to_dict(self) -> Dict[str, Any]:
73
+ return {
74
+ "job_id": self.job_id,
75
+ "job_status": self.report.job_status.value,
76
+ "rules": {
77
+ name: {
78
+ "counts": r.counts,
79
+ "leaves": [
80
+ {"method": c.method_name, "status": c.status.value, "assert": c.assert_message}
81
+ for c in r.leaves
82
+ ],
83
+ }
84
+ for name, r in self.rules.items()
85
+ },
86
+ "counterexamples": [vars(c) for c in self.counterexamples],
87
+ "report": self.report.to_dict(),
88
+ }
89
+
90
+
91
+ def job_verdicts(api: Any, job_input: str, chain: str) -> JobVerdicts:
92
+ """Read the job's leaf checks and render the counterexample of every violated one."""
93
+ shape = TRACE_SHAPES[chain]
94
+ leaves: List[CheckResult] = api.get_leaf_checks(job_input)
95
+ leaves += _failed_vacuity_checks(api, job_input, leaves)
96
+ result = JobVerdicts(job_id=job_input, report=api.get_job_report(job_input))
97
+
98
+ by_rule: Dict[str, List[CheckResult]] = {}
99
+ for leaf in leaves:
100
+ by_rule.setdefault(leaf.rule_name, []).append(leaf)
101
+ for name, rule_leaves in by_rule.items():
102
+ failed = any(not c.is_verified for c in rule_leaves)
103
+ result.rules[name] = RuleVerdicts(
104
+ counts=dict(Counter(c.status.value for c in rule_leaves).most_common()),
105
+ leaves=[c for c in rule_leaves if not c.is_verified or (failed and not c.method_name)],
106
+ )
107
+
108
+ seen = set()
109
+ for leaf in leaves:
110
+ if not leaf.is_violated:
111
+ continue
112
+ for name in leaf.output_files or []:
113
+ if name in seen:
114
+ continue
115
+ seen.add(name)
116
+ dump = api.get_calltrace(job_input, name).trace_data
117
+ rendered = render_counterexample(dump, shape) if isinstance(dump, dict) else None
118
+ if rendered:
119
+ result.counterexamples.append(
120
+ RenderedCounterexample(leaf.rule_name, leaf.method_name, name, rendered)
121
+ )
122
+ return result
123
+
124
+
125
+ def format_verdicts_text(verdicts: JobVerdicts) -> str:
126
+ """One line per rule with its leaf statuses counted, a line per listed leaf, then each
127
+ counterexample."""
128
+ lines = [f"job {verdicts.report.job_status.value}"]
129
+ for name, rule in verdicts.rules.items():
130
+ lines.append(name + "\t" + ", ".join(f"{s} {n}" for s, n in rule.counts.items()))
131
+ for leaf in rule.leaves:
132
+ where = leaf.method_name or leaf.assert_message or ""
133
+ if where and where != name:
134
+ lines.append(f"\t{where}\t{leaf.status.value}")
135
+ for cex in verdicts.counterexamples:
136
+ lines.append("")
137
+ lines.append(f"== {cex.rule} {cex.method or ''}".rstrip())
138
+ lines.append(cex.counterexample)
139
+ return "\n".join(lines)
@@ -0,0 +1,65 @@
1
+ # SPDX-FileCopyrightText: 2026 Certora Ltd.
2
+ #
3
+ # SPDX-License-Identifier: GPL-3.0-only
4
+
5
+ """
6
+ Wait, for a bounded time, until a set of prover jobs has stopped.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import time
12
+ from dataclasses import asdict, dataclass, field
13
+ from typing import Any, Dict, List
14
+
15
+ #: The status of a job whose status request failed. It does not end the wait.
16
+ CHECK_FAILED = "CHECK_FAILED"
17
+
18
+
19
+ @dataclass
20
+ class WaitRow:
21
+ """One job, as given (id or link), with its status."""
22
+
23
+ target: str
24
+ status: str
25
+ final: bool
26
+
27
+
28
+ @dataclass
29
+ class WaitResult:
30
+ ended: bool
31
+ jobs: List[WaitRow] = field(default_factory=list)
32
+
33
+ def to_dict(self) -> Dict[str, Any]:
34
+ return {"ended": self.ended, "jobs": [asdict(r) for r in self.jobs]}
35
+
36
+
37
+ def check_jobs(api: Any, jobs: List[str]) -> List[WaitRow]:
38
+ """One row per job id or job link."""
39
+ rows = []
40
+ for job in jobs:
41
+ try:
42
+ status = api.get_job_status(job)
43
+ rows.append(WaitRow(job, status.value, status.is_final))
44
+ except Exception:
45
+ rows.append(WaitRow(job, CHECK_FAILED, False))
46
+ return rows
47
+
48
+
49
+ def wait_for_jobs(api: Any, jobs: List[str], timeout: float, interval: float = 30) -> WaitResult:
50
+ """Check the jobs every ``interval`` seconds until every one has stopped or ``timeout`` seconds
51
+ have passed, with a last check at the deadline. ``ended`` says which."""
52
+ deadline = time.monotonic() + timeout
53
+ while True:
54
+ rows = check_jobs(api, jobs)
55
+ ended = all(r.final for r in rows)
56
+ remaining = deadline - time.monotonic()
57
+ if ended or remaining <= 0:
58
+ return WaitResult(ended=ended, jobs=rows)
59
+ time.sleep(min(interval, remaining))
60
+
61
+
62
+ def format_wait_text(result: WaitResult) -> str:
63
+ lines = [f"{r.target} {r.status}" for r in result.jobs]
64
+ lines.append("ENDED" if result.ended else "RUNNING")
65
+ return "\n".join(lines)