certora-prover-cli 0.2.5__tar.gz → 0.2.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/PKG-INFO +2 -1
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/pyproject.toml +1 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/_version.py +2 -2
- certora_prover_cli-0.2.6/src/prover_output_utility/budget.py +118 -0
- certora_prover_cli-0.2.6/src/prover_output_utility/counterexample.py +126 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/models.py +17 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/prover_cli.py +149 -3
- certora_prover_cli-0.2.6/src/prover_output_utility/verdicts.py +139 -0
- certora_prover_cli-0.2.6/src/prover_output_utility/wait.py +65 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/.gitignore +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/LICENSE +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/README.md +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/__init__.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/__init__.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/aws_data_fetcher.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/base_data_fetcher.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/cache.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/data_fetcher.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/local_data_fetcher.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/main.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/tree_parser.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/url_utils.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/auth.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/aws_auth.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/breadcrumb.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/exceptions.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/group_summary.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/job_report.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/parsers.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/prover_api.py +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/py.typed +0 -0
- {certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/sighash.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: certora-prover-cli
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.6
|
|
4
4
|
Summary: A Python API and CLI for parsing outputs from the Certora Prover
|
|
5
5
|
Project-URL: Homepage, https://github.com/Certora/ProverCLI
|
|
6
6
|
Project-URL: Repository, https://github.com/Certora/ProverCLI
|
|
@@ -21,6 +21,7 @@ Requires-Dist: requests>=2.28.0
|
|
|
21
21
|
Requires-Dist: urllib3>=1.26.0
|
|
22
22
|
Provides-Extra: dev
|
|
23
23
|
Requires-Dist: build>=1.0; extra == 'dev'
|
|
24
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
24
25
|
Requires-Dist: reuse>=5.0; extra == 'dev'
|
|
25
26
|
Description-Content-Type: text/markdown
|
|
26
27
|
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.2.
|
|
22
|
-
__version_tuple__ = version_tuple = (0, 2,
|
|
21
|
+
__version__ = version = '0.2.6'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 2, 6)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 Certora Ltd.
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: GPL-3.0-only
|
|
4
|
+
|
|
5
|
+
"""
|
|
6
|
+
How many prover jobs and prover minutes a group of cloud jobs has used, against a budget.
|
|
7
|
+
|
|
8
|
+
The jobs are the ones the data API lists for the group (``certoraRun --group_id``), read in one
|
|
9
|
+
request. A finished job counts its runtime on the server, rounded up to whole minutes. A job not
|
|
10
|
+
yet finished counts ``reserve`` minutes or its time so far, whichever is larger: never less than it
|
|
11
|
+
has used, but less than it will use if it runs past ``reserve``. The use is also broken down by
|
|
12
|
+
the first word of each job's ``msg``. A job submitted moments ago may not be listed yet.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import math
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
from datetime import datetime, timedelta, timezone
|
|
20
|
+
from typing import Any, Dict, List, Optional
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class Usage:
|
|
25
|
+
jobs: int = 0
|
|
26
|
+
minutes: int = 0
|
|
27
|
+
running: int = 0
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class BudgetUsage:
|
|
32
|
+
"""Budget, use, what is left, use per ``msg`` prefix, and whether ``more_jobs`` more fit."""
|
|
33
|
+
|
|
34
|
+
group_id: str
|
|
35
|
+
max_jobs: int
|
|
36
|
+
max_minutes: int
|
|
37
|
+
reserve_minutes: int
|
|
38
|
+
used: Usage
|
|
39
|
+
more_jobs: int = 0
|
|
40
|
+
by_msg: Dict[str, Usage] = field(default_factory=dict)
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def left_jobs(self) -> int:
|
|
44
|
+
return self.max_jobs - self.used.jobs
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def left_minutes(self) -> int:
|
|
48
|
+
return self.max_minutes - self.used.minutes
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def fits(self) -> bool:
|
|
52
|
+
return self.more_jobs <= self.left_jobs and self.more_jobs * self.reserve_minutes <= self.left_minutes
|
|
53
|
+
|
|
54
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
55
|
+
return {
|
|
56
|
+
"group_id": self.group_id,
|
|
57
|
+
"budget": {"jobs": self.max_jobs, "minutes": self.max_minutes},
|
|
58
|
+
"reserve_minutes": self.reserve_minutes,
|
|
59
|
+
"used": vars(self.used),
|
|
60
|
+
"left": {"jobs": self.left_jobs, "minutes": self.left_minutes},
|
|
61
|
+
"more_jobs": self.more_jobs,
|
|
62
|
+
"fits": self.fits,
|
|
63
|
+
"by_msg": {prefix: vars(u) for prefix, u in self.by_msg.items()},
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _time(value: Optional[str]) -> Optional[datetime]:
|
|
68
|
+
return datetime.fromisoformat(value.replace("Z", "+00:00")) if value else None
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def job_minutes(job: Dict[str, Any], reserve: int, now: datetime) -> tuple[int, bool]:
|
|
72
|
+
"""A listed job's minutes, and whether it has finished."""
|
|
73
|
+
start, finish = _time(job.get("start_time")), _time(job.get("finish_time"))
|
|
74
|
+
if finish:
|
|
75
|
+
return (math.ceil((finish - start).total_seconds() / 60) if start else 0), True
|
|
76
|
+
elapsed = math.ceil((now - start).total_seconds() / 60) if start else 0
|
|
77
|
+
return max(elapsed, reserve), False
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def budget_usage(
|
|
81
|
+
api: Any,
|
|
82
|
+
group_id: str,
|
|
83
|
+
max_jobs: int,
|
|
84
|
+
max_minutes: int,
|
|
85
|
+
reserve: int = 30,
|
|
86
|
+
more_jobs: int = 0,
|
|
87
|
+
days_back: int = 365,
|
|
88
|
+
) -> BudgetUsage:
|
|
89
|
+
now = datetime.now(timezone.utc)
|
|
90
|
+
jobs: List[Dict[str, Any]] = api.data_fetcher.fetch_group_jobs(
|
|
91
|
+
group_id, (now - timedelta(days=days_back)).isoformat(), now.isoformat()
|
|
92
|
+
)
|
|
93
|
+
usage = BudgetUsage(group_id, max_jobs, max_minutes, reserve, Usage(), more_jobs)
|
|
94
|
+
for job in jobs:
|
|
95
|
+
minutes, final = job_minutes(job, reserve, now)
|
|
96
|
+
prefix = (job.get("msg") or "").split(" ", 1)[0] or "(no msg)"
|
|
97
|
+
for u in (usage.used, usage.by_msg.setdefault(prefix, Usage())):
|
|
98
|
+
u.jobs += 1
|
|
99
|
+
u.minutes += minutes
|
|
100
|
+
u.running += not final
|
|
101
|
+
usage.by_msg = dict(sorted(usage.by_msg.items(), key=lambda kv: -kv[1].minutes))
|
|
102
|
+
return usage
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def format_budget_text(u: BudgetUsage) -> str:
|
|
106
|
+
lines = [
|
|
107
|
+
f"group {u.group_id}",
|
|
108
|
+
f"budget {u.max_jobs} jobs, {u.max_minutes} min",
|
|
109
|
+
f"used {u.used.jobs} jobs, {u.used.minutes} min "
|
|
110
|
+
f"({u.used.running} running, counted at {u.reserve_minutes} min or their time so far)",
|
|
111
|
+
f"left {u.left_jobs} jobs, {u.left_minutes} min",
|
|
112
|
+
]
|
|
113
|
+
if u.more_jobs:
|
|
114
|
+
verdict = "fit" if u.fits else "do NOT fit"
|
|
115
|
+
lines.append(f"{u.more_jobs} more jobs (up to {u.more_jobs * u.reserve_minutes} min) {verdict}")
|
|
116
|
+
for prefix, p in u.by_msg.items():
|
|
117
|
+
lines.append(f" {prefix:44} {p.jobs:4} jobs {p.minutes:6} min {p.running:3} running")
|
|
118
|
+
return "\n".join(lines)
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 Certora Ltd.
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: GPL-3.0-only
|
|
4
|
+
|
|
5
|
+
"""
|
|
6
|
+
Render a violated rule's counterexample as compact XML.
|
|
7
|
+
|
|
8
|
+
A rule's raw output (``prover-cli --calltrace rule_output_<k>.json``) holds the failing assertion,
|
|
9
|
+
its source location, and the call trace as a tree of frames. The rendering keeps the frames a
|
|
10
|
+
reader needs, substitutes each frame's values into its message, and leaves out the frames every
|
|
11
|
+
trace of a chain carries.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import re
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from typing import Any, Dict, Optional
|
|
19
|
+
|
|
20
|
+
#: The prover writes a frame's message as a format template: literal text with ``'{n}'`` standing in
|
|
21
|
+
#: for the n-th of the message's ``arguments``.
|
|
22
|
+
_PLACEHOLDER = re.compile(r"'\{(\d+)\}'")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _frame(message: Dict[str, Any]) -> str:
|
|
26
|
+
"""A frame's identity: its message template with every value replaced by ``{}``.
|
|
27
|
+
|
|
28
|
+
A counterexample's values live in ``arguments``, never in ``text``, so frames emitted from the
|
|
29
|
+
same site share this string whatever values they took.
|
|
30
|
+
"""
|
|
31
|
+
return _PLACEHOLDER.sub("{}", message.get("text", ""))
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _render_message(message: Dict[str, Any]) -> str:
|
|
35
|
+
"""The message template with this counterexample's values substituted for its placeholders."""
|
|
36
|
+
arguments = message.get("arguments") or []
|
|
37
|
+
|
|
38
|
+
def value(placeholder: re.Match[str]) -> str:
|
|
39
|
+
index = int(placeholder.group(1))
|
|
40
|
+
return f"'{arguments[index].get('value')}'" if index < len(arguments) else placeholder.group(0)
|
|
41
|
+
|
|
42
|
+
return _PLACEHOLDER.sub(value, message.get("text", ""))
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True)
|
|
46
|
+
class TraceShape:
|
|
47
|
+
"""Which call-trace frames are worth rendering for one chain.
|
|
48
|
+
|
|
49
|
+
Frames are named by their message template (``{}`` where a value goes). ``dropped`` removes a
|
|
50
|
+
frame and everything under it. ``elided`` keeps the frame and replaces its subtree with a count.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
dropped: frozenset[str] = frozenset()
|
|
54
|
+
elided: frozenset[str] = frozenset()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
#: Frames every chain's traces carry and no counterexample needs.
|
|
58
|
+
_GENERIC_NOISE = frozenset({"Setup", "Global State"})
|
|
59
|
+
|
|
60
|
+
GENERIC_TRACE = TraceShape(dropped=_GENERIC_NOISE)
|
|
61
|
+
|
|
62
|
+
EVM_TRACE = TraceShape(
|
|
63
|
+
dropped=_GENERIC_NOISE | {"Evaluate branch condition", "unknown loop source code"},
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
#: Solana traces are dominated by account materialization under
|
|
67
|
+
#: ``cvlr_deserialize_nondet_accounts``. That subtree is elided rather than dropped, so a reader can
|
|
68
|
+
#: tell setup happened and how much of it was hidden. ``unknown loop source code`` is kept: a
|
|
69
|
+
#: loop-unwinding violation nests its failure under it. rustc 1.89 and later name the allocator shim
|
|
70
|
+
#: ``__rustc::__rust_alloc``, so both spellings are dropped.
|
|
71
|
+
SOLANA_TRACE = TraceShape(
|
|
72
|
+
dropped=_GENERIC_NOISE | {"__rust_alloc: {}", "__rustc::__rust_alloc: {}", "CVT_alloc_slice: {}"},
|
|
73
|
+
elided=frozenset({"cvlr_solana::layout::cvlr_deserialize_nondet_accounts(...)"}),
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
TRACE_SHAPES: Dict[str, TraceShape] = {
|
|
77
|
+
"evm": EVM_TRACE,
|
|
78
|
+
"solana": SOLANA_TRACE,
|
|
79
|
+
"soroban": GENERIC_TRACE,
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _descendants(node: Dict[str, Any]) -> int:
|
|
84
|
+
return sum(1 + _descendants(child) for child in node.get("childrenList") or [])
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def calltrace_to_xml(node: Dict[str, Any], shape: TraceShape) -> str:
|
|
88
|
+
"""Render one call-trace node and the children ``shape`` keeps."""
|
|
89
|
+
parts = [f"<message>{_render_message(node.get('message') or {})}</message>"]
|
|
90
|
+
for child in node.get("childrenList") or []:
|
|
91
|
+
message = child.get("message") or {}
|
|
92
|
+
frame = _frame(message)
|
|
93
|
+
if frame in shape.dropped:
|
|
94
|
+
continue
|
|
95
|
+
if frame in shape.elided:
|
|
96
|
+
parts.append(
|
|
97
|
+
f"<child><message>{_render_message(message)}</message>"
|
|
98
|
+
f"<elided>{_descendants(child)} frames of setup</elided></child>"
|
|
99
|
+
)
|
|
100
|
+
continue
|
|
101
|
+
parts.append(f"<child>{calltrace_to_xml(child, shape)}</child>")
|
|
102
|
+
return "".join(parts)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def render_counterexample(dump: Dict[str, Any], shape: TraceShape) -> Optional[str]:
|
|
106
|
+
"""One violated rule's counterexample, or None when its output carries no call trace.
|
|
107
|
+
|
|
108
|
+
``dump`` is the rule's raw output file (``assertMessage``, ``jumpToDefinition``,
|
|
109
|
+
``callTrace``). The result is ``<counterexample>`` with the failing ``<assert>``, its
|
|
110
|
+
``<source>`` as ``file:line``, and the rendered trace. ``shape`` has no default: another
|
|
111
|
+
chain's shape would silently drop frames this chain's traces need.
|
|
112
|
+
"""
|
|
113
|
+
trace = dump.get("callTrace")
|
|
114
|
+
if not isinstance(trace, dict):
|
|
115
|
+
return None
|
|
116
|
+
parts = []
|
|
117
|
+
assertion = dump.get("assertMessage")
|
|
118
|
+
if isinstance(assertion, str) and assertion:
|
|
119
|
+
parts.append(f"<assert>{assertion}</assert>")
|
|
120
|
+
location = dump.get("jumpToDefinition")
|
|
121
|
+
if isinstance(location, dict):
|
|
122
|
+
file, start = location.get("file"), location.get("start")
|
|
123
|
+
if isinstance(file, str) and isinstance(start, dict) and isinstance(start.get("line"), int):
|
|
124
|
+
parts.append(f"<source>{file}:{start['line']}</source>")
|
|
125
|
+
parts.append(calltrace_to_xml(trace, shape))
|
|
126
|
+
return "<counterexample>" + "".join(parts) + "</counterexample>"
|
|
@@ -62,6 +62,23 @@ class JobStatus(str, Enum):
|
|
|
62
62
|
# Unknown status (fallback for unrecognized values)
|
|
63
63
|
UNKNOWN = "UNKNOWN"
|
|
64
64
|
|
|
65
|
+
@property
|
|
66
|
+
def is_final(self) -> bool:
|
|
67
|
+
"""Whether the job has stopped: its verdicts are final, or it ended without a result."""
|
|
68
|
+
return self in _FINAL_JOB_STATUSES
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
_FINAL_JOB_STATUSES = frozenset(
|
|
72
|
+
{
|
|
73
|
+
JobStatus.SUCCEEDED,
|
|
74
|
+
JobStatus.FAILED,
|
|
75
|
+
JobStatus.CANCELED,
|
|
76
|
+
JobStatus.HALTED,
|
|
77
|
+
JobStatus.SERVICE_UNAVAILABLE,
|
|
78
|
+
JobStatus.UPLOAD_FAILED,
|
|
79
|
+
}
|
|
80
|
+
)
|
|
81
|
+
|
|
65
82
|
|
|
66
83
|
class NodeType(str, Enum):
|
|
67
84
|
"""Type of a tree view node."""
|
{certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/prover_cli.py
RENAMED
|
@@ -17,10 +17,17 @@ from pathlib import Path
|
|
|
17
17
|
|
|
18
18
|
from .api import ProverOutputAPI
|
|
19
19
|
from .breadcrumb import format_breadcrumbs_text
|
|
20
|
+
from .budget import budget_usage, format_budget_text
|
|
21
|
+
from .counterexample import TRACE_SHAPES
|
|
20
22
|
from .exceptions import AuthenticationError, JobNotFoundError, ProverAPIError
|
|
21
23
|
from .group_summary import build_group_summary, extract_group_id, format_text as format_group_text
|
|
22
24
|
from .models import CalltraceInfo, CheckResult, ParsedAlert
|
|
23
25
|
from .sighash import handle_sighash_command
|
|
26
|
+
from .verdicts import format_verdicts_text, job_verdicts
|
|
27
|
+
from .wait import format_wait_text, wait_for_jobs
|
|
28
|
+
|
|
29
|
+
#: Exit code of ``--budget --more-jobs K`` when K more jobs do not fit.
|
|
30
|
+
OVER_BUDGET = 3
|
|
24
31
|
|
|
25
32
|
|
|
26
33
|
def format_check_result(check_result: CheckResult, index: int) -> str:
|
|
@@ -126,8 +133,14 @@ Examples:
|
|
|
126
133
|
%(prog)s --job-id 12345 --full-report
|
|
127
134
|
%(prog)s --job-id 12345 --full-report --format json
|
|
128
135
|
%(prog)s --job-id 12345 --download-sources ./job_sources
|
|
136
|
+
%(prog)s --job-id 12345 --verdicts --save-report report.json
|
|
129
137
|
%(prog)s --cancel 12345 67890
|
|
130
138
|
|
|
139
|
+
# Waiting for jobs, and a budget over a group of jobs
|
|
140
|
+
%(prog)s --wait 12345 "https://prover.certora.com/output/..." --timeout 90
|
|
141
|
+
%(prog)s --budget --group-id GROUP --max-jobs 500 --max-minutes 5000 --more-jobs 3
|
|
142
|
+
%(prog)s --budget --budget-file budget.json --more-jobs 3
|
|
143
|
+
|
|
131
144
|
# Group summary
|
|
132
145
|
%(prog)s --group-id b78ea54a-a924-4f05-b1a7-58b29c1585ae
|
|
133
146
|
%(prog)s --group-id "https://prover.certora.com/?groupIds=b78ea54a-..." --format json
|
|
@@ -182,6 +195,13 @@ Environment Variables:
|
|
|
182
195
|
help="List recent jobs from the data API. "
|
|
183
196
|
"Use --days-back, --limit, --my-jobs-only to refine.",
|
|
184
197
|
)
|
|
198
|
+
job_group.add_argument(
|
|
199
|
+
"--wait",
|
|
200
|
+
nargs="+",
|
|
201
|
+
metavar="JOB",
|
|
202
|
+
help="Wait until these jobs (ids or links) have stopped, for at most --timeout seconds. "
|
|
203
|
+
"Prints one line per job and then ENDED or RUNNING.",
|
|
204
|
+
)
|
|
185
205
|
|
|
186
206
|
parser.add_argument("--output", "-o", help="Output JSON file path")
|
|
187
207
|
parser.add_argument("--status-only", action="store_true", help="Only get job status")
|
|
@@ -233,6 +253,65 @@ Environment Variables:
|
|
|
233
253
|
"individually in parallel; a completed fetch is skipped on rerun."
|
|
234
254
|
),
|
|
235
255
|
)
|
|
256
|
+
parser.add_argument(
|
|
257
|
+
"--verdicts",
|
|
258
|
+
action="store_true",
|
|
259
|
+
help="The job's verdicts rule by rule, from its leaf checks (one per method and per "
|
|
260
|
+
"assertion), with a rendered counterexample for every violation.",
|
|
261
|
+
)
|
|
262
|
+
parser.add_argument(
|
|
263
|
+
"--save-report",
|
|
264
|
+
metavar="FILE",
|
|
265
|
+
help="Used with --verdicts: also write the verdicts, counterexamples and full job report "
|
|
266
|
+
"to FILE as JSON, and still print the verdicts.",
|
|
267
|
+
)
|
|
268
|
+
parser.add_argument(
|
|
269
|
+
"--chain",
|
|
270
|
+
choices=sorted(TRACE_SHAPES),
|
|
271
|
+
default="evm",
|
|
272
|
+
help="Which frames a counterexample keeps (default: evm). Used with --verdicts.",
|
|
273
|
+
)
|
|
274
|
+
parser.add_argument(
|
|
275
|
+
"--timeout",
|
|
276
|
+
type=float,
|
|
277
|
+
default=90,
|
|
278
|
+
help="Seconds --wait waits at most (default: 90).",
|
|
279
|
+
)
|
|
280
|
+
parser.add_argument(
|
|
281
|
+
"--interval",
|
|
282
|
+
type=float,
|
|
283
|
+
default=30,
|
|
284
|
+
help="Seconds between checks while --wait waits (default: 30).",
|
|
285
|
+
)
|
|
286
|
+
parser.add_argument(
|
|
287
|
+
"--budget",
|
|
288
|
+
action="store_true",
|
|
289
|
+
help="Prover jobs and minutes used by the cloud jobs of a group (--group-id, or "
|
|
290
|
+
"group_id in --budget-file), against --max-jobs/--max-minutes or the budget file. "
|
|
291
|
+
f"With --more-jobs K, exits {OVER_BUDGET} when K more jobs do not fit.",
|
|
292
|
+
)
|
|
293
|
+
parser.add_argument(
|
|
294
|
+
"--budget-file",
|
|
295
|
+
metavar="FILE",
|
|
296
|
+
help='Used with --budget: a JSON file {"jobs": N, "minutes": M, "group_id": G}; flags '
|
|
297
|
+
"given on the command line take precedence.",
|
|
298
|
+
)
|
|
299
|
+
parser.add_argument("--max-jobs", type=int, help="Job budget for --budget.")
|
|
300
|
+
parser.add_argument("--max-minutes", type=int, help="Minute budget for --budget.")
|
|
301
|
+
parser.add_argument(
|
|
302
|
+
"--reserve-minutes",
|
|
303
|
+
type=int,
|
|
304
|
+
default=30,
|
|
305
|
+
help="Minutes --budget counts for a job that has not finished (or its time so far, if "
|
|
306
|
+
"larger), and for each of --more-jobs (default: 30).",
|
|
307
|
+
)
|
|
308
|
+
parser.add_argument(
|
|
309
|
+
"--more-jobs",
|
|
310
|
+
type=int,
|
|
311
|
+
default=0,
|
|
312
|
+
metavar="K",
|
|
313
|
+
help=f"Used with --budget: exit {OVER_BUDGET} unless K more jobs fit.",
|
|
314
|
+
)
|
|
236
315
|
parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")
|
|
237
316
|
parser.add_argument("--format", choices=["json", "text"], default="text", help="Output format")
|
|
238
317
|
parser.add_argument(
|
|
@@ -334,8 +413,8 @@ Environment Variables:
|
|
|
334
413
|
)
|
|
335
414
|
sys.exit(1)
|
|
336
415
|
|
|
337
|
-
# Handle --group-id (after auth check, doesn't need job input)
|
|
338
|
-
if args.group_id:
|
|
416
|
+
# Handle --group-id (after auth check, doesn't need job input; --budget reads it itself)
|
|
417
|
+
if args.group_id and not args.budget:
|
|
339
418
|
try:
|
|
340
419
|
group_id = extract_group_id(args.group_id)
|
|
341
420
|
api = ProverOutputAPI(use_local=False)
|
|
@@ -402,10 +481,65 @@ Environment Variables:
|
|
|
402
481
|
print(f"❌ API error: {e}", file=sys.stderr)
|
|
403
482
|
sys.exit(1)
|
|
404
483
|
|
|
484
|
+
# Handle --wait (after auth check)
|
|
485
|
+
if args.wait:
|
|
486
|
+
try:
|
|
487
|
+
api = ProverOutputAPI(use_local=False)
|
|
488
|
+
result = wait_for_jobs(api, args.wait, timeout=args.timeout, interval=args.interval)
|
|
489
|
+
if args.output:
|
|
490
|
+
with open(args.output, "w") as f:
|
|
491
|
+
json.dump(result.to_dict(), f, indent=2)
|
|
492
|
+
print(f"{'ENDED' if result.ended else 'RUNNING'}. Results written to {args.output}")
|
|
493
|
+
elif args.format == "json":
|
|
494
|
+
print(json.dumps(result.to_dict(), indent=2))
|
|
495
|
+
else:
|
|
496
|
+
print(format_wait_text(result))
|
|
497
|
+
return
|
|
498
|
+
except AuthenticationError as e:
|
|
499
|
+
print(f"❌ Authentication failed: {e}", file=sys.stderr)
|
|
500
|
+
sys.exit(1)
|
|
501
|
+
|
|
502
|
+
# Handle --budget (after auth check; reads the group's jobs from the data API)
|
|
503
|
+
if args.budget:
|
|
504
|
+
limits = json.loads(Path(args.budget_file).read_text()) if args.budget_file else {}
|
|
505
|
+
group_id = args.group_id or limits.get("group_id")
|
|
506
|
+
max_jobs = args.max_jobs if args.max_jobs is not None else limits.get("jobs")
|
|
507
|
+
max_minutes = args.max_minutes if args.max_minutes is not None else limits.get("minutes")
|
|
508
|
+
if not group_id or max_jobs is None or max_minutes is None:
|
|
509
|
+
parser.error("--budget needs a group id and both limits, from flags or --budget-file")
|
|
510
|
+
try:
|
|
511
|
+
api = ProverOutputAPI(use_local=False)
|
|
512
|
+
usage = budget_usage(
|
|
513
|
+
api,
|
|
514
|
+
extract_group_id(group_id),
|
|
515
|
+
max_jobs,
|
|
516
|
+
max_minutes,
|
|
517
|
+
reserve=args.reserve_minutes,
|
|
518
|
+
more_jobs=args.more_jobs,
|
|
519
|
+
days_back=args.days_back,
|
|
520
|
+
)
|
|
521
|
+
except AuthenticationError as e:
|
|
522
|
+
print(f"❌ Authentication failed: {e}", file=sys.stderr)
|
|
523
|
+
sys.exit(1)
|
|
524
|
+
except ProverAPIError as e:
|
|
525
|
+
print(f"❌ API error: {e}", file=sys.stderr)
|
|
526
|
+
sys.exit(1)
|
|
527
|
+
if args.output:
|
|
528
|
+
with open(args.output, "w") as f:
|
|
529
|
+
json.dump(usage.to_dict(), f, indent=2)
|
|
530
|
+
print(f"{usage.left_jobs} jobs, {usage.left_minutes} min left. Results written to {args.output}")
|
|
531
|
+
elif args.format == "json":
|
|
532
|
+
print(json.dumps(usage.to_dict(), indent=2))
|
|
533
|
+
else:
|
|
534
|
+
print(format_budget_text(usage))
|
|
535
|
+
if args.more_jobs and not usage.fits:
|
|
536
|
+
sys.exit(OVER_BUDGET)
|
|
537
|
+
return
|
|
538
|
+
|
|
405
539
|
# Check that we have a job input (unless it's who-am-i or sighash which were handled above)
|
|
406
540
|
if not args.cancel and not (args.job_id or args.job_url or args.local_path):
|
|
407
541
|
parser.error(
|
|
408
|
-
"Please provide either --job-id, --job-url, --local-path, --cancel, --group-id, --sighash, --recent-jobs, or --who-am-i"
|
|
542
|
+
"Please provide either --job-id, --job-url, --local-path, --cancel, --group-id, --sighash, --recent-jobs, --wait, --budget, or --who-am-i"
|
|
409
543
|
)
|
|
410
544
|
|
|
411
545
|
# Handle cancel case differently since it takes multiple inputs
|
|
@@ -433,6 +567,16 @@ Environment Variables:
|
|
|
433
567
|
"result": cancel_result,
|
|
434
568
|
}
|
|
435
569
|
|
|
570
|
+
elif args.verdicts:
|
|
571
|
+
verdicts = job_verdicts(api, job_input, chain=args.chain)
|
|
572
|
+
result = verdicts.to_dict()
|
|
573
|
+
if args.save_report:
|
|
574
|
+
with open(args.save_report, "w") as f:
|
|
575
|
+
json.dump(result, f, indent=1)
|
|
576
|
+
if not args.output and args.format != "json":
|
|
577
|
+
print(format_verdicts_text(verdicts))
|
|
578
|
+
return
|
|
579
|
+
|
|
436
580
|
elif args.status_only:
|
|
437
581
|
# Just get job status
|
|
438
582
|
status = api.get_job_status(job_input)
|
|
@@ -557,6 +701,8 @@ Environment Variables:
|
|
|
557
701
|
print(f"Found {count} alerts. Results written to {args.output}")
|
|
558
702
|
elif args.full_report:
|
|
559
703
|
print(f"✅ Full report retrieved. Results written to {args.output}")
|
|
704
|
+
elif args.verdicts:
|
|
705
|
+
print(f"Verdicts for {len(result['rules'])} rules. Results written to {args.output}")
|
|
560
706
|
elif args.download_sources:
|
|
561
707
|
print(
|
|
562
708
|
f"Downloaded {result['files_count']} source files to {result['sources_dir']}. "
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 Certora Ltd.
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: GPL-3.0-only
|
|
4
|
+
|
|
5
|
+
"""
|
|
6
|
+
A job's verdicts, rule by rule, with a counterexample for every violation.
|
|
7
|
+
|
|
8
|
+
The verdicts are the job's leaf checks: one per method of a parametric rule or invariant, and one
|
|
9
|
+
per assertion. A rule's top-level node only summarizes them. It can read ERROR while one of its
|
|
10
|
+
methods is VIOLATED, or VIOLATED while other methods are SANITY_FAILED, and it carries no
|
|
11
|
+
counterexample file, so the leaves are what to read.
|
|
12
|
+
|
|
13
|
+
A failed vacuity check (``rule_not_vacuous``, status SANITY_FAILED) is a verdict too. For a method
|
|
14
|
+
of a parametric rule the prover also marks the method SANITY_FAILED, but for a rule split into
|
|
15
|
+
sub-checks (a ``satisfy``, auto-generated assertions) the vacuity check is the only place it shows,
|
|
16
|
+
so it is added as a leaf of its own wherever its rule or method is not already SANITY_FAILED.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from collections import Counter
|
|
22
|
+
from dataclasses import dataclass, field
|
|
23
|
+
from typing import Any, Dict, List, Optional
|
|
24
|
+
|
|
25
|
+
from .counterexample import TRACE_SHAPES, render_counterexample
|
|
26
|
+
from .job_report import JobReport
|
|
27
|
+
from .models import CheckResult, NodeStatus, NodeType
|
|
28
|
+
|
|
29
|
+
VACUITY_CHECK = "rule_not_vacuous"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _failed_vacuity_checks(api: Any, job_input: str, leaves: List[CheckResult]) -> List[CheckResult]:
|
|
33
|
+
"""The failed vacuity checks no leaf already reports as SANITY_FAILED."""
|
|
34
|
+
reported = {(c.rule_name, c.method_name) for c in leaves if c.status == NodeStatus.SANITY_FAILED}
|
|
35
|
+
return [
|
|
36
|
+
c
|
|
37
|
+
for c in api.get_all_checks(job_input, include_rule_not_vacuous=True)
|
|
38
|
+
if c.node_type == NodeType.SANITY
|
|
39
|
+
and c.assert_message == VACUITY_CHECK
|
|
40
|
+
and c.status == NodeStatus.SANITY_FAILED
|
|
41
|
+
and (c.rule_name, c.method_name) not in reported
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass
|
|
46
|
+
class RuleVerdicts:
|
|
47
|
+
"""One rule's leaf statuses counted, and the leaves worth naming: none when every leaf is
|
|
48
|
+
VERIFIED; otherwise every leaf that is not, plus the VERIFIED ones that are not methods (a
|
|
49
|
+
rule's sub-checks, which the counts alone would leave unnamed)."""
|
|
50
|
+
|
|
51
|
+
counts: Dict[str, int] = field(default_factory=dict)
|
|
52
|
+
leaves: List[CheckResult] = field(default_factory=list)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass
|
|
56
|
+
class RenderedCounterexample:
|
|
57
|
+
rule: str
|
|
58
|
+
method: Optional[str]
|
|
59
|
+
output_file: str
|
|
60
|
+
counterexample: str
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass
|
|
64
|
+
class JobVerdicts:
|
|
65
|
+
"""A job's verdicts per rule, its rendered counterexamples, and its full report."""
|
|
66
|
+
|
|
67
|
+
job_id: str
|
|
68
|
+
report: JobReport
|
|
69
|
+
rules: Dict[str, RuleVerdicts] = field(default_factory=dict)
|
|
70
|
+
counterexamples: List[RenderedCounterexample] = field(default_factory=list)
|
|
71
|
+
|
|
72
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
73
|
+
return {
|
|
74
|
+
"job_id": self.job_id,
|
|
75
|
+
"job_status": self.report.job_status.value,
|
|
76
|
+
"rules": {
|
|
77
|
+
name: {
|
|
78
|
+
"counts": r.counts,
|
|
79
|
+
"leaves": [
|
|
80
|
+
{"method": c.method_name, "status": c.status.value, "assert": c.assert_message}
|
|
81
|
+
for c in r.leaves
|
|
82
|
+
],
|
|
83
|
+
}
|
|
84
|
+
for name, r in self.rules.items()
|
|
85
|
+
},
|
|
86
|
+
"counterexamples": [vars(c) for c in self.counterexamples],
|
|
87
|
+
"report": self.report.to_dict(),
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def job_verdicts(api: Any, job_input: str, chain: str) -> JobVerdicts:
|
|
92
|
+
"""Read the job's leaf checks and render the counterexample of every violated one."""
|
|
93
|
+
shape = TRACE_SHAPES[chain]
|
|
94
|
+
leaves: List[CheckResult] = api.get_leaf_checks(job_input)
|
|
95
|
+
leaves += _failed_vacuity_checks(api, job_input, leaves)
|
|
96
|
+
result = JobVerdicts(job_id=job_input, report=api.get_job_report(job_input))
|
|
97
|
+
|
|
98
|
+
by_rule: Dict[str, List[CheckResult]] = {}
|
|
99
|
+
for leaf in leaves:
|
|
100
|
+
by_rule.setdefault(leaf.rule_name, []).append(leaf)
|
|
101
|
+
for name, rule_leaves in by_rule.items():
|
|
102
|
+
failed = any(not c.is_verified for c in rule_leaves)
|
|
103
|
+
result.rules[name] = RuleVerdicts(
|
|
104
|
+
counts=dict(Counter(c.status.value for c in rule_leaves).most_common()),
|
|
105
|
+
leaves=[c for c in rule_leaves if not c.is_verified or (failed and not c.method_name)],
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
seen = set()
|
|
109
|
+
for leaf in leaves:
|
|
110
|
+
if not leaf.is_violated:
|
|
111
|
+
continue
|
|
112
|
+
for name in leaf.output_files or []:
|
|
113
|
+
if name in seen:
|
|
114
|
+
continue
|
|
115
|
+
seen.add(name)
|
|
116
|
+
dump = api.get_calltrace(job_input, name).trace_data
|
|
117
|
+
rendered = render_counterexample(dump, shape) if isinstance(dump, dict) else None
|
|
118
|
+
if rendered:
|
|
119
|
+
result.counterexamples.append(
|
|
120
|
+
RenderedCounterexample(leaf.rule_name, leaf.method_name, name, rendered)
|
|
121
|
+
)
|
|
122
|
+
return result
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def format_verdicts_text(verdicts: JobVerdicts) -> str:
|
|
126
|
+
"""One line per rule with its leaf statuses counted, a line per listed leaf, then each
|
|
127
|
+
counterexample."""
|
|
128
|
+
lines = [f"job {verdicts.report.job_status.value}"]
|
|
129
|
+
for name, rule in verdicts.rules.items():
|
|
130
|
+
lines.append(name + "\t" + ", ".join(f"{s} {n}" for s, n in rule.counts.items()))
|
|
131
|
+
for leaf in rule.leaves:
|
|
132
|
+
where = leaf.method_name or leaf.assert_message or ""
|
|
133
|
+
if where and where != name:
|
|
134
|
+
lines.append(f"\t{where}\t{leaf.status.value}")
|
|
135
|
+
for cex in verdicts.counterexamples:
|
|
136
|
+
lines.append("")
|
|
137
|
+
lines.append(f"== {cex.rule} {cex.method or ''}".rstrip())
|
|
138
|
+
lines.append(cex.counterexample)
|
|
139
|
+
return "\n".join(lines)
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 Certora Ltd.
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: GPL-3.0-only
|
|
4
|
+
|
|
5
|
+
"""
|
|
6
|
+
Wait, for a bounded time, until a set of prover jobs has stopped.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import time
|
|
12
|
+
from dataclasses import asdict, dataclass, field
|
|
13
|
+
from typing import Any, Dict, List
|
|
14
|
+
|
|
15
|
+
#: The status of a job whose status request failed. It does not end the wait.
|
|
16
|
+
CHECK_FAILED = "CHECK_FAILED"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class WaitRow:
|
|
21
|
+
"""One job, as given (id or link), with its status."""
|
|
22
|
+
|
|
23
|
+
target: str
|
|
24
|
+
status: str
|
|
25
|
+
final: bool
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class WaitResult:
|
|
30
|
+
ended: bool
|
|
31
|
+
jobs: List[WaitRow] = field(default_factory=list)
|
|
32
|
+
|
|
33
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
34
|
+
return {"ended": self.ended, "jobs": [asdict(r) for r in self.jobs]}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def check_jobs(api: Any, jobs: List[str]) -> List[WaitRow]:
|
|
38
|
+
"""One row per job id or job link."""
|
|
39
|
+
rows = []
|
|
40
|
+
for job in jobs:
|
|
41
|
+
try:
|
|
42
|
+
status = api.get_job_status(job)
|
|
43
|
+
rows.append(WaitRow(job, status.value, status.is_final))
|
|
44
|
+
except Exception:
|
|
45
|
+
rows.append(WaitRow(job, CHECK_FAILED, False))
|
|
46
|
+
return rows
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def wait_for_jobs(api: Any, jobs: List[str], timeout: float, interval: float = 30) -> WaitResult:
|
|
50
|
+
"""Check the jobs every ``interval`` seconds until every one has stopped or ``timeout`` seconds
|
|
51
|
+
have passed, with a last check at the deadline. ``ended`` says which."""
|
|
52
|
+
deadline = time.monotonic() + timeout
|
|
53
|
+
while True:
|
|
54
|
+
rows = check_jobs(api, jobs)
|
|
55
|
+
ended = all(r.final for r in rows)
|
|
56
|
+
remaining = deadline - time.monotonic()
|
|
57
|
+
if ended or remaining <= 0:
|
|
58
|
+
return WaitResult(ended=ended, jobs=rows)
|
|
59
|
+
time.sleep(min(interval, remaining))
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def format_wait_text(result: WaitResult) -> str:
|
|
63
|
+
lines = [f"{r.target} {r.status}" for r in result.jobs]
|
|
64
|
+
lines.append("ENDED" if result.ended else "RUNNING")
|
|
65
|
+
return "\n".join(lines)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/cache.py
RENAMED
|
File without changes
|
{certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/data_fetcher.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/tree_parser.py
RENAMED
|
File without changes
|
{certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/api/url_utils.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/breadcrumb.py
RENAMED
|
File without changes
|
{certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/exceptions.py
RENAMED
|
File without changes
|
{certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/group_summary.py
RENAMED
|
File without changes
|
{certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/job_report.py
RENAMED
|
File without changes
|
|
File without changes
|
{certora_prover_cli-0.2.5 → certora_prover_cli-0.2.6}/src/prover_output_utility/prover_api.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|