verovo 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- verovo-0.11.0/PKG-INFO +427 -0
- verovo-0.11.0/README.md +410 -0
- verovo-0.11.0/pyproject.toml +47 -0
- verovo-0.11.0/setup.cfg +4 -0
- verovo-0.11.0/tests/test_checks.py +289 -0
- verovo-0.11.0/tests/test_claude_adapter.py +120 -0
- verovo-0.11.0/tests/test_cli.py +138 -0
- verovo-0.11.0/tests/test_codex_adapter.py +103 -0
- verovo-0.11.0/tests/test_commerce.py +357 -0
- verovo-0.11.0/tests/test_commerce_v2.py +410 -0
- verovo-0.11.0/tests/test_console.py +306 -0
- verovo-0.11.0/tests/test_contract_lane.py +195 -0
- verovo-0.11.0/tests/test_cost_optimization.py +104 -0
- verovo-0.11.0/tests/test_dispute.py +146 -0
- verovo-0.11.0/tests/test_drilldown_decisions.py +268 -0
- verovo-0.11.0/tests/test_export.py +127 -0
- verovo-0.11.0/tests/test_history.py +160 -0
- verovo-0.11.0/tests/test_input_inspect.py +80 -0
- verovo-0.11.0/tests/test_invoice_intake.py +55 -0
- verovo-0.11.0/tests/test_mcp_server.py +76 -0
- verovo-0.11.0/tests/test_outcome_schema.py +123 -0
- verovo-0.11.0/tests/test_parity.py +103 -0
- verovo-0.11.0/tests/test_period_intake.py +83 -0
- verovo-0.11.0/tests/test_pipeline_edges.py +262 -0
- verovo-0.11.0/tests/test_prepare.py +223 -0
- verovo-0.11.0/tests/test_productwave.py +420 -0
- verovo-0.11.0/tests/test_reconcile_period.py +45 -0
- verovo-0.11.0/tests/test_recount_evidence_headline.py +14 -0
- verovo-0.11.0/tests/test_recovery.py +187 -0
- verovo-0.11.0/tests/test_response_audit.py +58 -0
- verovo-0.11.0/tests/test_review.py +160 -0
- verovo-0.11.0/tests/test_review_regressions.py +295 -0
- verovo-0.11.0/tests/test_schema_and_example.py +272 -0
- verovo-0.11.0/tests/test_sdk_events.py +107 -0
- verovo-0.11.0/tests/test_settle.py +490 -0
- verovo-0.11.0/tests/test_share.py +82 -0
- verovo-0.11.0/tests/test_showcase.py +69 -0
- verovo-0.11.0/tests/test_snapshot_and_filters.py +223 -0
- verovo-0.11.0/tests/test_task_economics.py +200 -0
- verovo-0.11.0/tests/test_vendors.py +297 -0
- verovo-0.11.0/tests/test_verify_pipeline.py +369 -0
- verovo-0.11.0/verovo.egg-info/PKG-INFO +427 -0
- verovo-0.11.0/verovo.egg-info/SOURCES.txt +123 -0
- verovo-0.11.0/verovo.egg-info/dependency_links.txt +1 -0
- verovo-0.11.0/verovo.egg-info/entry_points.txt +3 -0
- verovo-0.11.0/verovo.egg-info/top_level.txt +1 -0
- verovo-0.11.0/verovo_healthcheck/__init__.py +17 -0
- verovo-0.11.0/verovo_healthcheck/audit.py +148 -0
- verovo-0.11.0/verovo_healthcheck/benchmark.py +156 -0
- verovo-0.11.0/verovo_healthcheck/certify.py +157 -0
- verovo-0.11.0/verovo_healthcheck/checks.py +953 -0
- verovo-0.11.0/verovo_healthcheck/claude.py +286 -0
- verovo-0.11.0/verovo_healthcheck/cli.py +2502 -0
- verovo-0.11.0/verovo_healthcheck/codex.py +424 -0
- verovo-0.11.0/verovo_healthcheck/commerce.py +439 -0
- verovo-0.11.0/verovo_healthcheck/console.py +1091 -0
- verovo-0.11.0/verovo_healthcheck/cost_drivers.py +193 -0
- verovo-0.11.0/verovo_healthcheck/crm.py +137 -0
- verovo-0.11.0/verovo_healthcheck/crosscheck.py +208 -0
- verovo-0.11.0/verovo_healthcheck/dashboard.py +394 -0
- verovo-0.11.0/verovo_healthcheck/decisions.py +126 -0
- verovo-0.11.0/verovo_healthcheck/delivery.py +186 -0
- verovo-0.11.0/verovo_healthcheck/demo/buyer-observations.json +5 -0
- verovo-0.11.0/verovo_healthcheck/demo/buyer-review.json +39 -0
- verovo-0.11.0/verovo_healthcheck/demo/demo-codex.jsonl +26 -0
- verovo-0.11.0/verovo_healthcheck/demo/sample-recount-export.csv +6 -0
- verovo-0.11.0/verovo_healthcheck/discover.py +186 -0
- verovo-0.11.0/verovo_healthcheck/dispute.py +388 -0
- verovo-0.11.0/verovo_healthcheck/drilldown.py +216 -0
- verovo-0.11.0/verovo_healthcheck/export.py +500 -0
- verovo-0.11.0/verovo_healthcheck/fixtures/claude-duplicates.jsonl +4 -0
- verovo-0.11.0/verovo_healthcheck/fixtures/claude-empty.jsonl +0 -0
- verovo-0.11.0/verovo_healthcheck/fixtures/claude-ok.jsonl +9 -0
- verovo-0.11.0/verovo_healthcheck/fixtures/claude-retries.jsonl +10 -0
- verovo-0.11.0/verovo_healthcheck/fixtures/codex-cache.jsonl +11 -0
- verovo-0.11.0/verovo_healthcheck/fixtures/codex-corrupt.jsonl +10 -0
- verovo-0.11.0/verovo_healthcheck/fixtures/codex-duplicates.jsonl +9 -0
- verovo-0.11.0/verovo_healthcheck/fixtures/codex-empty.jsonl +0 -0
- verovo-0.11.0/verovo_healthcheck/fixtures/codex-ok.jsonl +13 -0
- verovo-0.11.0/verovo_healthcheck/fixtures/codex-resolution.jsonl +13 -0
- verovo-0.11.0/verovo_healthcheck/fixtures/codex-retries.jsonl +15 -0
- verovo-0.11.0/verovo_healthcheck/fixtures/codex-stability.jsonl +10 -0
- verovo-0.11.0/verovo_healthcheck/fixtures/vendor-export-intercom.csv +11 -0
- verovo-0.11.0/verovo_healthcheck/history.py +182 -0
- verovo-0.11.0/verovo_healthcheck/incrementality.py +149 -0
- verovo-0.11.0/verovo_healthcheck/input_inspect.py +76 -0
- verovo-0.11.0/verovo_healthcheck/invoice_intake.py +94 -0
- verovo-0.11.0/verovo_healthcheck/jsonl.py +53 -0
- verovo-0.11.0/verovo_healthcheck/lab/__init__.py +13 -0
- verovo-0.11.0/verovo_healthcheck/lab/analysis.py +504 -0
- verovo-0.11.0/verovo_healthcheck/lab/prereg.py +20 -0
- verovo-0.11.0/verovo_healthcheck/lab/schemas/funnel-event.schema.json +70 -0
- verovo-0.11.0/verovo_healthcheck/lab/schemas.py +115 -0
- verovo-0.11.0/verovo_healthcheck/lab/stats.py +96 -0
- verovo-0.11.0/verovo_healthcheck/lab/value.py +78 -0
- verovo-0.11.0/verovo_healthcheck/ledger.py +247 -0
- verovo-0.11.0/verovo_healthcheck/mcp_server.py +238 -0
- verovo-0.11.0/verovo_healthcheck/model.py +187 -0
- verovo-0.11.0/verovo_healthcheck/narrative.py +130 -0
- verovo-0.11.0/verovo_healthcheck/optimization.py +279 -0
- verovo-0.11.0/verovo_healthcheck/outcomes.py +231 -0
- verovo-0.11.0/verovo_healthcheck/pack.py +350 -0
- verovo-0.11.0/verovo_healthcheck/pack_invariants.json +52 -0
- verovo-0.11.0/verovo_healthcheck/period_intake.py +443 -0
- verovo-0.11.0/verovo_healthcheck/periods.py +118 -0
- verovo-0.11.0/verovo_healthcheck/pnl.py +280 -0
- verovo-0.11.0/verovo_healthcheck/policy.py +184 -0
- verovo-0.11.0/verovo_healthcheck/prepare.py +190 -0
- verovo-0.11.0/verovo_healthcheck/recovery.py +240 -0
- verovo-0.11.0/verovo_healthcheck/report_html.py +282 -0
- verovo-0.11.0/verovo_healthcheck/report_text.py +99 -0
- verovo-0.11.0/verovo_healthcheck/response_audit.py +90 -0
- verovo-0.11.0/verovo_healthcheck/review.py +280 -0
- verovo-0.11.0/verovo_healthcheck/rulesdiff.py +117 -0
- verovo-0.11.0/verovo_healthcheck/scaffold.py +146 -0
- verovo-0.11.0/verovo_healthcheck/schema.py +201 -0
- verovo-0.11.0/verovo_healthcheck/sdk_events.py +89 -0
- verovo-0.11.0/verovo_healthcheck/settle.py +857 -0
- verovo-0.11.0/verovo_healthcheck/share.py +128 -0
- verovo-0.11.0/verovo_healthcheck/snapshot.py +250 -0
- verovo-0.11.0/verovo_healthcheck/task_economics.py +484 -0
- verovo-0.11.0/verovo_healthcheck/taxonomy.py +170 -0
- verovo-0.11.0/verovo_healthcheck/templates.py +128 -0
- verovo-0.11.0/verovo_healthcheck/vendor-rules.json +212 -0
- verovo-0.11.0/verovo_healthcheck/vendors.py +495 -0
verovo-0.11.0/PKG-INFO
ADDED
|
@@ -0,0 +1,427 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: verovo
|
|
3
|
+
Version: 0.11.0
|
|
4
|
+
Summary: Buyer-side AI service charge and delivery review, supported bill recounts and local agent diagnostics. Zero external dependencies.
|
|
5
|
+
License: MIT
|
|
6
|
+
Keywords: agent,codex,claude,observability,measurement,healthcheck,conformance
|
|
7
|
+
Classifier: Development Status :: 3 - Alpha
|
|
8
|
+
Classifier: Environment :: Console
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
15
|
+
Requires-Python: >=3.9
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
# Verovo local CLI
|
|
19
|
+
|
|
20
|
+
Buyer-side tools for AI service charges and delivery, with local developer diagnostics.
|
|
21
|
+
The **`review`** command evaluates separate charge and delivery criteria using
|
|
22
|
+
local JSON evidence. It is part of the v0.10.0 release tag.
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
python3 healthcheck/verovo review --sample --html buyer-review.html
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Run from the repository root. [Buyer task guide](../docs/buyer-review.md) ·
|
|
29
|
+
[Product definition](../docs/product-definition.md).
|
|
30
|
+
|
|
31
|
+
## Developer Healthcheck
|
|
32
|
+
|
|
33
|
+
**A local check-up report for your coding agent runs.**
|
|
34
|
+
Point it at the logs your agent runtime already writes. It answers three
|
|
35
|
+
questions with evidence — *which records are duplicated, where did the agent
|
|
36
|
+
pay retries for one logical operation, and where did the same tool fail over
|
|
37
|
+
and over* — and writes a terminal summary plus a local HTML report.
|
|
38
|
+
|
|
39
|
+
Same honesty rules as the [conformance pack](../conformance/pack/README.md):
|
|
40
|
+
every verdict is `OK / FINDING / UNPROVABLE`, and UNPROVABLE is a first-class
|
|
41
|
+
result — when the logs cannot decide, that is disclosed, never zeroed.
|
|
42
|
+
|
|
43
|
+
> Status 2026-11-01: CLI **v0.10.0** is tagged on GitHub; the PyPI project `verovo`
|
|
44
|
+
> is not published yet, so `pip install verovo` fails until the first upload (the
|
|
45
|
+
> previous `agentmeasure` project stays at 0.8.0). Use the pinned Git install below
|
|
46
|
+
> in the meantime. Provider SDK is **v0.3.0** (`@verovo/mcp`). 563 Healthcheck tests
|
|
47
|
+
> passed locally. See [release notes](https://roy-tong.github.io/Verovo/releases.html).
|
|
48
|
+
|
|
49
|
+
## Install
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
# Pinned Git install of 0.10.0 (works today; the PyPI project `verovo` is not published yet).
|
|
53
|
+
pipx install "git+https://github.com/roy-tong/verovo@v0.10.0#subdirectory=healthcheck"
|
|
54
|
+
verovo demo
|
|
55
|
+
|
|
56
|
+
# Once `verovo` is on PyPI this is the normal path.
|
|
57
|
+
pip install verovo # or: pipx install verovo
|
|
58
|
+
|
|
59
|
+
# Or run from a repository checkout (use a release tag for reproducibility).
|
|
60
|
+
python3 healthcheck/verovo demo
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Historical note: the 0.4.0 wheel (2026-09) had a known first-command crash.
|
|
64
|
+
The release pipeline builds and smoke-tests the packaged wheel. For the renamed
|
|
65
|
+
Verovo project, use the Git installation above until its first PyPI publication.
|
|
66
|
+
|
|
67
|
+
Python 3.9+ standard library only; zero runtime dependencies; no network code
|
|
68
|
+
(a test enforces both).
|
|
69
|
+
|
|
70
|
+
The 0.7.0 wheel's conformance command is broken because a bundled dependency was
|
|
71
|
+
omitted. **Upgrade to 0.7.2 or later** for `conformance`; Healthcheck itself was unaffected.
|
|
72
|
+
[Installed conformance quickstart](../conformance/pack/README.md#first-run-from-an-installed-cli).
|
|
73
|
+
|
|
74
|
+
[Installation and error recovery](https://roy-tong.github.io/Verovo/#install-help)
|
|
75
|
+
includes Windows and virtual environments. The report path is printed after a
|
|
76
|
+
successful check. Keep full JSON/HTML local; only share an inspected summary.
|
|
77
|
+
|
|
78
|
+
## Quick start
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
verovo demo # synthetic session — see it work, no data needed
|
|
82
|
+
verovo check # your agent logs, last 7 days (Codex or Claude Code)
|
|
83
|
+
verovo check --all # every local session
|
|
84
|
+
verovo check --runtime claude # force the Claude Code adapter
|
|
85
|
+
verovo check --dir ~/.codex/sessions/2026/09/05
|
|
86
|
+
verovo selftest # adapters + checks on bundled fixtures
|
|
87
|
+
verovo validate export.json # check an export against its schema
|
|
88
|
+
verovo share run.json # PREVIEW the sanitized summary (writes nothing)
|
|
89
|
+
verovo share run.json --out summary.md # export after review
|
|
90
|
+
verovo --version
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
(Without installing, prefix the commands with `python3 healthcheck/verovo`.)
|
|
94
|
+
|
|
95
|
+
Options for `check` / `demo`: `--html PATH` (default `verovo-report.html`),
|
|
96
|
+
`--json PATH` (full machine-readable export), `--share PATH` (sanitized summary,
|
|
97
|
+
`.md` or `.json`), `--days N`, `--save-snapshot [PATH]`, `--no-history`.
|
|
98
|
+
`demo` accepts the same output paths and `--no-history`.
|
|
99
|
+
|
|
100
|
+
Input filters for `check` and `compare`:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
--since 2026-09-01 --until 2026-09-05 # date range (overrides --days)
|
|
104
|
+
--project myrepo # keep sessions whose project matches
|
|
105
|
+
# (project = cwd basename, case-insensitive)
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
## Outcome-billing verification: prepare → recount → dispute → recovery
|
|
109
|
+
|
|
110
|
+
For a hands-on first run, I walk through two reproducible synthetic examples in
|
|
111
|
+
[Check an Intercom Fin bill locally in 10 minutes](../docs/intercom-bill-walkthrough.md):
|
|
112
|
+
a CSV recount and a separate confirmed-versus-assumed settlement statement.
|
|
113
|
+
|
|
114
|
+
The buyer-side bill check for per-resolution vendors (Intercom Fin, Zendesk AI
|
|
115
|
+
Agents, …). Rules live in the [vendor-rules registry](../registry/vendor-rules.json)
|
|
116
|
+
(single source of truth, each rule cited and confidence-graded). Four commands,
|
|
117
|
+
one pipeline:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
verovo prepare --export native-export.csv --vendor zendesk --inspect
|
|
121
|
+
verovo prepare --export native-export.csv --vendor zendesk --out prepared.csv
|
|
122
|
+
# native Intercom/Zendesk export -> canonical skeleton. Judgement columns
|
|
123
|
+
# stay EMPTY for human review; native signal columns travel as hint_*
|
|
124
|
+
# cells; prov_* cells record where every value came from. No verdicts.
|
|
125
|
+
|
|
126
|
+
verovo recount --export prepared.csv --vendor zendesk --inspect
|
|
127
|
+
verovo recount --export prepared.csv --vendor zendesk --json recount.json
|
|
128
|
+
verovo recount --export prepared.csv --vendor zendesk --contract contract.json
|
|
129
|
+
# Tier 1: the vendor's own published rules, line by line. Findings in BOTH
|
|
130
|
+
# directions; absent evidence is UNPROVABLE, never guessed. --contract adds
|
|
131
|
+
# the buyer's outcome standard (e.g. reopen_window_hours: 72) as a separate
|
|
132
|
+
# outcome lane — renewal leverage, never netted into the claim.
|
|
133
|
+
|
|
134
|
+
verovo dispute --export prepared.csv --vendor zendesk --contract contract.json
|
|
135
|
+
# the negotiable pack: cover letter citing the vendor's own rule per
|
|
136
|
+
# finding + appendix + (optionally) Tier 2 under the VSS-1 standard.
|
|
137
|
+
|
|
138
|
+
verovo recovery --pack pack/dispute-pack.json --confirmations confirmations.csv
|
|
139
|
+
# what the vendor actually conceded: credited/paid per disputed line, with
|
|
140
|
+
# confirmation dates and evidence refs. Realized value is counted
|
|
141
|
+
# separately from the claim — an argument won is not cash.
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
`verovo recount --list-vendors` shows the registry with each rule's
|
|
145
|
+
source and confidence (P = primary/official, S = secondary, U = unverified).
|
|
146
|
+
The browser-local recount at [the website](../website/) runs the same Tier 1
|
|
147
|
+
with no upload; a cross-language parity test keeps the two from drifting.
|
|
148
|
+
|
|
149
|
+
Before a client review, confirm the contract version and outcome subtype.
|
|
150
|
+
Current public Intercom terms include Procedure outcomes and handoffs. Uncovered
|
|
151
|
+
terms must remain unresolved rather than be automatically marked unbillable.
|
|
152
|
+
CRM enrichment also has two reproduced input-boundary issues (missing or aliased
|
|
153
|
+
columns, and multi-record ordering) awaiting fixes; verify enriched values before
|
|
154
|
+
using them as client evidence.
|
|
155
|
+
|
|
156
|
+
### The whole story in one artifact
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
verovo crosscheck --export prepared.csv --vendor zendesk --ledger billing.csv
|
|
160
|
+
# your billing ledger (计费流水) against the export: charges the export
|
|
161
|
+
# doesn't flag, billed flags the ledger doesn't confirm, amounts off the
|
|
162
|
+
# published price. Disagreements are named, never resolved.
|
|
163
|
+
|
|
164
|
+
verovo verify --export prepared.csv --vendor zendesk --contract contract.json \
|
|
165
|
+
--ledger billing.csv --confirmations confirmations.csv --out vledger
|
|
166
|
+
# the Verified Ledger: Tier 1 + outcome lane + ledger cross-check +
|
|
167
|
+
# realized recovery in ONE document (json + md), each lane labelled
|
|
168
|
+
# claim / leverage / realized and never summed across.
|
|
169
|
+
|
|
170
|
+
verovo dashboard --pack vledger/verified-ledger.json --out dashboard.html
|
|
171
|
+
# offline single-file HTML: cards, tables, recovery bar. No JS, no
|
|
172
|
+
# network, printable — renders, never recomputes.
|
|
173
|
+
|
|
174
|
+
verovo rules-diff last-month-registry.json
|
|
175
|
+
# the monthly ritual: diff last period's rules snapshot against the
|
|
176
|
+
# packaged registry. Exit 1 = a material change (trigger, price, reopen
|
|
177
|
+
# rule) — re-verify the period under the new rule and disclose it.
|
|
178
|
+
|
|
179
|
+
verovo narrative --pack pack/dispute-pack.json --audience vendor \
|
|
180
|
+
--emit-prompt prompt.txt
|
|
181
|
+
# step 1 of two: the drafting prompt, built from the pack. Run YOUR OWN
|
|
182
|
+
# model on it, then:
|
|
183
|
+
verovo narrative --pack pack/dispute-pack.json --check-draft draft.txt --out letter.md
|
|
184
|
+
# step 2: every number in the draft is checked against the pack; any
|
|
185
|
+
# invented or rounded number refuses the draft and the rule-based
|
|
186
|
+
# template ships. The model never touches the numbers. No model? The
|
|
187
|
+
# template is the default.
|
|
188
|
+
|
|
189
|
+
verovo delivery --log delivery.jsonl --log-event \
|
|
190
|
+
--engagement ACME --vendor zendesk --period-index 1 --phase mapping --minutes 45
|
|
191
|
+
verovo delivery --log delivery.jsonl
|
|
192
|
+
# appendix-E metrics from logged phases: first look ≤10h, third period
|
|
193
|
+
# ≤1h, repeat-period reuse ≥70%. Use the mapping fingerprint `prepare`
|
|
194
|
+
# prints to track which engagements share the reusable mapping.
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Outcome pricing beyond per-resolution support (qualified lead, completed
|
|
198
|
+
case) is defined — schema first, per the BP — at
|
|
199
|
+
[schemas/outcome-unit.schema.json](../schemas/outcome-unit.schema.json) with
|
|
200
|
+
worked examples. The outcome **engine** now ships too:
|
|
201
|
+
`verovo outcomes --units units.jsonl` verifies them fail-closed, and the
|
|
202
|
+
schema covers action and usage-aggregate units — an engineering extension beyond the current support-billing sales scope.
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
verovo crm-join --export prepared.csv --crm crm.csv --out joined.csv
|
|
206
|
+
verovo outcomes --units outcome-units.jsonl --json outcome-ledger.json
|
|
207
|
+
verovo period-compare last-month/verified-ledger.json vledger/verified-ledger.json
|
|
208
|
+
verovo template --export --vendor intercom --fingerprint <fp> --contract contract.json
|
|
209
|
+
verovo certify --pack vledger/verified-ledger.json --digest-out digest.txt
|
|
210
|
+
verovo incrementality --events treatment-holdout.csv
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
### Agent commerce: existing engineering reference
|
|
214
|
+
|
|
215
|
+
Commercial development of this line is paused in the October buyer-review plan.
|
|
216
|
+
The following documents existing commands, not a current sales offer.
|
|
217
|
+
|
|
218
|
+
Traditional attribution starts at the click; agent commerce has none. The
|
|
219
|
+
merchant-side measurement receipt (F2.1-F2.9):
|
|
220
|
+
|
|
221
|
+
```bash
|
|
222
|
+
verovo commerce-ledger \
|
|
223
|
+
--events agent-events.jsonl --orders merchant-orders.csv \
|
|
224
|
+
--payments merchant-payments.csv --policy-version 1.0.0
|
|
225
|
+
# metric + value + evidence table; retries reconcile to 1 Operation /
|
|
226
|
+
# 2 Attempts / 1 verified order; net GMV is strong; incremental is
|
|
227
|
+
# UNPROVABLE (no holdout, no claim); materiality prints the discrepancy
|
|
228
|
+
# vs the naive dashboard in currency AND percent.
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
live / replay / synthetic never mix inside one aggregation (error, not a
|
|
232
|
+
blend); metric names carry their strength prefix and are linted against the
|
|
233
|
+
evidence that backs them; every input is sha256-anchored. The operating layer on top:
|
|
234
|
+
|
|
235
|
+
```bash
|
|
236
|
+
verovo pnl-trees --orders orders.csv --campaigns campaigns.csv # Organic / Paid, ROAS in Paid only
|
|
237
|
+
verovo cm-ledger --orders orders.csv --costs costs.csv --period 2026-09
|
|
238
|
+
verovo demand-audit --measurements m.csv --thresholds t.json # Launch / Watch / Not ready
|
|
239
|
+
verovo intent-taxonomy --file clusters.json # Operating Cell health
|
|
240
|
+
verovo decision-audit --policy policy.json --executions log.jsonl
|
|
241
|
+
verovo benchmark-export --rows brand-rows.csv --min-brands 5 # aggregates only, ever
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
Schema:
|
|
245
|
+
[schemas/commerce-profile.schema.json](../schemas/commerce-profile.schema.json);
|
|
246
|
+
canonical retry vector pinned in CI at
|
|
247
|
+
`conformance/vectors/commerce-retry-baseline/`.
|
|
248
|
+
|
|
249
|
+
`certify` grades the artifact on the VSS-1 adoption ladder (Stage 0..3) and
|
|
250
|
+
emits the deterministic digest; hand the digest to your own signing tool —
|
|
251
|
+
this package never runs one. `incrementality` produces statistics about a
|
|
252
|
+
difference, never a verdict: the BP's line between rule judgement and causal
|
|
253
|
+
inference stays visible in the output itself.
|
|
254
|
+
|
|
255
|
+
## Sharing results: preview first, export second
|
|
256
|
+
|
|
257
|
+
The recommended flow never writes a shareable file until you have read it:
|
|
258
|
+
|
|
259
|
+
```bash
|
|
260
|
+
verovo check --json run.json # the report export embeds the summary
|
|
261
|
+
verovo share run.json # terminal PREVIEW — nothing is written
|
|
262
|
+
verovo share run.json --out summary.md # export only after you reviewed it
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
`share` re-checks the summary against its whitelist before showing or writing
|
|
266
|
+
anything, so a tampered or future-format export is refused rather than leaked.
|
|
267
|
+
`check --share PATH` remains available as the one-step explicit alternative.
|
|
268
|
+
|
|
269
|
+
## Snapshots & compare — did the change help?
|
|
270
|
+
|
|
271
|
+
The re-run preview: save a snapshot, change something (a prompt, a Skill, a
|
|
272
|
+
dependency), run the agent again, compare.
|
|
273
|
+
|
|
274
|
+
```bash
|
|
275
|
+
python3 healthcheck/verovo check --save-snapshot before.json
|
|
276
|
+
# …make your change and run the agent…
|
|
277
|
+
python3 healthcheck/verovo check --save-snapshot after.json
|
|
278
|
+
python3 healthcheck/verovo compare before.json after.json
|
|
279
|
+
# or skip the second snapshot: compare runs a fresh check as side B
|
|
280
|
+
python3 healthcheck/verovo compare --all before.json
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
`compare` prints metric deltas (failed executions, retry chains, …), check
|
|
284
|
+
verdict transitions (`HC-02 OK → FINDING`), and honest caveats: token totals
|
|
285
|
+
are differenced only when **both** sides are provable, and version / window /
|
|
286
|
+
mode mismatches are disclosed because they change what a delta is allowed to
|
|
287
|
+
mean. Snapshots are local artifacts (they contain project names and session
|
|
288
|
+
short-ids); the snapshot schema is versioned and validated on load.
|
|
289
|
+
|
|
290
|
+
## Versioned export schemas — how other tools integrate (R8)
|
|
291
|
+
|
|
292
|
+
Everything a third party needs is a JSON document with a version on it:
|
|
293
|
+
|
|
294
|
+
| document | written by | schema |
|
|
295
|
+
| --- | --- | --- |
|
|
296
|
+
| report export | `check --json PATH` / `demo --json PATH` | `report-v1` |
|
|
297
|
+
| snapshot | `--save-snapshot PATH` | `snapshot-v1` (schema field = 1) |
|
|
298
|
+
| comparison | `compare --json PATH` | `compare-v1` |
|
|
299
|
+
|
|
300
|
+
Reference schemas live in [`schemas/`](schemas/). Compatibility policy:
|
|
301
|
+
**within a schema version, fields are only added — never renamed, retyped, or
|
|
302
|
+
removed**; a consumer that meets an unknown newer version must stop and say so.
|
|
303
|
+
`verovo validate FILE…` checks any export against its schema, and
|
|
304
|
+
[`examples/track-weekly.py`](examples/track-weekly.py) is a complete external
|
|
305
|
+
consumer: it reads snapshots through the schema only, imports nothing from the
|
|
306
|
+
package, and refuses future schema versions. MCP-style servers can wrap the
|
|
307
|
+
same exports later; the exports, not internal modules, are the integration
|
|
308
|
+
surface.
|
|
309
|
+
|
|
310
|
+
## What it checks
|
|
311
|
+
|
|
312
|
+
| id | check | question it answers | verdict basis |
|
|
313
|
+
| --- | --- | --- | --- |
|
|
314
|
+
| HC-01 | Duplicate records | byte-identical lines, repeated call/execution ids, sessions split across files, and the format's **by-design dual recording** when model-side calls and command events are both present | exact line hashes, id counts |
|
|
315
|
+
| HC-02 | Retry amplification | same command failed then re-run: how many executions carried one logical operation | consecutive same-command blocks whose first attempt failed |
|
|
316
|
+
| HC-03 | Tool error runs | ≥3 back-to-back failures of the same tool with no success between | consecutive failed executions, same tool kind |
|
|
317
|
+
|
|
318
|
+
Plus a coverage overview: sessions, turns, executions by outcome, token
|
|
319
|
+
consumption (with subset discipline), compactions, sub-agent activity, corrupt
|
|
320
|
+
lines, and a **per-project breakdown** (project = cwd basename, canonical
|
|
321
|
+
deduplicated counts). Every finding carries evidence — file, line number, exit
|
|
322
|
+
codes — and a concrete next step.
|
|
323
|
+
|
|
324
|
+
## Honest-number rules (the part most tools get wrong)
|
|
325
|
+
|
|
326
|
+
- **Executions are counted from exactly one stream.** Codex Desktop can write a
|
|
327
|
+
tool call as both a `response_item` record and an `item_completed` event.
|
|
328
|
+
Summing both — or grepping the file — double-counts when both are present.
|
|
329
|
+
HC-01 discloses the observed ratio instead of hiding it.
|
|
330
|
+
- **Token subsets are never added into totals.** `cached_input` ⊆ input,
|
|
331
|
+
`reasoning_output` ⊆ output in Codex logs. The report shows subsets
|
|
332
|
+
alongside, never inside, totals.
|
|
333
|
+
- **Token totals use the last cumulative snapshot per session**, not sums of
|
|
334
|
+
per-event deltas (which double-count after context compaction). The aggregate
|
|
335
|
+
is marked UNPROVABLE unless every logical session has one valid snapshot;
|
|
336
|
+
missing or malformed snapshots are reported separately.
|
|
337
|
+
- **Corrupt or truncated input lowers every count.** All numbers are lower bounds
|
|
338
|
+
where lines failed to parse or a file hit the safety cap; the limitation is
|
|
339
|
+
reported, never dropped.
|
|
340
|
+
- **UNPROVABLE, not zero.** Sessions whose format lacks outcome events make
|
|
341
|
+
HC-02/HC-03 UNPROVABLE for that part — reported as such.
|
|
342
|
+
|
|
343
|
+
## Statistics definitions (for support and accounting)
|
|
344
|
+
|
|
345
|
+
| term | definition |
|
|
346
|
+
| --- | --- |
|
|
347
|
+
| execution | one `CommandExecution` / `McpToolCall` event; its outcome is `ok`, `failed`, or `unknown` |
|
|
348
|
+
| retry chain | a maximal block of consecutive executions of the same command whose first attempt failed; one chain = one logical operation executed N times |
|
|
349
|
+
| resolved chain | a chain whose last attempt succeeded |
|
|
350
|
+
| tool error run | ≥3 consecutive failed executions of the same tool kind with no success or unknown in between |
|
|
351
|
+
| token total | last cumulative `total_token_usage` snapshot per session, summed only when every session has a valid snapshot; otherwise UNPROVABLE |
|
|
352
|
+
| project | basename of the session's recorded working directory (`session_meta.cwd`); used for the breakdown, `--project` filtering, and snapshots — never included in the share summary |
|
|
353
|
+
| snapshot | versioned JSON record of one run's window, verdicts, and aggregate counters (schema 1); local artifact |
|
|
354
|
+
| comparison delta | `B − A` per metric; tokens are differenced only when provable on both sides, and version/window/mode mismatches are disclosed as caveats |
|
|
355
|
+
| demo run | executed on the bundled synthetic session; counted separately (`synthetic-demo`) |
|
|
356
|
+
| own-data run | executed on real local logs (`own-data`) |
|
|
357
|
+
| run number | local run history at `~/.verovo/history.jsonl` distinguishes first vs repeated runs; delete the file to reset |
|
|
358
|
+
|
|
359
|
+
## Privacy
|
|
360
|
+
|
|
361
|
+
- **No network.** The package contains no network code; a test enforces this by
|
|
362
|
+
scanning every module for network imports.
|
|
363
|
+
- **Raw data stays local.** Prompts, message text, and reasoning content are
|
|
364
|
+
never read into the model (only envelope types, ids, exit codes, durations,
|
|
365
|
+
and token counters).
|
|
366
|
+
- **Two artifacts, two levels.** The HTML report is personal (local paths and
|
|
367
|
+
evidence metadata — raw commands are represented by hashes, and the banner
|
|
368
|
+
says so). The `--share` summary
|
|
369
|
+
is built from a fixed whitelist of aggregate counts: no prompts, paths,
|
|
370
|
+
commands, repo names, or session ids can appear in it, and a planted-string
|
|
371
|
+
test enforces this.
|
|
372
|
+
- **Run history** (`~/.verovo/history.jsonl`) stays on the machine and is
|
|
373
|
+
deletable.
|
|
374
|
+
|
|
375
|
+
## Verified runtime support
|
|
376
|
+
|
|
377
|
+
| runtime | status | verified against |
|
|
378
|
+
| --- | --- | --- |
|
|
379
|
+
| Codex rollout JSONL (Desktop originator) | supported | cli 0.142.3 – 0.153.0; review snapshot 54 local files, 48 sessions, ~83k lines (2026-06-30 → 2026-09-05). Sessions without `CommandExecution` events (older format) run with HC-02/HC-03 UNPROVABLE, disclosed. |
|
|
380
|
+
| Codex CLI (`codex_cli_rs` originator) | expected compatible, **not yet verified** — no local samples | pending real samples |
|
|
381
|
+
| Claude Code (`~/.claude/projects`) | detected, **not yet supported** | adapter ships after validation against real samples |
|
|
382
|
+
|
|
383
|
+
New cli versions are handled defensively: unknown event types are accounted
|
|
384
|
+
(`unknown type` counts), never silently dropped.
|
|
385
|
+
|
|
386
|
+
## Exit codes
|
|
387
|
+
|
|
388
|
+
| code | meaning |
|
|
389
|
+
| --- | --- |
|
|
390
|
+
| 0 | the check ran and produced a report (findings or not) |
|
|
391
|
+
| 1 | selftest fixture mismatch (development only) |
|
|
392
|
+
| 2 | input/output error: no readable logs in the window, bad `--dir`/arguments, or an output path cannot be written |
|
|
393
|
+
|
|
394
|
+
## Development
|
|
395
|
+
|
|
396
|
+
```bash
|
|
397
|
+
python3 -m unittest discover -s healthcheck/tests # 511 tests verified 2026-10-15
|
|
398
|
+
python3 healthcheck/verovo selftest # fixtures + redaction + compare + schemas
|
|
399
|
+
bash healthcheck/scripts/smoke_install.sh # clean-venv install smoke (packaged artifact)
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
The install smoke is also a CI job (Python 3.9/3.11): it builds the package,
|
|
403
|
+
installs it into a fresh venv, and runs `selftest`, `demo`, `validate`, and the
|
|
404
|
+
external example from the installed artifact — the repository checkout is not
|
|
405
|
+
used.
|
|
406
|
+
|
|
407
|
+
Python 3.9+ standard library only. Zero dependencies, by policy: if any step
|
|
408
|
+
asks you to register, connect to a network, or pay — that is a bug.
|
|
409
|
+
|
|
410
|
+
## Feedback
|
|
411
|
+
|
|
412
|
+
- [Open an issue](https://github.com/roy-tong/verovo/issues) — include
|
|
413
|
+
the `--json` export (it contains local paths; strip them first if the log
|
|
414
|
+
location is sensitive).
|
|
415
|
+
- [Email](mailto:tongroy18@gmail.com?subject=Verovo%20Healthcheck%20feedback)
|
|
416
|
+
|
|
417
|
+
## Identify an input before checking it
|
|
418
|
+
|
|
419
|
+
```bash
|
|
420
|
+
verovo inspect-input sample.jsonl --json input-routing.json
|
|
421
|
+
```
|
|
422
|
+
|
|
423
|
+
This reads every nonempty JSONL row and identifies canonical provider observations, FMT-002, or Codex rollout shapes. Mixed, invalid and unknown inputs exit 2 with next steps. Exit 0 means a format was recognized, **not** that the schema or measurement passed. The report includes no source rows. Canonical observations use the repository validator and provider onboarding path; do not invent fields to force them into FMT-002.
|
|
424
|
+
|
|
425
|
+
## Compare Claude response snapshots (0.8.0)
|
|
426
|
+
|
|
427
|
+
`verovo response-audit session.jsonl fork.jsonl --json responses.json` compares explicit inputs without rewriting logs or claiming invoice totals. See [scope and next steps](../docs/response-audit.md).
|