fp-cloud-cli 0.0.1b1__tar.gz → 0.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/CHANGELOG.md +163 -0
- {fp_cloud_cli-0.0.1b1/fp_cloud_cli.egg-info → fp_cloud_cli-0.0.2}/PKG-INFO +163 -4
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/README.md +161 -2
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/_click_compat.py +31 -8
- fp_cloud_cli-0.0.2/fp_cli/_version.py +1 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/app.py +19 -1
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/auth.py +29 -4
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/client.py +397 -27
- fp_cloud_cli-0.0.2/fp_cli/cloud_output.py +523 -0
- fp_cloud_cli-0.0.2/fp_cli/commands/_cloud.py +45 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/_write.py +3 -1
- fp_cloud_cli-0.0.2/fp_cli/commands/account_cmds.py +171 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/agent_cmds.py +80 -7
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/alerts_cmds.py +23 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/audits_cmds.py +2 -0
- fp_cloud_cli-0.0.2/fp_cli/commands/contracts_cmds.py +67 -0
- fp_cloud_cli-0.0.2/fp_cli/commands/dashboards_cmds.py +159 -0
- fp_cloud_cli-0.0.2/fp_cli/commands/evaluators_cmds.py +380 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/events_cmds.py +14 -1
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/fleet_cmds.py +78 -16
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/incidents_cmds.py +305 -8
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/keys_cmds.py +17 -3
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/list_cmds.py +4 -1
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/orgs_cmds.py +17 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/policies_cmds.py +227 -16
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/queries_cmds.py +70 -0
- fp_cloud_cli-0.0.2/fp_cli/commands/sentiment_cmds.py +49 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/sessions_cmds.py +95 -1
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/settings_cmds.py +2 -0
- fp_cloud_cli-0.0.2/fp_cli/commands/usage_cmds.py +55 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/users_cmds.py +14 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/enforcement.py +124 -6
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/errors.py +20 -3
- fp_cloud_cli-0.0.2/fp_cli/evaluator_spec.py +266 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/models.py +100 -6
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/output.py +469 -101
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/permissions.py +23 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/policy_check.py +172 -0
- fp_cloud_cli-0.0.2/fp_cli/theme.py +50 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2/fp_cloud_cli.egg-info}/PKG-INFO +163 -4
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cloud_cli.egg-info/SOURCES.txt +35 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/pyproject.toml +10 -3
- fp_cloud_cli-0.0.2/tests/__pycache__/__init__.cpython-312.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/conftest.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_alerting.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_analytics.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_audits.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_auth.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_auth_mode.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_click_compat.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_client.cpython-312-pytest-9.1.1.pyc +0 -0
- fp_cloud_cli-0.0.2/tests/__pycache__/test_cloud_account.cpython-312-pytest-9.1.1.pyc +0 -0
- fp_cloud_cli-0.0.2/tests/__pycache__/test_cloud_agent.cpython-312-pytest-9.1.1.pyc +0 -0
- fp_cloud_cli-0.0.2/tests/__pycache__/test_cloud_human_views.cpython-312-pytest-9.1.1.pyc +0 -0
- fp_cloud_cli-0.0.2/tests/__pycache__/test_cloud_jev.cpython-312-pytest-9.1.1.pyc +0 -0
- fp_cloud_cli-0.0.2/tests/__pycache__/test_cloud_observability.cpython-312-pytest-9.1.1.pyc +0 -0
- fp_cloud_cli-0.0.2/tests/__pycache__/test_cloud_transport.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_commands.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_config.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_dashboards_agent.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_dates.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_enforcement_logic.cpython-312-pytest-9.1.1.pyc +0 -0
- fp_cloud_cli-0.0.2/tests/__pycache__/test_evaluator_authoring.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_facets.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_failproofai_home.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_failproofai_home.cpython-312.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_fp_home_contract.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_hardening.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_help_table_coverage.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_keys_queries.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_list.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_multivalue.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_no_customer_identifiers.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_operator.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_orgs.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_output.cpython-312-pytest-9.1.1.pyc +0 -0
- fp_cloud_cli-0.0.2/tests/__pycache__/test_pagination.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_policy_check.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_readme_matches_reality.cpython-312-pytest-9.1.1.pyc +0 -0
- fp_cloud_cli-0.0.2/tests/__pycache__/test_request_id.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_review_fixes.cpython-312-pytest-9.1.1.pyc +0 -0
- fp_cloud_cli-0.0.2/tests/__pycache__/test_round4_behaviour.cpython-312-pytest-9.1.1.pyc +0 -0
- fp_cloud_cli-0.0.2/tests/__pycache__/test_screen_language.cpython-312-pytest-9.1.1.pyc +0 -0
- fp_cloud_cli-0.0.2/tests/__pycache__/test_session_reevaluate.cpython-312-pytest-9.1.1.pyc +0 -0
- fp_cloud_cli-0.0.2/tests/__pycache__/test_staging_parity.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_telemetry_completeness.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_usage.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_v1_origin_diagnostic.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_v1_routing.cpython-312-pytest-9.1.1.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__pycache__/test_whoami.cpython-312-pytest-9.1.1.pyc +0 -0
- fp_cloud_cli-0.0.2/tests/fake_cloud_smoke.py +312 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_alerting.py +184 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_click_compat.py +54 -0
- fp_cloud_cli-0.0.2/tests/test_cloud_account.py +125 -0
- fp_cloud_cli-0.0.2/tests/test_cloud_agent.py +253 -0
- fp_cloud_cli-0.0.2/tests/test_cloud_human_views.py +287 -0
- fp_cloud_cli-0.0.2/tests/test_cloud_jev.py +527 -0
- fp_cloud_cli-0.0.2/tests/test_cloud_observability.py +116 -0
- fp_cloud_cli-0.0.2/tests/test_cloud_transport.py +141 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_commands.py +3 -3
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_dashboards_agent.py +3 -3
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_enforcement_logic.py +46 -0
- fp_cloud_cli-0.0.2/tests/test_evaluator_authoring.py +449 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_facets.py +1 -1
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_help_table_coverage.py +15 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_keys_queries.py +65 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_operator.py +1 -1
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_orgs.py +1 -1
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_output.py +87 -37
- fp_cloud_cli-0.0.2/tests/test_pagination.py +204 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_policy_check.py +61 -0
- fp_cloud_cli-0.0.2/tests/test_request_id.py +111 -0
- fp_cloud_cli-0.0.2/tests/test_round4_behaviour.py +144 -0
- fp_cloud_cli-0.0.2/tests/test_screen_language.py +80 -0
- fp_cloud_cli-0.0.2/tests/test_session_reevaluate.py +33 -0
- fp_cloud_cli-0.0.2/tests/test_staging_parity.py +198 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_whoami.py +1 -1
- fp_cloud_cli-0.0.1b1/fp_cli/_version.py +0 -1
- fp_cloud_cli-0.0.1b1/fp_cli/commands/usage_cmds.py +0 -35
- fp_cloud_cli-0.0.1b1/fp_cli/theme.py +0 -53
- fp_cloud_cli-0.0.1b1/tests/__pycache__/__init__.cpython-312.pyc +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/LICENSE +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/MANIFEST.in +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/__init__.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/__main__.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/_context.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/analytics.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/analytics_config.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/analytics_registry.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/__init__.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/auth_cmds.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/errors_cmds.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/evals_cmds.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/commands/guardrails_cmds.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/config.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/dates.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/orgs.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/py.typed +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cli/select.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cloud_cli.egg-info/dependency_links.txt +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cloud_cli.egg-info/entry_points.txt +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cloud_cli.egg-info/requires.txt +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/fp_cloud_cli.egg-info/top_level.txt +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/setup.cfg +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/__init__.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/conftest.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_analytics.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_audits.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_auth.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_auth_mode.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_client.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_config.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_dates.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_failproofai_home.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_fp_home_contract.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_hardening.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_list.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_multivalue.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_no_customer_identifiers.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_readme_matches_reality.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_review_fixes.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_telemetry_completeness.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_usage.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_v1_origin_diagnostic.py +0 -0
- {fp_cloud_cli-0.0.1b1 → fp_cloud_cli-0.0.2}/tests/test_v1_routing.py +0 -0
|
@@ -1,5 +1,168 @@
|
|
|
1
1
|
# Changelog — `fp` CLI
|
|
2
2
|
|
|
3
|
+
## 0.0.2 — 2026-10-25
|
|
4
|
+
|
|
5
|
+
**Stable release.** The `0.0.2b0` line the `bump` job opened after `0.0.1` is cut stable
|
|
6
|
+
here rather than published, because the server endpoints these commands drive are now
|
|
7
|
+
deployed to FailproofAI Cloud and the CLI should not trail them as a pre-release.
|
|
8
|
+
|
|
9
|
+
Nothing about the command surface changes at the cut: the entries below are the entire
|
|
10
|
+
diff from `0.0.1`, and the `bump` job opens `0.0.3b0` after the upload.
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- Hosted evaluator authoring for code, LLM judge and Jev: create/deploy, AI draft specs, dry-run, labels, toggles, instances, schema, jobs and backfill.
|
|
15
|
+
- Dashboards and tiles; session overview/export/history/re-evaluate; latency and sentiment; query composition/export; alert recipients/issues and agent contracts.
|
|
16
|
+
- Policy author/backtest/candidacy, assistant artifact tables, fail-closed interactive write approvals, and agent-curated home briefs.
|
|
17
|
+
- Organization creation, permission-set roles, member deletion/defaults, usage windows, offers/trial, model context windows, OTEL and Slack controls.
|
|
18
|
+
- `fp policies backtest` covers every policy kind: a published `jev`/`both` policy, or `--semantic checks.json` (with or without a source file), runs the dashboard's every-kind backtest — a free estimate by default, `--run [--sample N]` to spend Jev allowance. New `--kind --semantic --hours --agent-id --env --run --sample`; a regex file or id still uses the module backtest.
|
|
19
|
+
- `fp policies author --kind regex|jev|both` asks the assistant for Jev checks or a hybrid policy.
|
|
20
|
+
- `fp issues list|count` take the inbox filters (`--agent-id --env --machine --source --audit-id --assignee --archived --search --since`, and `count --alert-id`); new `fp issues facets` lists the values they can take.
|
|
21
|
+
- `fp sessions import FILE...` imports Claude Code transcripts, Codex rollouts or event JSONL (`--format --env --dry-run`), as the dashboard's import dialog does.
|
|
22
|
+
- `fp list trigger_events`, `fp list eval_envs` and `fp list metric_keys`.
|
|
23
|
+
- `fp fleet deploy --target POLICY=INTEGRATION[/agt_ID]` scopes a machine's assignment; repeat the flag to include multiple alternatives. `--all-agents POLICY` explicitly clears its scope. Deploy plans, show, history, rollback and JSON readback retain assignment targets.
|
|
24
|
+
- `fp policies publish <id> --kind regex|jev|both [--source f.mjs] [--semantic f.json]`: a
|
|
25
|
+
FailproofAI Cloud policy can carry Jev checks — `jev` (declarations only, no JavaScript) or
|
|
26
|
+
`both` (JavaScript reviewable by exactly its own checks; the server derives the authority).
|
|
27
|
+
The kind follows from what is given when `--kind` is omitted, and a kind that contradicts its
|
|
28
|
+
inputs is exit 2 before anything is sent. The positional source still works.
|
|
29
|
+
- No Jev mode is set by hand: FailproofAI Cloud derives a machine's Jev mode from what is
|
|
30
|
+
deployed to it (on while a `jev`/`both` policy is, off otherwise), so `fp fleet deploy` has
|
|
31
|
+
no `--jev-mode` and there is no `fp fleet jev-mode`. An `observe` effect applies to `jev`
|
|
32
|
+
and `both` policies like any other: their checks are recorded and decide nothing.
|
|
33
|
+
- `fp fleet rollback` says when the Jev mode differs (`jev mode <now> → <then>`, and
|
|
34
|
+
`jevModeBefore` in `--json`); `fp fleet history` shows each generation's Jev mode.
|
|
35
|
+
- The `fleet deploy` and `policies publish` help says Jev checks run on
|
|
36
|
+
FailproofAI Cloud and nothing is installed on the machine, and `fleet show`
|
|
37
|
+
names the reasons a machine with a mode cannot ask (`jev_unconfigured`, `transcripts_disabled`)
|
|
38
|
+
and the `jev_budget` report.
|
|
39
|
+
- `fp policies list/show` print each version's kind and Jev check names (`show` also the
|
|
40
|
+
declarations); `fp fleet list/show` print the machine's Jev mode and the policy errors it
|
|
41
|
+
reported, and `--json` carries `kind`/`semantic`/`semanticSha256`/`authority`/`reviewedBy`,
|
|
42
|
+
`jevMode` and `policyErrors`/`policyErrorsAt` when the server sends them.
|
|
43
|
+
|
|
44
|
+
- `jev:evaluate` is a known permission — Jev through FailproofAI Cloud, charged to the
|
|
45
|
+
org's plan — so `fp keys` and `fp users` accept it instead of refusing it as unknown,
|
|
46
|
+
and the `admin` preset carries it, as the server's built-in admin set now does. The
|
|
47
|
+
server refuses a KEY carrying it without both `events:add` and `policies:pull` (422,
|
|
48
|
+
naming what to add in `missing_permissions`); the `admin` preset has both. (#833)
|
|
49
|
+
- `fp keys create/update --permission-set machine`: the dashboard's key-only preset
|
|
50
|
+
(`events:add` + `policies:pull` + `jev:evaluate`), for an enrolled machine using Jev through
|
|
51
|
+
FailproofAI Cloud. `fp keys` also refuses `jev:evaluate` without both prerequisites itself
|
|
52
|
+
(exit 2, naming what is missing) instead of sending the key for the server's 422. (#833)
|
|
53
|
+
- `fp policies list` and `fp policies show` print what a `jev`/`both` version's checks take of a
|
|
54
|
+
machine's Jev question budget (`jevChars`), and a deploy refused with `jev_budget_exceeded`
|
|
55
|
+
lists each Jev policy's share, largest first (the `hint` under `--json`), so the refusal says
|
|
56
|
+
which policy to leave off.
|
|
57
|
+
|
|
58
|
+
### Changed
|
|
59
|
+
|
|
60
|
+
- New Cloud API families work in key mode; unsupported session-only commands deny before HTTP and unknown-route 404s exit 6 with an availability message.
|
|
61
|
+
- Shared screens use the mint/pink wordmark and grey labels, without boxes/rules or shortened ids; existing JSON contracts are unchanged.
|
|
62
|
+
- The agent skill, command reference, README and Cloud CLI docs include end-to-end evaluator workflows. No version bump or release is performed.
|
|
63
|
+
- `fp policies publish`'s refusal of Jev fields in the JavaScript now says where Jev checks go:
|
|
64
|
+
the policy's own declarations, `--kind both --semantic checks.json`.
|
|
65
|
+
- `fp guardrails summary` shows how many machines run Jev in each mode, reading the summary's
|
|
66
|
+
`jev.machines.observe` and `jev.machines.enforce`.
|
|
67
|
+
- `fp policies publish` (and `policies compose --publish`) refuses a source carrying Jev
|
|
68
|
+
fields a cloud policy never reads — a `semanticPolicies.add(...)` call, or a registration
|
|
69
|
+
declaring `authority: "reviewable"` — before anything is sent (exit 1, `--no-verify` does
|
|
70
|
+
not skip it). Both were published as versions that read Jev-aware and are not: semantic
|
|
71
|
+
checks load only from a pack, and a cloud policy's authority comes from its deployment, so
|
|
72
|
+
it stays hard. Ship Jev checks in a pack with `failproofai publish`. Same rule, and the
|
|
73
|
+
same comment-aware detection, as the dashboard's publish form; the server does not read
|
|
74
|
+
the source, so nothing else refuses it for `fp`. (#833)
|
|
75
|
+
|
|
76
|
+
### Fixed
|
|
77
|
+
|
|
78
|
+
- Remove the unused colour-keyed permission rank, document full-id rendering, emit UTC `Z` timestamps for deploy backfills, and report approval-answer failures without a traceback while restoring the terminal (#885).
|
|
79
|
+
- Reject natural-language binary judge steps, distinguish unknown routes from missing records on 404, normalize standalone backfill ranges to UTC `Z`, expose all subcommands in help, and render artifact headers before titles with consistent session-only wording (#885).
|
|
80
|
+
- New Cloud commands use deliberate human summaries/tables and actionable empty states at 80/140 columns without raw response blobs. Evaluator previews display before writes and require explicit `--force` after failures; plan/claim deny key mode, deploy specs can omit the positional key, and older settings-route 404s degrade clearly. Existing JSON payloads stay unchanged (#885).
|
|
81
|
+
- `fp evaluators deploy` keeps the previous version's condition and labels unless `--condition`/`--label` change them, so a new version no longer silently widens to every session (#885).
|
|
82
|
+
- Evaluation backfill covers code, LLM judge and Jev evaluators, as FailproofAI Cloud now does: `fp evaluators deploy --backfill` no longer refuses judge/Jev, and both commands warn that judge/Jev backfills spend model budget instead of saying they are skipped.
|
|
83
|
+
- `fp usage windows` says it needs an API key (the dashboard has no session route for it) instead of reporting the route as unavailable.
|
|
84
|
+
- `fp issues close --help` no longer claims exit 9 for an issue that already ended: it is exit 1, as it always was, and it names `fp audits dismiss` rather than a command that does not exist.
|
|
85
|
+
- `fp fleet history` keeps `gen`, `when` and `jev` visible at 80 columns (rollback needs `gen`): the policy list is shortened, then dropped, and `change` wraps first.
|
|
86
|
+
- `fp fleet show` and `fp fleet history` name an exact-profile target by the profile label the machine reports (`claude/.claude-work`, with the `agt_…` id kept in `show`), falling back to the bare id.
|
|
87
|
+
|
|
88
|
+
## 0.0.1 — 2026-09-22
|
|
89
|
+
|
|
90
|
+
**First stable release.** `0.0.1b1` and `0.0.1b2` are the two betas behind it; the
|
|
91
|
+
`0.0.1b3` line the `bump` job opened is cut stable here rather than published, because
|
|
92
|
+
the server endpoints these commands drive are deployed to FailproofAI Cloud and the
|
|
93
|
+
CLI half should not stay a pre-release behind them. Until now `pipx install
|
|
94
|
+
fp-cloud-cli` resolved a beta only because no stable existed — a default that would
|
|
95
|
+
have silently changed the day one did. `Development Status` moves to
|
|
96
|
+
`5 - Production/Stable` with the version, so the classifier and the one string pip
|
|
97
|
+
reads say the same thing.
|
|
98
|
+
|
|
99
|
+
Nothing about the command surface changes at the cut: the entries below are the
|
|
100
|
+
entire diff from `0.0.1b2`, and `0.0.2b0` opens the next beta line.
|
|
101
|
+
|
|
102
|
+
### Added
|
|
103
|
+
|
|
104
|
+
- `fp issues close <id>` — end an issue as won't-fix. Distinct from `resolve`: a
|
|
105
|
+
recurring audit finding reopens a **resolved** issue and leaves a **closed** one
|
|
106
|
+
alone. On an audit issue it marks the finding `dismissed`, without writing the
|
|
107
|
+
org-wide fingerprint suppression that `fp audits dismiss` writes. Exit 9 if the
|
|
108
|
+
issue already ended. (#815)
|
|
109
|
+
- `fp issues archive <id>` / `unarchive <id>` — take an issue off the board and put
|
|
110
|
+
it back. Orthogonal to state, allowed in any state, no confirmation prompt. (#815)
|
|
111
|
+
- `fp issues clear (--audit <id> | --all-audits | --everything) [--dry-run]` — resolve
|
|
112
|
+
every open issue in a scope plus the audit findings behind them, in one server-side
|
|
113
|
+
transaction. Exactly one scope flag is required. Needs `issues:close` **and**
|
|
114
|
+
`audits:write`. Writes no suppression, so anything still broken reopens its issue
|
|
115
|
+
on the next run. The confirm names the count from a dry run that shares its
|
|
116
|
+
scope predicate with the write. (#815)
|
|
117
|
+
- `issues list --state closed` is accepted, and `Incident` carries `closed_at` and
|
|
118
|
+
`archived_at`. (#815)
|
|
119
|
+
|
|
120
|
+
### Fixes
|
|
121
|
+
|
|
122
|
+
- The recurrence sentence said two different things in one release. `audits resolve`'s
|
|
123
|
+
confirm line (`a genuine recurrence re-opens as new`), the skill and the command
|
|
124
|
+
reference implied a recurrence opens a **different** issue; `issues close --help`, the
|
|
125
|
+
audits guide and the CLI reference said the resolved issue **reopens**. The second is
|
|
126
|
+
the one the rest of the model needs — a `closed` issue has to have something to stay
|
|
127
|
+
closed through — so `reopens` is now what every surface says. (#815)
|
|
128
|
+
- `issues clear --help` claimed the `--dry-run` count "cannot disagree" with the write.
|
|
129
|
+
The two requests share a scope predicate, not a row set, so an issue entering the scope
|
|
130
|
+
between them is cleared without being in the confirmed number — correct for a scoped
|
|
131
|
+
clear, but not the guarantee that was written. The help now states the real one, and
|
|
132
|
+
names the closing line as the count of what changed. (#815)
|
|
133
|
+
- The skill answered "clear all our issues" with `--all-audits`, which leaves alert-born
|
|
134
|
+
and hand-opened issues untouched. Settling the scope is now the first step. (#815)
|
|
135
|
+
|
|
136
|
+
## 0.0.1b2 — 2026-08-25
|
|
137
|
+
|
|
138
|
+
Open for the next release. `0.0.1b1` published on 2026-08-24 and the `bump` job
|
|
139
|
+
moved the version here automatically. Add entries as changes merge — this section
|
|
140
|
+
becomes the GitHub Release body when it ships.
|
|
141
|
+
|
|
142
|
+
### Fixes
|
|
143
|
+
|
|
144
|
+
- **Typed errors keep their exit codes under typer 0.27.2.** That release moved
|
|
145
|
+
`Abort` out of its vendored `typer._click.exceptions`, and `_click_compat`
|
|
146
|
+
imported all six of its symbols under a single
|
|
147
|
+
`try: … except ImportError: from click import …`. One missing name was enough
|
|
148
|
+
to send the whole block to the fallback, binding `ClickException`,
|
|
149
|
+
`UsageError`, `BadParameter`, `Command` and `Parameter` to the pip `click`
|
|
150
|
+
distribution — which is *not* the Click Typer runs, and Typer catches only its
|
|
151
|
+
own. So every typed error escaped its handler: `fp alerts show ghost` exited
|
|
152
|
+
**1 with an empty stderr** instead of **6** with `no alert named "ghost"`, and
|
|
153
|
+
likewise for exits 2, 3, 4 and 5. Nothing warned; the CLI imported and every
|
|
154
|
+
happy path passed. The Click is now decided **once**, on whether `typer._click`
|
|
155
|
+
exists at all, and every symbol imported from that decision, so a future move
|
|
156
|
+
fails at import — a CLI that refuses to start — instead of silently flattening
|
|
157
|
+
the exit-code contract. `Abort` is resolved from `typer.Abort`, which is by
|
|
158
|
+
construction the class `typer.prompt` raises and typer's own `_main` catches on
|
|
159
|
+
every version in range. Suite verified green against typer 0.25.1, 0.27.0,
|
|
160
|
+
0.27.1 and 0.27.2 (#771)
|
|
161
|
+
|
|
162
|
+
### Dependencies
|
|
163
|
+
|
|
164
|
+
- typer 0.27.1 → 0.27.2, click 8.4.2 → 8.5.0, posthog 7.42.0 → 7.44.2 (#771)
|
|
165
|
+
|
|
3
166
|
## 0.0.1b1 — 2026-08-24
|
|
4
167
|
|
|
5
168
|
### The package is `fp-cloud-cli`; the command is still `fp`
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fp-cloud-cli
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.2
|
|
4
4
|
Summary: Command line client for FailproofAI Cloud — sessions, events, evaluations and audits from your terminal
|
|
5
5
|
Author-email: Failproof AI <failproofai@exosphere.host>
|
|
6
6
|
License: MIT License
|
|
@@ -51,7 +51,7 @@ Project-URL: Documentation, https://docs.befailproof.ai/reference/cloud-cli
|
|
|
51
51
|
Project-URL: Source, https://github.com/FailproofAI/failproofai
|
|
52
52
|
Project-URL: Changelog, https://github.com/FailproofAI/failproofai/blob/main/fp-cloud-cli/CHANGELOG.md
|
|
53
53
|
Keywords: agents,observability,ai,monitoring,failproofai,cli
|
|
54
|
-
Classifier: Development Status ::
|
|
54
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
55
55
|
Classifier: Environment :: Console
|
|
56
56
|
Classifier: Intended Audience :: Developers
|
|
57
57
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -166,7 +166,7 @@ fp keys list|show|create|update|disable|regenerate # API keys (secret shown
|
|
|
166
166
|
fp users list|show|create|update|disable|enable
|
|
167
167
|
fp settings list|schema|set
|
|
168
168
|
fp alerts list|show|create|update|delete|test
|
|
169
|
-
fp issues list|count|show|ack|assign|resolve|comment-add|comment-list|comment-delete|subscribe|subscribers|unsubscribe|open
|
|
169
|
+
fp issues list|count|show|ack|assign|resolve|close|archive|unarchive|clear|comment-add|comment-list|comment-delete|subscribe|subscribers|unsubscribe|open
|
|
170
170
|
fp audits list|show|create|edit|delete|run|runs|findings|finding # scheduled audits
|
|
171
171
|
fp audits ack|assign|resolve|dismiss|mute|reopen # triage a finding
|
|
172
172
|
fp audits context-show|context-set|context-refresh # reference context
|
|
@@ -215,6 +215,46 @@ fp fleet deploy ci-runner-01 --add no-force-push
|
|
|
215
215
|
fp guardrails summary --since 24h
|
|
216
216
|
```
|
|
217
217
|
|
|
218
|
+
A policy has a **kind**: `regex` (JavaScript — the default, and what every
|
|
219
|
+
earlier version is), `jev` (Jev checks only, from a JSON file of declarations —
|
|
220
|
+
each exactly a failproofai pack manifest's semantic entry), or `both`
|
|
221
|
+
(JavaScript whose verdict its own Jev checks may clear, deployed as one).
|
|
222
|
+
Jev checks run on FailproofAI Cloud; nothing is installed on the machine.
|
|
223
|
+
While a `jev` or `both` policy is deployed to a machine, FailproofAI Cloud
|
|
224
|
+
turns Jev on there and the machine sends each checked tool call to it; with
|
|
225
|
+
none, Jev is off. There is no Jev mode to set: trial a policy with `:observe`
|
|
226
|
+
(its checks are recorded, never applied), then deploy it as `enforce`, where
|
|
227
|
+
Jev's checks **block** calls as well as clearing the regex verdicts they
|
|
228
|
+
review.
|
|
229
|
+
|
|
230
|
+
```bash
|
|
231
|
+
fp policies publish prod-db-intent --kind jev --semantic ./checks.json
|
|
232
|
+
fp policies publish no-prod-db --kind both --source ./rule.mjs --semantic ./checks.json
|
|
233
|
+
fp fleet deploy ci-runner-01 --add prod-db-intent:observe
|
|
234
|
+
fp fleet show ci-runner-01 # its Jev mode, and any policy errors it reported
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
Published policy versions do not contain agent-routing code. Each assignment
|
|
238
|
+
can instead target every agent (the default), an integration including future
|
|
239
|
+
profiles, or a particular profile reported by that machine. The same target
|
|
240
|
+
applies to the regex and Jev parts of a `both` policy:
|
|
241
|
+
|
|
242
|
+
```bash
|
|
243
|
+
fp fleet deploy ci-runner-01 --add no-prod-db --target no-prod-db=hermes
|
|
244
|
+
fp fleet deploy ci-runner-01 --target no-prod-db=hermes/agt_1234567890abcdef
|
|
245
|
+
fp fleet deploy ci-runner-01 --all-agents no-prod-db # explicitly remove targeting
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
Repeat `--target` for multiple alternatives on one assignment. `fp fleet
|
|
249
|
+
show` and `fp fleet history` display the assigned scope; `--json` includes
|
|
250
|
+
`agentTargets` on each targeted policy. A machine must report agent-scoping
|
|
251
|
+
support before accepting a new target. When its identity cannot be resolved,
|
|
252
|
+
a targeted policy does not match; unscoped policy remains in force.
|
|
253
|
+
Newly installed shell hooks carry their settings scope. Reinstall older hooks
|
|
254
|
+
to add the hint; without it, or for an integration that cannot report the
|
|
255
|
+
source of multiple installed scopes, the machine reports
|
|
256
|
+
`agent_scope_unresolved` rather than guessing which profile ran the call.
|
|
257
|
+
|
|
218
258
|
**A deploy REPLACES a machine's whole policy set.** The server takes the full
|
|
219
259
|
list and does not merge, so `fleet deploy` reads what the machine currently runs,
|
|
220
260
|
applies your `--add`/`--remove`, prints the complete resulting set, and writes
|
|
@@ -247,6 +287,126 @@ These commands are **session-only** (`fp login`). They are absent from the
|
|
|
247
287
|
versioned API an API key authenticates against, so `--api-key` exits 2 with the
|
|
248
288
|
reason rather than failing at the request.
|
|
249
289
|
|
|
290
|
+
## Author an evaluator
|
|
291
|
+
|
|
292
|
+
`fp evaluators` authors hosted definitions; `fp evals` continues to read results.
|
|
293
|
+
Creation enables a definition by default. Check `fp evaluators instances` first:
|
|
294
|
+
Cloud needs an enabled managed instance. Writes need `evaluations:trigger`;
|
|
295
|
+
session dry-runs and `evaluators schema` also need `events:read`.
|
|
296
|
+
|
|
297
|
+
Write `error_free.py` as a restricted Python expression (no imports or functions):
|
|
298
|
+
|
|
299
|
+
```python
|
|
300
|
+
EvalResult(score=Score(1.0 if session.count("error") == 0 else 0.0), reasoning="Error-free session")
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
```bash
|
|
304
|
+
fp --json evaluators create --key error_free --name "Error free" --code error_free.py \
|
|
305
|
+
--result-kind score --dry-run-on run-001
|
|
306
|
+
fp --json evaluators dry-run --code error_free.py --session run-001
|
|
307
|
+
fp --json evaluators deploy error_free --code error_free.py --version 2 --dry-run-on run-001
|
|
308
|
+
fp --json evaluators list --key error_free --all
|
|
309
|
+
```
|
|
310
|
+
|
|
311
|
+
LLM judge (Cloud selects the model; judge dry-runs are refused):
|
|
312
|
+
|
|
313
|
+
```bash
|
|
314
|
+
fp --json evaluators create --key relevance --name Relevance --judge \
|
|
315
|
+
--criteria "Grade how directly the answer addresses the request" --threshold 0.7
|
|
316
|
+
fp --json evaluators deploy relevance --version 2 --judge \
|
|
317
|
+
--criteria "Grade relevance and completeness" --threshold 0.8
|
|
318
|
+
fp --json sessions re-evaluate run-001
|
|
319
|
+
fp --json sessions history run-001
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
Jev binary or graded rubric:
|
|
323
|
+
|
|
324
|
+
```bash
|
|
325
|
+
fp --json evaluators create --key resolved --name Resolved --jev noul \
|
|
326
|
+
--instructions "Was the request resolved?" --true "Resolved" --false "Unresolved"
|
|
327
|
+
fp --json evaluators dry-run --jev noul --instructions "Was the request resolved?" \
|
|
328
|
+
--true "Resolved" --false "Unresolved" --session run-001
|
|
329
|
+
fp --json evaluators deploy resolved --version 2 --jev score \
|
|
330
|
+
--instructions "Grade resolution quality" \
|
|
331
|
+
--level "Unresolved" --level "Partly resolved" --level "Fully resolved"
|
|
332
|
+
```
|
|
333
|
+
|
|
334
|
+
Jev dry-runs spend model budget. Rubrics need 3–5 distinct levels, worst first.
|
|
335
|
+
Thresholds are 0–1; judge/Jev always return a score. `deploy` creates the new
|
|
336
|
+
immutable version before disabling older enabled versions. If disabling fails,
|
|
337
|
+
the CLI names the published id; inspect `list --all` and recover explicitly.
|
|
338
|
+
`--backfill 7d` works for every kind (judge/Jev spend model budget); `evaluators backfill --from T --to T` accepts ≤90 days and `--dry-run` counts the work first.
|
|
339
|
+
|
|
340
|
+
Human output shows each `--dry-run-on` session's outcome before creation or
|
|
341
|
+
deployment. Any failed run stops the write; after reviewing the failure, an
|
|
342
|
+
explicit `--force` overrides this guard. Existing `--json` response shapes stay
|
|
343
|
+
unchanged.
|
|
344
|
+
|
|
345
|
+
AI drafting is session-only and writes an API spec that create/deploy accept:
|
|
346
|
+
|
|
347
|
+
```bash
|
|
348
|
+
fp login
|
|
349
|
+
fp --json evaluators draft "Measure error-free sessions with code" --kind code --out evaluator.json
|
|
350
|
+
# Review the spec, especially eval_key and evaluator_source, before enabling it.
|
|
351
|
+
fp --json evaluators create --file evaluator.json --dry-run-on run-001
|
|
352
|
+
fp --json evaluators dry-run --file evaluator.json
|
|
353
|
+
```
|
|
354
|
+
|
|
355
|
+
Spec fields: `eval_key`, `display_name`, `eval_version`, `result_kind`, `labels`,
|
|
356
|
+
`condition_source`, `evaluator_source`, `timeout_seconds`. Judge/Jev sources are
|
|
357
|
+
JSON envelope **strings**, not nested objects. `--file` is exclusive with discrete
|
|
358
|
+
spec flags; deploy may override a file's version. Files written by the CLI are
|
|
359
|
+
created exclusively, so existing files are never silently replaced.
|
|
360
|
+
See [the command reference](skill/references/commands.md#author-an-evaluator).
|
|
361
|
+
|
|
362
|
+
## Dashboard parity
|
|
363
|
+
|
|
364
|
+
```text
|
|
365
|
+
fp dashboards list|show|create|update|delete
|
|
366
|
+
fp dashboards tiles <id> list|add|update|remove|layout
|
|
367
|
+
fp sessions show|export|history|re-evaluate <id>
|
|
368
|
+
fp events latency
|
|
369
|
+
fp sentiment summary|messages|sessions|session <id>
|
|
370
|
+
fp query compose "question" [--run] # session
|
|
371
|
+
fp query export [<id>] [--sql SQL] --format csv|json [--out FILE]
|
|
372
|
+
fp alerts recipients
|
|
373
|
+
fp alerts issues <id> [--create --title ... --summary ...]
|
|
374
|
+
fp audits contracts list|show|create|update
|
|
375
|
+
fp policies author "intent"|backtest <file-or-id>|candidacy --issue <id> # session
|
|
376
|
+
fp agent brief [--range 24h|7d] # session
|
|
377
|
+
fp orgs create --name N [--slug S] # session
|
|
378
|
+
fp roles list|show|create|update|delete
|
|
379
|
+
fp users delete <id>
|
|
380
|
+
fp users defaults
|
|
381
|
+
fp usage windows
|
|
382
|
+
fp plan # session
|
|
383
|
+
fp plan claim <offer> # session
|
|
384
|
+
fp settings context-windows list|set <model> <tokens>|delete <model>
|
|
385
|
+
fp otel status|enable|disable
|
|
386
|
+
fp slack status|disconnect # session
|
|
387
|
+
```
|
|
388
|
+
|
|
389
|
+
`fp agent ask` runs artifact SQL through the permission-gated query runner and
|
|
390
|
+
renders tables/stats. It prompts before approving a write on a terminal.
|
|
391
|
+
Scripts and `--json` **decline** unless you explicitly pass `--approve`.
|
|
392
|
+
Normal create/update commands keep their existing non-interactive confirmation
|
|
393
|
+
behavior; assistant approvals are a separate safety boundary.
|
|
394
|
+
|
|
395
|
+
All new commands accept the global `--json` before the command. Session-only
|
|
396
|
+
commands fail before HTTP in API-key mode, with exit 2 and a `fp login` hint.
|
|
397
|
+
Unknown-route 404s exit 6 with “not available on this FailproofAI Cloud yet”.
|
|
398
|
+
Missing-record 404s also exit 6 but report that the record was not found.
|
|
399
|
+
`plan` and `plan claim` require a signed-in person and deny API-key mode before
|
|
400
|
+
HTTP with the standard `fp login` message.
|
|
401
|
+
Human screens use the mint/pink wordmark, grey labels, open tables and full ids;
|
|
402
|
+
existing JSON shapes are unchanged.
|
|
403
|
+
|
|
404
|
+
Human commands show summaries, tables and wrapped text rather than response
|
|
405
|
+
objects. Use global `--json` for raw API results. For `sessions export` or
|
|
406
|
+
`query export --format json`, use `--out FILE` to save the JSON; without either
|
|
407
|
+
option, the terminal shows a readable session or query view. CSV exports remain
|
|
408
|
+
plain CSV for piping.
|
|
409
|
+
|
|
250
410
|
## Configuration
|
|
251
411
|
|
|
252
412
|
| Setting | Flag | Env var | Default |
|
|
@@ -332,4 +492,3 @@ working as an opt-out if it is ever switched back on. See
|
|
|
332
492
|
cd fp-cloud-cli
|
|
333
493
|
uv run --extra dev pytest
|
|
334
494
|
```
|
|
335
|
-
|
|
@@ -90,7 +90,7 @@ fp keys list|show|create|update|disable|regenerate # API keys (secret shown
|
|
|
90
90
|
fp users list|show|create|update|disable|enable
|
|
91
91
|
fp settings list|schema|set
|
|
92
92
|
fp alerts list|show|create|update|delete|test
|
|
93
|
-
fp issues list|count|show|ack|assign|resolve|comment-add|comment-list|comment-delete|subscribe|subscribers|unsubscribe|open
|
|
93
|
+
fp issues list|count|show|ack|assign|resolve|close|archive|unarchive|clear|comment-add|comment-list|comment-delete|subscribe|subscribers|unsubscribe|open
|
|
94
94
|
fp audits list|show|create|edit|delete|run|runs|findings|finding # scheduled audits
|
|
95
95
|
fp audits ack|assign|resolve|dismiss|mute|reopen # triage a finding
|
|
96
96
|
fp audits context-show|context-set|context-refresh # reference context
|
|
@@ -139,6 +139,46 @@ fp fleet deploy ci-runner-01 --add no-force-push
|
|
|
139
139
|
fp guardrails summary --since 24h
|
|
140
140
|
```
|
|
141
141
|
|
|
142
|
+
A policy has a **kind**: `regex` (JavaScript — the default, and what every
|
|
143
|
+
earlier version is), `jev` (Jev checks only, from a JSON file of declarations —
|
|
144
|
+
each exactly a failproofai pack manifest's semantic entry), or `both`
|
|
145
|
+
(JavaScript whose verdict its own Jev checks may clear, deployed as one).
|
|
146
|
+
Jev checks run on FailproofAI Cloud; nothing is installed on the machine.
|
|
147
|
+
While a `jev` or `both` policy is deployed to a machine, FailproofAI Cloud
|
|
148
|
+
turns Jev on there and the machine sends each checked tool call to it; with
|
|
149
|
+
none, Jev is off. There is no Jev mode to set: trial a policy with `:observe`
|
|
150
|
+
(its checks are recorded, never applied), then deploy it as `enforce`, where
|
|
151
|
+
Jev's checks **block** calls as well as clearing the regex verdicts they
|
|
152
|
+
review.
|
|
153
|
+
|
|
154
|
+
```bash
|
|
155
|
+
fp policies publish prod-db-intent --kind jev --semantic ./checks.json
|
|
156
|
+
fp policies publish no-prod-db --kind both --source ./rule.mjs --semantic ./checks.json
|
|
157
|
+
fp fleet deploy ci-runner-01 --add prod-db-intent:observe
|
|
158
|
+
fp fleet show ci-runner-01 # its Jev mode, and any policy errors it reported
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Published policy versions do not contain agent-routing code. Each assignment
|
|
162
|
+
can instead target every agent (the default), an integration including future
|
|
163
|
+
profiles, or a particular profile reported by that machine. The same target
|
|
164
|
+
applies to the regex and Jev parts of a `both` policy:
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
fp fleet deploy ci-runner-01 --add no-prod-db --target no-prod-db=hermes
|
|
168
|
+
fp fleet deploy ci-runner-01 --target no-prod-db=hermes/agt_1234567890abcdef
|
|
169
|
+
fp fleet deploy ci-runner-01 --all-agents no-prod-db # explicitly remove targeting
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
Repeat `--target` for multiple alternatives on one assignment. `fp fleet
|
|
173
|
+
show` and `fp fleet history` display the assigned scope; `--json` includes
|
|
174
|
+
`agentTargets` on each targeted policy. A machine must report agent-scoping
|
|
175
|
+
support before accepting a new target. When its identity cannot be resolved,
|
|
176
|
+
a targeted policy does not match; unscoped policy remains in force.
|
|
177
|
+
Newly installed shell hooks carry their settings scope. Reinstall older hooks
|
|
178
|
+
to add the hint; without it, or for an integration that cannot report the
|
|
179
|
+
source of multiple installed scopes, the machine reports
|
|
180
|
+
`agent_scope_unresolved` rather than guessing which profile ran the call.
|
|
181
|
+
|
|
142
182
|
**A deploy REPLACES a machine's whole policy set.** The server takes the full
|
|
143
183
|
list and does not merge, so `fleet deploy` reads what the machine currently runs,
|
|
144
184
|
applies your `--add`/`--remove`, prints the complete resulting set, and writes
|
|
@@ -171,6 +211,126 @@ These commands are **session-only** (`fp login`). They are absent from the
|
|
|
171
211
|
versioned API an API key authenticates against, so `--api-key` exits 2 with the
|
|
172
212
|
reason rather than failing at the request.
|
|
173
213
|
|
|
214
|
+
## Author an evaluator
|
|
215
|
+
|
|
216
|
+
`fp evaluators` authors hosted definitions; `fp evals` continues to read results.
|
|
217
|
+
Creation enables a definition by default. Check `fp evaluators instances` first:
|
|
218
|
+
Cloud needs an enabled managed instance. Writes need `evaluations:trigger`;
|
|
219
|
+
session dry-runs and `evaluators schema` also need `events:read`.
|
|
220
|
+
|
|
221
|
+
Write `error_free.py` as a restricted Python expression (no imports or functions):
|
|
222
|
+
|
|
223
|
+
```python
|
|
224
|
+
EvalResult(score=Score(1.0 if session.count("error") == 0 else 0.0), reasoning="Error-free session")
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
fp --json evaluators create --key error_free --name "Error free" --code error_free.py \
|
|
229
|
+
--result-kind score --dry-run-on run-001
|
|
230
|
+
fp --json evaluators dry-run --code error_free.py --session run-001
|
|
231
|
+
fp --json evaluators deploy error_free --code error_free.py --version 2 --dry-run-on run-001
|
|
232
|
+
fp --json evaluators list --key error_free --all
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
LLM judge (Cloud selects the model; judge dry-runs are refused):
|
|
236
|
+
|
|
237
|
+
```bash
|
|
238
|
+
fp --json evaluators create --key relevance --name Relevance --judge \
|
|
239
|
+
--criteria "Grade how directly the answer addresses the request" --threshold 0.7
|
|
240
|
+
fp --json evaluators deploy relevance --version 2 --judge \
|
|
241
|
+
--criteria "Grade relevance and completeness" --threshold 0.8
|
|
242
|
+
fp --json sessions re-evaluate run-001
|
|
243
|
+
fp --json sessions history run-001
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
Jev binary or graded rubric:
|
|
247
|
+
|
|
248
|
+
```bash
|
|
249
|
+
fp --json evaluators create --key resolved --name Resolved --jev noul \
|
|
250
|
+
--instructions "Was the request resolved?" --true "Resolved" --false "Unresolved"
|
|
251
|
+
fp --json evaluators dry-run --jev noul --instructions "Was the request resolved?" \
|
|
252
|
+
--true "Resolved" --false "Unresolved" --session run-001
|
|
253
|
+
fp --json evaluators deploy resolved --version 2 --jev score \
|
|
254
|
+
--instructions "Grade resolution quality" \
|
|
255
|
+
--level "Unresolved" --level "Partly resolved" --level "Fully resolved"
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
Jev dry-runs spend model budget. Rubrics need 3–5 distinct levels, worst first.
|
|
259
|
+
Thresholds are 0–1; judge/Jev always return a score. `deploy` creates the new
|
|
260
|
+
immutable version before disabling older enabled versions. If disabling fails,
|
|
261
|
+
the CLI names the published id; inspect `list --all` and recover explicitly.
|
|
262
|
+
`--backfill 7d` works for every kind (judge/Jev spend model budget); `evaluators backfill --from T --to T` accepts ≤90 days and `--dry-run` counts the work first.
|
|
263
|
+
|
|
264
|
+
Human output shows each `--dry-run-on` session's outcome before creation or
|
|
265
|
+
deployment. Any failed run stops the write; after reviewing the failure, an
|
|
266
|
+
explicit `--force` overrides this guard. Existing `--json` response shapes stay
|
|
267
|
+
unchanged.
|
|
268
|
+
|
|
269
|
+
AI drafting is session-only and writes an API spec that create/deploy accept:
|
|
270
|
+
|
|
271
|
+
```bash
|
|
272
|
+
fp login
|
|
273
|
+
fp --json evaluators draft "Measure error-free sessions with code" --kind code --out evaluator.json
|
|
274
|
+
# Review the spec, especially eval_key and evaluator_source, before enabling it.
|
|
275
|
+
fp --json evaluators create --file evaluator.json --dry-run-on run-001
|
|
276
|
+
fp --json evaluators dry-run --file evaluator.json
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
Spec fields: `eval_key`, `display_name`, `eval_version`, `result_kind`, `labels`,
|
|
280
|
+
`condition_source`, `evaluator_source`, `timeout_seconds`. Judge/Jev sources are
|
|
281
|
+
JSON envelope **strings**, not nested objects. `--file` is exclusive with discrete
|
|
282
|
+
spec flags; deploy may override a file's version. Files written by the CLI are
|
|
283
|
+
created exclusively, so existing files are never silently replaced.
|
|
284
|
+
See [the command reference](skill/references/commands.md#author-an-evaluator).
|
|
285
|
+
|
|
286
|
+
## Dashboard parity
|
|
287
|
+
|
|
288
|
+
```text
|
|
289
|
+
fp dashboards list|show|create|update|delete
|
|
290
|
+
fp dashboards tiles <id> list|add|update|remove|layout
|
|
291
|
+
fp sessions show|export|history|re-evaluate <id>
|
|
292
|
+
fp events latency
|
|
293
|
+
fp sentiment summary|messages|sessions|session <id>
|
|
294
|
+
fp query compose "question" [--run] # session
|
|
295
|
+
fp query export [<id>] [--sql SQL] --format csv|json [--out FILE]
|
|
296
|
+
fp alerts recipients
|
|
297
|
+
fp alerts issues <id> [--create --title ... --summary ...]
|
|
298
|
+
fp audits contracts list|show|create|update
|
|
299
|
+
fp policies author "intent"|backtest <file-or-id>|candidacy --issue <id> # session
|
|
300
|
+
fp agent brief [--range 24h|7d] # session
|
|
301
|
+
fp orgs create --name N [--slug S] # session
|
|
302
|
+
fp roles list|show|create|update|delete
|
|
303
|
+
fp users delete <id>
|
|
304
|
+
fp users defaults
|
|
305
|
+
fp usage windows
|
|
306
|
+
fp plan # session
|
|
307
|
+
fp plan claim <offer> # session
|
|
308
|
+
fp settings context-windows list|set <model> <tokens>|delete <model>
|
|
309
|
+
fp otel status|enable|disable
|
|
310
|
+
fp slack status|disconnect # session
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
`fp agent ask` runs artifact SQL through the permission-gated query runner and
|
|
314
|
+
renders tables/stats. It prompts before approving a write on a terminal.
|
|
315
|
+
Scripts and `--json` **decline** unless you explicitly pass `--approve`.
|
|
316
|
+
Normal create/update commands keep their existing non-interactive confirmation
|
|
317
|
+
behavior; assistant approvals are a separate safety boundary.
|
|
318
|
+
|
|
319
|
+
All new commands accept the global `--json` before the command. Session-only
|
|
320
|
+
commands fail before HTTP in API-key mode, with exit 2 and a `fp login` hint.
|
|
321
|
+
Unknown-route 404s exit 6 with “not available on this FailproofAI Cloud yet”.
|
|
322
|
+
Missing-record 404s also exit 6 but report that the record was not found.
|
|
323
|
+
`plan` and `plan claim` require a signed-in person and deny API-key mode before
|
|
324
|
+
HTTP with the standard `fp login` message.
|
|
325
|
+
Human screens use the mint/pink wordmark, grey labels, open tables and full ids;
|
|
326
|
+
existing JSON shapes are unchanged.
|
|
327
|
+
|
|
328
|
+
Human commands show summaries, tables and wrapped text rather than response
|
|
329
|
+
objects. Use global `--json` for raw API results. For `sessions export` or
|
|
330
|
+
`query export --format json`, use `--out FILE` to save the JSON; without either
|
|
331
|
+
option, the terminal shows a readable session or query view. CSV exports remain
|
|
332
|
+
plain CSV for piping.
|
|
333
|
+
|
|
174
334
|
## Configuration
|
|
175
335
|
|
|
176
336
|
| Setting | Flag | Env var | Default |
|
|
@@ -256,4 +416,3 @@ working as an opt-out if it is ever switched back on. See
|
|
|
256
416
|
cd fp-cloud-cli
|
|
257
417
|
uv run --extra dev pytest
|
|
258
418
|
```
|
|
259
|
-
|
|
@@ -11,9 +11,10 @@ compiles, and every happy path still passes:
|
|
|
11
11
|
should print ``✗ …`` and exit 5 exits **1 with an empty stderr**, and the
|
|
12
12
|
``rich_format_error`` hook in ``app.py`` (reached only *after* that catch) never
|
|
13
13
|
runs. Same for the ``UsageError``/``BadParameter`` we raise by hand.
|
|
14
|
-
* ``Abort`` — ``typer.prompt`` raises
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
* ``Abort`` — ``typer.prompt`` raises the ``Abort`` **typer** exports on closed stdin
|
|
15
|
+
(0.27.2 moved that class out of the vendored Click entirely), so a pip-Click
|
|
16
|
+
``except click.Abort`` stops matching and the clean "no TTY, pass a slug" usage error
|
|
17
|
+
becomes a bare abort.
|
|
17
18
|
* Options — Typer 0.26+ has no ``Option`` class in its vendored Click *at all*: every
|
|
18
19
|
option in a Typer-built tree is a ``typer.core.TyperOption``, subclassing
|
|
19
20
|
``Parameter`` directly. ``isinstance(param, click.Option)`` is then quietly always
|
|
@@ -27,14 +28,36 @@ again, and that our errors are still the class Typer catches.
|
|
|
27
28
|
|
|
28
29
|
from __future__ import annotations
|
|
29
30
|
|
|
31
|
+
# `Abort` is resolved from `typer` itself rather than from either Click, because it is
|
|
32
|
+
# the one symbol here that is not stably a Click class: typer 0.27.2 moved it out of the
|
|
33
|
+
# vendored `typer._click.exceptions` into a plain `typer.exceptions.Abort(RuntimeError)`.
|
|
34
|
+
# `typer.Abort` tracks that move — it is `click.Abort` before 0.26, the vendored class
|
|
35
|
+
# through 0.27.1, the RuntimeError from 0.27.2 — and it is by construction the class
|
|
36
|
+
# `typer.prompt` raises and typer's own `_main` catches, which is the only property
|
|
37
|
+
# `select.py`'s `except click.Abort` needs.
|
|
38
|
+
from typer import Abort as Abort
|
|
39
|
+
|
|
40
|
+
# Pick the Click ONCE, on whether typer vendors one at all, then import every symbol
|
|
41
|
+
# from that choice. Deciding per symbol — a single `try` around the whole vendored
|
|
42
|
+
# import block, falling back to pip `click` on any ImportError — is what shipped
|
|
43
|
+
# through 0.27.1 and it failed exactly as silently as this module exists to prevent:
|
|
44
|
+
# 0.27.2 removing `typer._click.exceptions.Abort` made that one missing name rebind ALL
|
|
45
|
+
# SIX symbols to pip Click, so every typed error escaped Typer's handler as exit 1 with
|
|
46
|
+
# an empty stderr. A name that goes missing inside the chosen Click must raise here, at
|
|
47
|
+
# import, where it is a CLI that refuses to start rather than one that silently stops
|
|
48
|
+
# reporting errors.
|
|
30
49
|
try:
|
|
31
|
-
# typer >= 0.26
|
|
32
|
-
# Click in play, so binding to it here would reintroduce the whole class of bug.
|
|
33
|
-
from typer._click import ClickException, Command, Parameter
|
|
34
|
-
from typer._click.exceptions import Abort, BadParameter, UsageError
|
|
50
|
+
import typer._click as _typer_click # typer >= 0.26 vendors its own Click
|
|
35
51
|
except ImportError: # typer < 0.26 drives the pip `click` distribution directly
|
|
52
|
+
_typer_click = None # type: ignore[assignment]
|
|
53
|
+
|
|
54
|
+
if _typer_click is not None:
|
|
55
|
+
# pip `click` may well still be installed — it is simply not the Click in play, so
|
|
56
|
+
# binding to it here would reintroduce the whole class of bug.
|
|
57
|
+
from typer._click import ClickException, Command, Parameter
|
|
58
|
+
from typer._click.exceptions import BadParameter, UsageError
|
|
59
|
+
else:
|
|
36
60
|
from click import ( # type: ignore[assignment]
|
|
37
|
-
Abort,
|
|
38
61
|
BadParameter,
|
|
39
62
|
ClickException,
|
|
40
63
|
Command,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.0.2"
|