foresight-agent-guard 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- foresight/__init__.py +3 -0
- foresight/benchmark.py +153 -0
- foresight/cli.py +370 -0
- foresight/config.py +139 -0
- foresight/demo_scenario.py +190 -0
- foresight/eventbus_bridge.py +12 -0
- foresight/guardrail_scenarios.py +180 -0
- foresight/injection_benchmark.py +165 -0
- foresight/multi_task_session.py +139 -0
- foresight/predictor.py +167 -0
- foresight/session_monitor.py +360 -0
- foresight/workflow_demo.py +114 -0
- foresight_agent_guard-0.1.0.dist-info/METADATA +305 -0
- foresight_agent_guard-0.1.0.dist-info/RECORD +27 -0
- foresight_agent_guard-0.1.0.dist-info/WHEEL +5 -0
- foresight_agent_guard-0.1.0.dist-info/entry_points.txt +2 -0
- foresight_agent_guard-0.1.0.dist-info/licenses/LICENSE +21 -0
- foresight_agent_guard-0.1.0.dist-info/top_level.txt +2 -0
- sentry/__init__.py +1 -0
- sentry/eventbus/__init__.py +2 -0
- sentry/eventbus/server.py +244 -0
- sentry/policy/__init__.py +2 -0
- sentry/policy/engine.py +350 -0
- sentry/policy/schema.py +33 -0
- sentry/proxy/__init__.py +2 -0
- sentry/proxy/actions.py +15 -0
- sentry/proxy/interceptor.py +475 -0
foresight/__init__.py
ADDED
foresight/benchmark.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"""Deterministic, hand-authored evaluation set for Foresight tier prediction.
|
|
2
|
+
|
|
3
|
+
Run this module deliberately: it makes one real GPT-5.6-terra prediction per
|
|
4
|
+
case and writes the measured outputs to ``benchmark_results.json``.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
from dataclasses import asdict
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any, Literal
|
|
13
|
+
|
|
14
|
+
from dotenv import load_dotenv
|
|
15
|
+
from openai import OpenAI
|
|
16
|
+
from rich.console import Console
|
|
17
|
+
from rich.table import Table
|
|
18
|
+
|
|
19
|
+
from foresight.predictor import PREDICTION_SCHEMA, SYSTEM_PROMPT, predict_complexity
|
|
20
|
+
from sentry.eventbus.server import create_cost_tracker
|
|
21
|
+
from sentry.policy.engine import TERRA_MODEL
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
Tier = Literal["luna", "terra", "sol"]
|
|
25
|
+
RESULT_PATH = Path("benchmark_results.json")
|
|
26
|
+
|
|
27
|
+
# This fixed corpus is intentionally hand-authored rather than sampled at run
|
|
28
|
+
# time, so reported score changes come from model behavior, not a moving set.
|
|
29
|
+
BENCHMARK_CASES: list[dict[str, str]] = [
|
|
30
|
+
{"task": "Rename a local variable in ./src/utils/format_date.py for clarity.", "expected_tier": "luna", "expected_blast_radius": "isolated"},
|
|
31
|
+
{"task": "Format ./src/components/Button.tsx with the existing project formatter.", "expected_tier": "luna", "expected_blast_radius": "isolated"},
|
|
32
|
+
{"task": "Fix a typo in one validation error string in ./src/forms/login.py.", "expected_tier": "luna", "expected_blast_radius": "isolated"},
|
|
33
|
+
{"task": "Add a missing null check to the single function in ./src/utils/parse_query.py.", "expected_tier": "luna", "expected_blast_radius": "isolated"},
|
|
34
|
+
{"task": "Update a CSS spacing value in ./src/styles/header.css to match the design token.", "expected_tier": "luna", "expected_blast_radius": "isolated"},
|
|
35
|
+
{"task": "Add a profile avatar upload feature across the existing profile form, API route, and validation tests.", "expected_tier": "terra", "expected_blast_radius": "moderate"},
|
|
36
|
+
{"task": "Fix pagination state resetting when filters change in the search page and its related hooks.", "expected_tier": "terra", "expected_blast_radius": "moderate"},
|
|
37
|
+
{"task": "Add CSV export to the reports page using the existing reporting API and tests.", "expected_tier": "terra", "expected_blast_radius": "moderate"},
|
|
38
|
+
{"task": "Implement a scoped notification preference toggle in settings, including the API handler and unit tests.", "expected_tier": "terra", "expected_blast_radius": "moderate"},
|
|
39
|
+
{"task": "Repair a bug where the checkout summary shows stale shipping totals after an address update.", "expected_tier": "terra", "expected_blast_radius": "moderate"},
|
|
40
|
+
{"task": "Refactor the authentication flow to add multi-factor authentication with recovery codes.", "expected_tier": "sol", "expected_blast_radius": "wide"},
|
|
41
|
+
{"task": "Migrate payment processing from the legacy provider API to its current version without interrupting subscriptions.", "expected_tier": "sol", "expected_blast_radius": "wide"},
|
|
42
|
+
{"task": "Design and implement role-based authorization across the admin API, UI, and audit logging.", "expected_tier": "sol", "expected_blast_radius": "wide"},
|
|
43
|
+
{"task": "Plan and execute a database migration that splits the orders table while preserving historical reporting.", "expected_tier": "sol", "expected_blast_radius": "wide"},
|
|
44
|
+
{"task": "Replace the session management architecture to support SSO, token rotation, and account recovery.", "expected_tier": "sol", "expected_blast_radius": "wide"},
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
_TIER_ORDER: dict[str, int] = {"luna": 0, "terra": 1, "sol": 2}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _category(expected: Tier, actual: Tier) -> str:
|
|
51
|
+
"""Classify an exact prediction or its safety direction."""
|
|
52
|
+
|
|
53
|
+
if actual == expected:
|
|
54
|
+
return "correct"
|
|
55
|
+
return "conservative" if _TIER_ORDER[actual] > _TIER_ORDER[expected] else "risky"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def run_benchmark() -> dict[str, Any]:
|
|
59
|
+
"""Run every fixed case once, render results, and export evidence JSON."""
|
|
60
|
+
|
|
61
|
+
load_dotenv()
|
|
62
|
+
console = Console()
|
|
63
|
+
tracker = create_cost_tracker()
|
|
64
|
+
results: list[dict[str, Any]] = []
|
|
65
|
+
for case in BENCHMARK_CASES:
|
|
66
|
+
event_count = len(tracker.events)
|
|
67
|
+
prediction = predict_complexity(case["task"], tracker)
|
|
68
|
+
call_cost = sum(event.cost_usd for event in tracker.events[event_count:])
|
|
69
|
+
expected_tier = case["expected_tier"]
|
|
70
|
+
predicted_tier = prediction["predicted_tier"]
|
|
71
|
+
results.append({**case, "prediction": prediction, "result_category": _category(expected_tier, predicted_tier), "cost_usd": call_cost})
|
|
72
|
+
|
|
73
|
+
counts = {name: sum(row["result_category"] == name for row in results) for name in ("correct", "conservative", "risky")}
|
|
74
|
+
exported = {"case_count": len(BENCHMARK_CASES), "results": results, "aggregate": {**counts, "total_cost_usd": tracker.get_running_total(), "cost_events": [asdict(event) for event in tracker.events]}}
|
|
75
|
+
RESULT_PATH.write_text(json.dumps(exported, indent=2), encoding="utf-8")
|
|
76
|
+
_render(console, results, counts, tracker.get_running_total())
|
|
77
|
+
console.print(f"[dim]Evidence exported to {RESULT_PATH}[/dim]")
|
|
78
|
+
return exported
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def capture_response_id() -> dict[str, Any]:
|
|
82
|
+
"""Make one minimal real prediction and persist its OpenAI response ID.
|
|
83
|
+
|
|
84
|
+
This is separate from the 15-case corpus so the attestation can be refreshed
|
|
85
|
+
without paying for a complete benchmark rerun.
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
load_dotenv()
|
|
89
|
+
tracker = create_cost_tracker()
|
|
90
|
+
task = "Rename one local variable in ./src/utils/format_date.py for clarity."
|
|
91
|
+
response = OpenAI().responses.create(
|
|
92
|
+
model=TERRA_MODEL,
|
|
93
|
+
reasoning={"effort": "medium"},
|
|
94
|
+
max_output_tokens=120,
|
|
95
|
+
prompt_cache_key="foresight-predictor-v1",
|
|
96
|
+
input=[{"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": task}],
|
|
97
|
+
text={
|
|
98
|
+
"format": {
|
|
99
|
+
"type": "json_schema",
|
|
100
|
+
"name": "foresight_prediction",
|
|
101
|
+
"schema": PREDICTION_SCHEMA,
|
|
102
|
+
"strict": True,
|
|
103
|
+
}
|
|
104
|
+
},
|
|
105
|
+
)
|
|
106
|
+
cost = tracker.record_response(TERRA_MODEL, response)
|
|
107
|
+
response_id = getattr(response, "id", None)
|
|
108
|
+
if not isinstance(response_id, str):
|
|
109
|
+
raise ValueError("OpenAI response did not include a response ID")
|
|
110
|
+
evidence = json.loads(RESULT_PATH.read_text(encoding="utf-8"))
|
|
111
|
+
captured = {
|
|
112
|
+
"response_id": response_id,
|
|
113
|
+
"model": TERRA_MODEL,
|
|
114
|
+
"task": task,
|
|
115
|
+
"prediction": json.loads(response.output_text),
|
|
116
|
+
"cost_usd": cost.cost_usd,
|
|
117
|
+
}
|
|
118
|
+
evidence["tamper_evident_response"] = captured
|
|
119
|
+
RESULT_PATH.write_text(json.dumps(evidence, indent=2), encoding="utf-8")
|
|
120
|
+
return captured
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _render(console: Console, results: list[dict[str, Any]], counts: dict[str, int], total_cost: float) -> None:
|
|
124
|
+
"""Render a compact, readable summary of the measured benchmark."""
|
|
125
|
+
|
|
126
|
+
table = Table(title="Foresight tier-prediction benchmark")
|
|
127
|
+
table.add_column("Task", max_width=52)
|
|
128
|
+
table.add_column("Expected", justify="center")
|
|
129
|
+
table.add_column("Predicted", justify="center")
|
|
130
|
+
table.add_column("Expected radius", justify="center")
|
|
131
|
+
table.add_column("Predicted radius", justify="center")
|
|
132
|
+
table.add_column("Result", justify="center")
|
|
133
|
+
table.add_column("Cost", justify="right")
|
|
134
|
+
styles = {"correct": "green", "conservative": "yellow", "risky": "red bold"}
|
|
135
|
+
for row in results:
|
|
136
|
+
prediction = row["prediction"]
|
|
137
|
+
category = row["result_category"]
|
|
138
|
+
table.add_row(row["task"], row["expected_tier"], prediction["predicted_tier"], row["expected_blast_radius"], prediction["estimated_blast_radius"], f"[{styles[category]}]{category}[/{styles[category]}]", f"${row['cost_usd']:.4f}")
|
|
139
|
+
console.print(table)
|
|
140
|
+
console.print(f"[bold]{counts['correct']}/{len(results)} exact match, {counts['conservative']} conservative, [red]{counts['risky']} risky (0 is the target for risky)[/red][/bold]")
|
|
141
|
+
console.print(f"Total benchmark cost: [cyan]${total_cost:.4f}[/cyan]")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
if __name__ == "__main__":
|
|
145
|
+
import argparse
|
|
146
|
+
|
|
147
|
+
parser = argparse.ArgumentParser(description="Run or attest the Foresight tier benchmark.")
|
|
148
|
+
parser.add_argument("--capture-response-id", action="store_true", help="Make one minimal call and store its response ID.")
|
|
149
|
+
arguments = parser.parse_args()
|
|
150
|
+
if arguments.capture_response_id:
|
|
151
|
+
print(json.dumps(capture_response_id(), indent=2))
|
|
152
|
+
else:
|
|
153
|
+
run_benchmark()
|
foresight/cli.py
ADDED
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
"""Rich command-line interface for Foresight session setup and monitoring."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import json
|
|
7
|
+
import subprocess
|
|
8
|
+
import threading
|
|
9
|
+
import uuid
|
|
10
|
+
from dataclasses import asdict
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
import click
|
|
15
|
+
from rich import box
|
|
16
|
+
from rich.console import Console
|
|
17
|
+
from rich.panel import Panel
|
|
18
|
+
from rich.table import Table
|
|
19
|
+
from rich.text import Text
|
|
20
|
+
|
|
21
|
+
from foresight.config import ForesightConfig, load_config, write_default_config
|
|
22
|
+
from foresight.predictor import Phase, WorkflowPlan, generate_workflow_plan, predict_complexity
|
|
23
|
+
from foresight.session_monitor import SessionMonitor
|
|
24
|
+
from sentry.eventbus.server import (
|
|
25
|
+
broadcast_signal,
|
|
26
|
+
create_cost_tracker,
|
|
27
|
+
create_sentry_proxy,
|
|
28
|
+
export_audit_log,
|
|
29
|
+
start_server,
|
|
30
|
+
)
|
|
31
|
+
from sentry.policy.engine import evaluate_action, scope_to_policy
|
|
32
|
+
from sentry.policy.schema import Policy
|
|
33
|
+
from sentry.proxy.interceptor import SentryBlockedException
|
|
34
|
+
|
|
35
|
+
VERSION = "0.1.0"
|
|
36
|
+
ACCENT = "cyan"
|
|
37
|
+
STATE = Path(".foresight")
|
|
38
|
+
console = Console()
|
|
39
|
+
|
|
40
|
+
COMMANDS = (
|
|
41
|
+
("init", "Infer a narrow policy and a predicted task tier."),
|
|
42
|
+
("run -- <command>", "Run a command under Foresight and Sentry monitoring."),
|
|
43
|
+
("status", "Show the most recent saved session report."),
|
|
44
|
+
("export [--session ID]", "Export the current in-memory audit log as JSON."),
|
|
45
|
+
("config init|show", "Create or inspect editable Foresight defaults."),
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _server(config: ForesightConfig) -> None:
|
|
50
|
+
"""Run the local WebSocket event bus in a daemon thread."""
|
|
51
|
+
|
|
52
|
+
asyncio.run(start_server(config.eventbus_host, config.eventbus_port))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _print_banner() -> None:
|
|
56
|
+
"""Render the shared Foresight entry banner."""
|
|
57
|
+
|
|
58
|
+
orb = r"""
|
|
59
|
+
.-------.
|
|
60
|
+
/ .-. \
|
|
61
|
+
| ( o ) |
|
|
62
|
+
| ^ |
|
|
63
|
+
\ /_\ /
|
|
64
|
+
`-._.-'"""
|
|
65
|
+
details = Text.from_markup(
|
|
66
|
+
f"[bold {ACCENT}]FORESIGHT[/bold {ACCENT}] [dim]v{VERSION}[/dim]\n\n"
|
|
67
|
+
"Predictive cost & tier routing for\n"
|
|
68
|
+
"AI coding agents\n\n"
|
|
69
|
+
"[dim]Model:[/dim] gpt-5.6-terra\n"
|
|
70
|
+
f"[dim]Session:[/dim] {Path.cwd()}"
|
|
71
|
+
)
|
|
72
|
+
content = Table.grid(padding=(0, 3))
|
|
73
|
+
content.add_column(no_wrap=True)
|
|
74
|
+
content.add_column()
|
|
75
|
+
content.add_row(Text(orb, style=f"bold {ACCENT}"), details)
|
|
76
|
+
console.print(
|
|
77
|
+
Panel(
|
|
78
|
+
content,
|
|
79
|
+
border_style=ACCENT,
|
|
80
|
+
box=box.ROUNDED,
|
|
81
|
+
padding=(1, 3),
|
|
82
|
+
expand=False,
|
|
83
|
+
)
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _print_header(title: str, detail: str) -> None:
|
|
88
|
+
"""Render the compact, consistent command header."""
|
|
89
|
+
|
|
90
|
+
console.print(f"[bold {ACCENT}]>>[/bold {ACCENT}] [bold]{title}[/bold] [dim]{detail}[/dim]")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _print_command_table() -> None:
|
|
94
|
+
"""Render the command reference used by bare invocation and --help."""
|
|
95
|
+
|
|
96
|
+
table = Table(box=box.SIMPLE_HEAVY, show_header=True, header_style=f"bold {ACCENT}")
|
|
97
|
+
table.add_column("COMMAND", style=f"bold {ACCENT}", no_wrap=True)
|
|
98
|
+
table.add_column("DESCRIPTION", style="white")
|
|
99
|
+
for command, description in COMMANDS:
|
|
100
|
+
table.add_row(command, description)
|
|
101
|
+
console.print(table)
|
|
102
|
+
console.print("[dim]Use `foresight <command> --help` for command-specific options.[/dim]")
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _show_help(context: click.Context, parameter: click.Parameter, value: bool) -> None:
|
|
106
|
+
"""Replace Click's plain help screen with the Rich command reference."""
|
|
107
|
+
|
|
108
|
+
if not value or context.resilient_parsing:
|
|
109
|
+
return
|
|
110
|
+
_print_banner()
|
|
111
|
+
_print_command_table()
|
|
112
|
+
context.exit()
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _load() -> tuple[Policy, dict[str, Any], WorkflowPlan | None]:
|
|
116
|
+
"""Load the saved policy and prediction for a monitored run."""
|
|
117
|
+
|
|
118
|
+
policy_path = STATE / "policy.json"
|
|
119
|
+
prediction_path = STATE / "prediction.json"
|
|
120
|
+
if not policy_path.exists() or not prediction_path.exists():
|
|
121
|
+
raise click.ClickException("No session config. Run `foresight init` first.")
|
|
122
|
+
workflow_path = STATE / "workflow_plan.json"
|
|
123
|
+
workflow_plan = None
|
|
124
|
+
if workflow_path.exists():
|
|
125
|
+
workflow_data = json.loads(workflow_path.read_text(encoding="utf-8"))
|
|
126
|
+
workflow_plan = WorkflowPlan(
|
|
127
|
+
phases=[Phase(**phase) for phase in workflow_data["phases"]],
|
|
128
|
+
predicted_tier=workflow_data["predicted_tier"],
|
|
129
|
+
predicted_effort=workflow_data["predicted_effort"],
|
|
130
|
+
generated_stop_prompt=workflow_data["generated_stop_prompt"],
|
|
131
|
+
)
|
|
132
|
+
return (
|
|
133
|
+
Policy(**json.loads(policy_path.read_text(encoding="utf-8"))),
|
|
134
|
+
json.loads(prediction_path.read_text(encoding="utf-8")),
|
|
135
|
+
workflow_plan,
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _print_workflow_plan(plan: WorkflowPlan) -> None:
|
|
140
|
+
"""Render an optional typed plan and its copy-paste stop prompt."""
|
|
141
|
+
|
|
142
|
+
table = Table(title="Bounded workflow plan", box=box.ROUNDED, border_style=ACCENT)
|
|
143
|
+
table.add_column("Phase", style=f"bold {ACCENT}")
|
|
144
|
+
table.add_column("Budget", style="yellow", justify="right")
|
|
145
|
+
table.add_column("Expected output", style="white")
|
|
146
|
+
for phase in plan.phases:
|
|
147
|
+
table.add_row(phase.name, str(phase.max_actions), phase.expected_output)
|
|
148
|
+
console.print(table)
|
|
149
|
+
console.print(
|
|
150
|
+
Panel(
|
|
151
|
+
plan.generated_stop_prompt,
|
|
152
|
+
title="Pasteable phase-gated prompt",
|
|
153
|
+
border_style="magenta",
|
|
154
|
+
box=box.ROUNDED,
|
|
155
|
+
)
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _print_config(config: ForesightConfig) -> None:
|
|
160
|
+
"""Render effective configuration with the file that supplied it."""
|
|
161
|
+
|
|
162
|
+
table = Table(title="Active Foresight configuration", box=box.ROUNDED, border_style=ACCENT)
|
|
163
|
+
table.add_column("Setting", style=f"bold {ACCENT}")
|
|
164
|
+
table.add_column("Value", style="white")
|
|
165
|
+
for name, value in (
|
|
166
|
+
("default_tier_bias", config.default_tier_bias),
|
|
167
|
+
("dashboard_url", config.dashboard_url),
|
|
168
|
+
("eventbus_host", config.eventbus_host),
|
|
169
|
+
("eventbus_port", str(config.eventbus_port)),
|
|
170
|
+
("isolated_ceiling_usd", f"${config.isolated_ceiling_usd:.2f}"),
|
|
171
|
+
("moderate_ceiling_usd", f"${config.moderate_ceiling_usd:.2f}"),
|
|
172
|
+
("wide_ceiling_usd", f"${config.wide_ceiling_usd:.2f}"),
|
|
173
|
+
("source", str(config.source) if config.source else "built-in defaults"),
|
|
174
|
+
):
|
|
175
|
+
table.add_row(name, value)
|
|
176
|
+
console.print(table)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
@click.group(context_settings={"help_option_names": []}, invoke_without_command=True)
|
|
180
|
+
@click.option("-h", "--help", "show_help", is_flag=True, is_eager=True, expose_value=False, callback=_show_help)
|
|
181
|
+
@click.version_option(VERSION, "--version", message=f"Foresight v{VERSION}")
|
|
182
|
+
@click.pass_context
|
|
183
|
+
def main(context: click.Context) -> None:
|
|
184
|
+
"""Predict, monitor, and audit AI coding-agent sessions."""
|
|
185
|
+
|
|
186
|
+
try:
|
|
187
|
+
context.ensure_object(dict)
|
|
188
|
+
context.obj["config"] = load_config()
|
|
189
|
+
except ValueError as error:
|
|
190
|
+
raise click.ClickException(str(error)) from error
|
|
191
|
+
if context.invoked_subcommand is None:
|
|
192
|
+
_print_banner()
|
|
193
|
+
_print_command_table()
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
@main.command(help="Infer policy and tier prediction from an interactive task scope.")
|
|
197
|
+
@click.pass_context
|
|
198
|
+
def init(context: click.Context) -> None:
|
|
199
|
+
"""Create and optionally save a policy and prediction for a task."""
|
|
200
|
+
|
|
201
|
+
_print_banner()
|
|
202
|
+
_print_header("INIT", "describe the task boundary")
|
|
203
|
+
config: ForesightConfig = context.find_root().obj["config"]
|
|
204
|
+
scope = click.prompt("Task scope")
|
|
205
|
+
tracker = create_cost_tracker()
|
|
206
|
+
policy = scope_to_policy(scope, tracker)
|
|
207
|
+
prediction = predict_complexity(scope, tracker, tier_bias=config.default_tier_bias)
|
|
208
|
+
|
|
209
|
+
table = Table(title="Foresight session plan", box=box.ROUNDED, border_style=ACCENT)
|
|
210
|
+
table.add_column("Policy scope", style="white")
|
|
211
|
+
table.add_column("Predicted tier", style=f"bold {ACCENT}")
|
|
212
|
+
table.add_column("Effort", style="yellow")
|
|
213
|
+
table.add_column("Blast radius", style="magenta")
|
|
214
|
+
table.add_row(
|
|
215
|
+
", ".join(policy.allowed_paths) or "(none)",
|
|
216
|
+
prediction["predicted_tier"],
|
|
217
|
+
prediction["predicted_effort"],
|
|
218
|
+
prediction["estimated_blast_radius"],
|
|
219
|
+
)
|
|
220
|
+
console.print(table)
|
|
221
|
+
if advisory := prediction.get("effort_advisory"):
|
|
222
|
+
console.print(f"[dim]Effort guidance: {advisory}[/dim]")
|
|
223
|
+
|
|
224
|
+
workflow_plan = None
|
|
225
|
+
if click.confirm("Generate a phased workflow plan for this task?", default=False):
|
|
226
|
+
workflow_plan = generate_workflow_plan(scope, tracker)
|
|
227
|
+
_print_workflow_plan(workflow_plan)
|
|
228
|
+
|
|
229
|
+
if not click.confirm("Save this session plan?", default=True):
|
|
230
|
+
console.print("[dim]Session plan not saved.[/dim]")
|
|
231
|
+
return
|
|
232
|
+
|
|
233
|
+
STATE.mkdir(exist_ok=True)
|
|
234
|
+
(STATE / "policy.json").write_text(json.dumps(asdict(policy), indent=2), encoding="utf-8")
|
|
235
|
+
(STATE / "prediction.json").write_text(json.dumps(prediction, indent=2), encoding="utf-8")
|
|
236
|
+
(STATE / "scope.txt").write_text(scope, encoding="utf-8")
|
|
237
|
+
workflow_path = STATE / "workflow_plan.json"
|
|
238
|
+
if workflow_plan is not None:
|
|
239
|
+
workflow_path.write_text(json.dumps(asdict(workflow_plan), indent=2), encoding="utf-8")
|
|
240
|
+
elif workflow_path.exists():
|
|
241
|
+
workflow_path.unlink()
|
|
242
|
+
console.print(f"[bold green]OK[/bold green] Saved session plan in [cyan]{STATE}[/cyan]")
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
@main.command(context_settings={"ignore_unknown_options": True}, help="Run a command under Sentry and Foresight monitoring.")
|
|
246
|
+
@click.argument("command", nargs=-1, type=click.UNPROCESSED)
|
|
247
|
+
@click.pass_context
|
|
248
|
+
def run(context: click.Context, command: tuple[str, ...]) -> None:
|
|
249
|
+
"""Run one command through the existing monitoring pipeline."""
|
|
250
|
+
|
|
251
|
+
_print_banner()
|
|
252
|
+
if not command:
|
|
253
|
+
raise click.UsageError("Pass a command after --, for example: foresight run -- python task.py")
|
|
254
|
+
|
|
255
|
+
_print_header("RUN", "starting monitored command")
|
|
256
|
+
config: ForesightConfig = context.find_root().obj["config"]
|
|
257
|
+
policy, predicted, workflow_plan = _load()
|
|
258
|
+
scope_path = STATE / "scope.txt"
|
|
259
|
+
scope = scope_path.read_text(encoding="utf-8") if scope_path.exists() else "Saved Foresight session"
|
|
260
|
+
tracker = create_cost_tracker()
|
|
261
|
+
thread = threading.Thread(target=_server, args=(config,), daemon=True)
|
|
262
|
+
thread.start()
|
|
263
|
+
|
|
264
|
+
session_id = f"foresight-{uuid.uuid4().hex[:8]}"
|
|
265
|
+
proxy = create_sentry_proxy(session_id, policy, lambda action, active_policy: evaluate_action(action, active_policy, tracker))
|
|
266
|
+
monitor = SessionMonitor(
|
|
267
|
+
scope,
|
|
268
|
+
predicted,
|
|
269
|
+
proxy,
|
|
270
|
+
broadcast_signal,
|
|
271
|
+
tracker,
|
|
272
|
+
workflow_plan=workflow_plan,
|
|
273
|
+
budget_overrides=config.budget_overrides,
|
|
274
|
+
)
|
|
275
|
+
killed = False
|
|
276
|
+
|
|
277
|
+
console.print(
|
|
278
|
+
Panel.fit(
|
|
279
|
+
f"[bold {ACCENT}]MONITORING[/bold {ACCENT}]\n[white]{' '.join(command)}[/white]",
|
|
280
|
+
title="Foresight run",
|
|
281
|
+
border_style=ACCENT,
|
|
282
|
+
box=box.ROUNDED,
|
|
283
|
+
)
|
|
284
|
+
)
|
|
285
|
+
if advisory := predicted.get("effort_advisory"):
|
|
286
|
+
console.print(f"[dim]Effort guidance: {advisory}[/dim]")
|
|
287
|
+
if workflow_plan is not None:
|
|
288
|
+
_print_workflow_plan(workflow_plan)
|
|
289
|
+
console.print(f"[dim]Dashboard: {config.dashboard_url}[/dim]")
|
|
290
|
+
try:
|
|
291
|
+
with proxy:
|
|
292
|
+
subprocess.run(list(command), check=False)
|
|
293
|
+
except SentryBlockedException as error:
|
|
294
|
+
killed = True
|
|
295
|
+
console.print(f"[bold red]BLOCKED[/bold red] {error}")
|
|
296
|
+
|
|
297
|
+
report = monitor.get_session_report()
|
|
298
|
+
STATE.mkdir(exist_ok=True)
|
|
299
|
+
(STATE / "last_report.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
|
|
300
|
+
color = "red" if killed else ("yellow" if report["flags_raised"] else "green")
|
|
301
|
+
console.print(
|
|
302
|
+
Panel(
|
|
303
|
+
json.dumps(report, indent=2),
|
|
304
|
+
title=f"[{color}]Foresight session report[/{color}]",
|
|
305
|
+
border_style=color,
|
|
306
|
+
box=box.ROUNDED,
|
|
307
|
+
)
|
|
308
|
+
)
|
|
309
|
+
if killed:
|
|
310
|
+
raise click.exceptions.Exit(1)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
@main.command(help="Show the last saved Foresight session report.")
|
|
314
|
+
def status() -> None:
|
|
315
|
+
"""Print the final report from the most recent monitored run."""
|
|
316
|
+
|
|
317
|
+
_print_header("STATUS", "last saved session report")
|
|
318
|
+
path = STATE / "last_report.json"
|
|
319
|
+
if not path.exists():
|
|
320
|
+
raise click.ClickException("No completed session report yet.")
|
|
321
|
+
console.print(
|
|
322
|
+
Panel(
|
|
323
|
+
path.read_text(encoding="utf-8"),
|
|
324
|
+
title="Foresight session report",
|
|
325
|
+
border_style=ACCENT,
|
|
326
|
+
box=box.ROUNDED,
|
|
327
|
+
)
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
@main.command(help="Export the in-memory audit log.")
|
|
332
|
+
@click.option("--session", default=None, help="Session label displayed with the export.")
|
|
333
|
+
def export(session: str | None) -> None:
|
|
334
|
+
"""Write the in-memory event bus audit log to JSON."""
|
|
335
|
+
|
|
336
|
+
_print_header("EXPORT", "writing audit log")
|
|
337
|
+
path = export_audit_log()
|
|
338
|
+
suffix = f" for {session}" if session else ""
|
|
339
|
+
console.print(f"[bold green]OK[/bold green] Audit exported: [cyan]{path}[/cyan]{suffix}")
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
@main.group(help="Create or inspect user-editable Foresight defaults.")
|
|
343
|
+
def config() -> None:
|
|
344
|
+
"""Manage the optional project-local ``foresight.toml`` file."""
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
@config.command("init", help="Write a commented default foresight.toml in the current directory.")
|
|
348
|
+
def config_init() -> None:
|
|
349
|
+
"""Create the project-local configuration template without overwriting it."""
|
|
350
|
+
|
|
351
|
+
_print_header("CONFIG", "creating project-local defaults")
|
|
352
|
+
path = Path.cwd() / "foresight.toml"
|
|
353
|
+
try:
|
|
354
|
+
write_default_config(path)
|
|
355
|
+
except FileExistsError as error:
|
|
356
|
+
raise click.ClickException(str(error)) from error
|
|
357
|
+
console.print(f"[bold green]OK[/bold green] Created [cyan]{path}[/cyan]")
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
@config.command("show", help="Print the merged effective config and its source.")
|
|
361
|
+
@click.pass_context
|
|
362
|
+
def config_show(context: click.Context) -> None:
|
|
363
|
+
"""Display active defaults after optional TOML values override built-ins."""
|
|
364
|
+
|
|
365
|
+
_print_header("CONFIG", "active merged defaults")
|
|
366
|
+
_print_config(context.find_root().obj["config"])
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
if __name__ == "__main__":
|
|
370
|
+
main()
|
foresight/config.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""User-editable Foresight defaults loaded from TOML without requiring setup."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import tomllib
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
VALID_TIER_BIASES = {"balanced", "cost_optimized", "quality_optimized"}
|
|
13
|
+
DEFAULT_CONFIG_TEXT = """# Foresight user defaults. Values in this file override built-in defaults.
|
|
14
|
+
|
|
15
|
+
[defaults]
|
|
16
|
+
# balanced, cost_optimized, or quality_optimized. This nudges tier recommendations.
|
|
17
|
+
default_tier_bias = "balanced"
|
|
18
|
+
# Where the local dashboard is expected to be available.
|
|
19
|
+
dashboard_url = "http://localhost:3000/dashboard"
|
|
20
|
+
# Local WebSocket event bus address used by `foresight run`.
|
|
21
|
+
eventbus_host = "localhost"
|
|
22
|
+
eventbus_port = 8765
|
|
23
|
+
|
|
24
|
+
[budgets]
|
|
25
|
+
# Per-session warning ceilings in USD, selected from predicted blast radius.
|
|
26
|
+
isolated_ceiling_usd = 0.10
|
|
27
|
+
moderate_ceiling_usd = 0.50
|
|
28
|
+
wide_ceiling_usd = 2.00
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class ForesightConfig:
|
|
34
|
+
"""Effective configuration after built-in defaults and one TOML file are merged."""
|
|
35
|
+
|
|
36
|
+
default_tier_bias: str = "balanced"
|
|
37
|
+
dashboard_url: str = "http://localhost:3000/dashboard"
|
|
38
|
+
eventbus_host: str = "localhost"
|
|
39
|
+
eventbus_port: int = 8765
|
|
40
|
+
isolated_ceiling_usd: float = 0.10
|
|
41
|
+
moderate_ceiling_usd: float = 0.50
|
|
42
|
+
wide_ceiling_usd: float = 2.00
|
|
43
|
+
source: Path | None = None
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def budget_overrides(self) -> dict[str, float]:
|
|
47
|
+
"""Return monitor ceilings keyed by predicted blast radius."""
|
|
48
|
+
|
|
49
|
+
return {
|
|
50
|
+
"isolated": self.isolated_ceiling_usd,
|
|
51
|
+
"moderate": self.moderate_ceiling_usd,
|
|
52
|
+
"wide": self.wide_ceiling_usd,
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def user_config_path() -> Path:
|
|
57
|
+
"""Return the platform-appropriate per-user configuration path."""
|
|
58
|
+
|
|
59
|
+
if os.name == "nt":
|
|
60
|
+
root = Path(os.environ.get("APPDATA", Path.home() / "AppData" / "Roaming"))
|
|
61
|
+
return root / "foresight" / "config.toml"
|
|
62
|
+
return Path.home() / ".config" / "foresight" / "config.toml"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def find_config_path(cwd: Path | None = None) -> Path | None:
|
|
66
|
+
"""Prefer ``foresight.toml`` in the working directory, then user config."""
|
|
67
|
+
|
|
68
|
+
local = (cwd or Path.cwd()) / "foresight.toml"
|
|
69
|
+
if local.is_file():
|
|
70
|
+
return local
|
|
71
|
+
user = user_config_path()
|
|
72
|
+
return user if user.is_file() else None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def load_config(cwd: Path | None = None) -> ForesightConfig:
|
|
76
|
+
"""Load optional TOML settings, keeping zero-config defaults when absent."""
|
|
77
|
+
|
|
78
|
+
path = find_config_path(cwd)
|
|
79
|
+
if path is None:
|
|
80
|
+
return ForesightConfig()
|
|
81
|
+
try:
|
|
82
|
+
# ``utf-8-sig`` accepts files written by Windows editors that add a BOM.
|
|
83
|
+
raw = tomllib.loads(path.read_text(encoding="utf-8-sig"))
|
|
84
|
+
except tomllib.TOMLDecodeError as error:
|
|
85
|
+
raise ValueError(f"Invalid Foresight config at {path}: {error}") from error
|
|
86
|
+
|
|
87
|
+
defaults = _section(raw, "defaults")
|
|
88
|
+
budgets = _section(raw, "budgets")
|
|
89
|
+
tier_bias = str(defaults.get("default_tier_bias", "balanced"))
|
|
90
|
+
if tier_bias not in VALID_TIER_BIASES:
|
|
91
|
+
raise ValueError(f"default_tier_bias must be one of: {', '.join(sorted(VALID_TIER_BIASES))}")
|
|
92
|
+
|
|
93
|
+
config = ForesightConfig(
|
|
94
|
+
default_tier_bias=tier_bias,
|
|
95
|
+
dashboard_url=str(defaults.get("dashboard_url", "http://localhost:3000/dashboard")),
|
|
96
|
+
eventbus_host=str(defaults.get("eventbus_host", "localhost")),
|
|
97
|
+
eventbus_port=_positive_int(defaults.get("eventbus_port", 8765), "eventbus_port"),
|
|
98
|
+
isolated_ceiling_usd=_positive_float(budgets.get("isolated_ceiling_usd", 0.10), "isolated_ceiling_usd"),
|
|
99
|
+
moderate_ceiling_usd=_positive_float(budgets.get("moderate_ceiling_usd", 0.50), "moderate_ceiling_usd"),
|
|
100
|
+
wide_ceiling_usd=_positive_float(budgets.get("wide_ceiling_usd", 2.00), "wide_ceiling_usd"),
|
|
101
|
+
source=path,
|
|
102
|
+
)
|
|
103
|
+
return config
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def write_default_config(path: Path) -> Path:
|
|
107
|
+
"""Create a commented project-local config file without overwriting one."""
|
|
108
|
+
|
|
109
|
+
if path.exists():
|
|
110
|
+
raise FileExistsError(f"Config already exists: {path}")
|
|
111
|
+
path.write_text(DEFAULT_CONFIG_TEXT, encoding="utf-8")
|
|
112
|
+
return path
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _section(raw: dict[str, Any], name: str) -> dict[str, Any]:
|
|
116
|
+
value = raw.get(name, {})
|
|
117
|
+
if not isinstance(value, dict):
|
|
118
|
+
raise ValueError(f"[{name}] must be a TOML table")
|
|
119
|
+
return value
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _positive_int(value: Any, name: str) -> int:
|
|
123
|
+
try:
|
|
124
|
+
parsed = int(value)
|
|
125
|
+
except (TypeError, ValueError) as error:
|
|
126
|
+
raise ValueError(f"{name} must be a positive integer") from error
|
|
127
|
+
if parsed <= 0:
|
|
128
|
+
raise ValueError(f"{name} must be a positive integer")
|
|
129
|
+
return parsed
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _positive_float(value: Any, name: str) -> float:
|
|
133
|
+
try:
|
|
134
|
+
parsed = float(value)
|
|
135
|
+
except (TypeError, ValueError) as error:
|
|
136
|
+
raise ValueError(f"{name} must be a positive number") from error
|
|
137
|
+
if parsed <= 0:
|
|
138
|
+
raise ValueError(f"{name} must be a positive number")
|
|
139
|
+
return parsed
|