flarebisect 0.3.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1 @@
1
+ __version__ = "0.3.1"
@@ -0,0 +1,37 @@
1
+ """Ask the configured LLM for a short root-cause read on the culprit commit."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from .bisect import BisectOutcome
6
+ from .providers import ProviderConfig, complete
7
+
8
+ PROMPT_TEMPLATE = """A commit changed a test's failure rate. Diagnose the likely cause.
9
+
10
+ commit {sha}: "{subject}"
11
+
12
+ flake rate before: {before_rate:.0%}
13
+ flake rate after: {culprit_rate:.0%}
14
+ bisection threshold: {threshold:.0%}
15
+ verdict: {verdict}
16
+
17
+ diff:
18
+ ```diff
19
+ {diff}
20
+ ```
21
+
22
+ Answer in exactly 1-2 short sentences, no preamble, no restating the numbers above.
23
+ Point at the specific code change responsible (shared state, missing lock, timing/ordering
24
+ assumption, resource leak, off-by-one, whatever it actually is)."""
25
+
26
+
27
+ def explain(outcome: BisectOutcome, diff: str, threshold: float, provider_cfg: ProviderConfig) -> str:
28
+ prompt = PROMPT_TEMPLATE.format(
29
+ sha=outcome.culprit.sha[:12],
30
+ subject=outcome.culprit.subject,
31
+ before_rate=outcome.before.result.flake_rate,
32
+ culprit_rate=outcome.culprit.result.flake_rate,
33
+ threshold=threshold,
34
+ verdict=outcome.verdict,
35
+ diff=diff[:6000],
36
+ )
37
+ return complete(provider_cfg, prompt)
flarebisect/bisect.py ADDED
@@ -0,0 +1,145 @@
1
+ """Flake-rate bisection: binary search over commits for the point where the
2
+ flake rate jumps by >= threshold relative to the known-good baseline."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import time
7
+ from dataclasses import dataclass
8
+ from pathlib import Path
9
+ from typing import Callable
10
+
11
+ from . import git_ops
12
+ from .runner import RunResult, measure_flake_rate, worker_count_for
13
+
14
+
15
+ @dataclass
16
+ class CommitMeasurement:
17
+ sha: str
18
+ subject: str
19
+ result: RunResult
20
+ position: int # -1 = good baseline, 0..n-1 = index within the good..bad range
21
+ status: str = "" # filled in once the culprit is known: good/stable/wobbling/flare/bad
22
+
23
+
24
+ @dataclass
25
+ class BisectOutcome:
26
+ culprit: CommitMeasurement
27
+ before: CommitMeasurement # last known-good-ish measurement (parent side)
28
+ good_baseline: RunResult
29
+ bad_baseline: RunResult
30
+ verdict: str # "clean break" | "flakiness regression"
31
+ measurements: list[CommitMeasurement] # every distinct commit measured, in range order
32
+ total_commits: int
33
+ runs: int
34
+ workers: int
35
+ elapsed_seconds: float
36
+
37
+
38
+ def _measure(repo: Path, sha: str, test_cmd: str, runs: int, position: int) -> CommitMeasurement:
39
+ wt = git_ops.add_worktree(repo, sha)
40
+ try:
41
+ result = measure_flake_rate(test_cmd, wt.path, runs)
42
+ finally:
43
+ git_ops.remove_worktree(repo, wt)
44
+ return CommitMeasurement(
45
+ sha=sha,
46
+ subject=git_ops.commit_subject(repo, sha),
47
+ result=result,
48
+ position=position,
49
+ )
50
+
51
+
52
+ def classify(good_rate: float, culprit_rate: float) -> str:
53
+ if culprit_rate >= 0.9 and good_rate <= 0.1:
54
+ return "clean break"
55
+ return "flakiness regression"
56
+
57
+
58
+ def status_label(rate: float, is_first: bool, is_last: bool, is_culprit: bool) -> str:
59
+ if is_culprit:
60
+ return "flare"
61
+ if is_last:
62
+ return "bad"
63
+ if rate <= 0.0:
64
+ return "good" if is_first else "stable"
65
+ return "wobbling"
66
+
67
+
68
+ def run_bisect(
69
+ repo: Path,
70
+ good: str,
71
+ bad: str,
72
+ test_cmd: str,
73
+ runs: int = 5,
74
+ threshold: float = 0.3,
75
+ on_measure: Callable[[str, CommitMeasurement], None] | None = None,
76
+ ) -> BisectOutcome:
77
+ started = time.perf_counter()
78
+
79
+ good_sha = git_ops.resolve_sha(repo, good)
80
+ bad_sha = git_ops.resolve_sha(repo, bad)
81
+
82
+ good_m = _measure(repo, good_sha, test_cmd, runs, position=-1)
83
+ if on_measure:
84
+ on_measure("good", good_m)
85
+
86
+ commits = git_ops.commit_list(repo, good_sha, bad_sha)
87
+ if not commits:
88
+ raise ValueError("no commits between good and bad")
89
+
90
+ baseline_rate = good_m.result.flake_rate
91
+
92
+ cache: dict[str, CommitMeasurement] = {}
93
+
94
+ def measure_idx(idx: int) -> CommitMeasurement:
95
+ sha = commits[idx]
96
+ if sha not in cache:
97
+ m = _measure(repo, sha, test_cmd, runs, position=idx)
98
+ cache[sha] = m
99
+ if on_measure:
100
+ on_measure(f"candidate[{idx}]", m)
101
+ return cache[sha]
102
+
103
+ bad_m = measure_idx(len(commits) - 1)
104
+
105
+ lo, hi = 0, len(commits) - 1
106
+ culprit_idx = hi # fall back to bad commit if search never narrows
107
+ culprit_measurement = bad_m
108
+
109
+ while lo <= hi:
110
+ mid = (lo + hi) // 2
111
+ m = measure_idx(mid)
112
+ jumped = (m.result.flake_rate - baseline_rate) >= threshold
113
+ if jumped:
114
+ culprit_idx = mid
115
+ culprit_measurement = m
116
+ hi = mid - 1
117
+ else:
118
+ lo = mid + 1
119
+
120
+ before_measurement = measure_idx(culprit_idx - 1) if culprit_idx > 0 else good_m
121
+
122
+ verdict = classify(before_measurement.result.flake_rate, culprit_measurement.result.flake_rate)
123
+
124
+ last_idx = len(commits) - 1
125
+ ordered = [good_m] + [cache[sha] for sha in commits if sha in cache]
126
+ for m in ordered:
127
+ m.status = status_label(
128
+ m.result.flake_rate,
129
+ is_first=(m.position == -1),
130
+ is_last=(m.position == last_idx),
131
+ is_culprit=(m.sha == culprit_measurement.sha),
132
+ )
133
+
134
+ return BisectOutcome(
135
+ culprit=culprit_measurement,
136
+ before=before_measurement,
137
+ good_baseline=good_m.result,
138
+ bad_baseline=bad_m.result,
139
+ verdict=verdict,
140
+ measurements=ordered,
141
+ total_commits=len(commits),
142
+ runs=runs,
143
+ workers=worker_count_for(runs),
144
+ elapsed_seconds=time.perf_counter() - started,
145
+ )
flarebisect/cli.py ADDED
@@ -0,0 +1,309 @@
1
+ from __future__ import annotations
2
+
3
+ import subprocess
4
+ from pathlib import Path
5
+ from typing import Optional
6
+
7
+ import typer
8
+ from rich.panel import Panel
9
+ from rich.prompt import Confirm, Prompt
10
+
11
+ from . import __version__, ai_explain, config as config_store, git_ops, hardware, report
12
+ from .bisect import run_bisect
13
+ from .providers import DEFAULT_BASE_URLS, DEFAULT_MODELS, NO_KEY_REQUIRED, PROVIDERS, ProviderError
14
+
15
+ app = typer.Typer(add_completion=False, help="git bisect that treats flakiness as a signal, not noise.")
16
+ config_app = typer.Typer(add_completion=False, help="Manage stored AI provider settings.")
17
+ models_app = typer.Typer(add_completion=False, help="Detect your GPU/VRAM and manage local Ollama models.")
18
+ app.add_typer(config_app, name="config")
19
+ app.add_typer(models_app, name="models")
20
+
21
+ PROVIDER_BLURBS = {
22
+ "anthropic": "Claude, cloud, needs an API key",
23
+ "openai": "GPT models, cloud, needs an API key",
24
+ "google": "Gemini, cloud, needs an API key",
25
+ "ollama": "local model via Ollama, no key needed",
26
+ "custom": "any OpenAI-compatible endpoint (LM Studio, llama.cpp, vLLM...)",
27
+ }
28
+
29
+
30
+ @app.command()
31
+ def version() -> None:
32
+ """Show the flarebisect version."""
33
+ typer.echo(__version__)
34
+
35
+
36
+ @app.command()
37
+ def run(
38
+ good: str = typer.Option(..., "--good", help="Known-good commit/ref."),
39
+ bad: str = typer.Option(..., "--bad", help="Known-bad (or known-flaky) commit/ref."),
40
+ test: str = typer.Option(..., "--test", help="Test command to run, e.g. 'pytest -k my_test'."),
41
+ repo: Path = typer.Option(Path.cwd(), "--repo", help="Path to the git repository."),
42
+ runs: int = typer.Option(20, "--runs", help="Test runs per commit."),
43
+ threshold: float = typer.Option(0.3, "--threshold", help="Flake-rate jump that counts as the culprit (0-1)."),
44
+ explain: bool = typer.Option(True, "--explain/--no-explain", help="Call an LLM for a root-cause explanation."),
45
+ provider: Optional[str] = typer.Option(
46
+ None, "--provider", help=f"Override the default provider ({', '.join(PROVIDERS)})."
47
+ ),
48
+ api_key: Optional[str] = typer.Option(None, "--api-key", help="Override the stored/env API key for this run."),
49
+ model: Optional[str] = typer.Option(None, "--model", help="Override the model for this run."),
50
+ base_url: Optional[str] = typer.Option(
51
+ None, "--base-url", help="Override the API base URL (for local/self-hosted servers)."
52
+ ),
53
+ ) -> None:
54
+ """Bisect on flake-rate jump instead of pass/fail."""
55
+ repo = repo.resolve()
56
+
57
+ try:
58
+ git_ops.resolve_sha(repo, good)
59
+ git_ops.resolve_sha(repo, bad)
60
+ except git_ops.GitError as e:
61
+ report.print_error(str(e))
62
+ raise typer.Exit(1)
63
+
64
+ def on_measure(label: str, m) -> None:
65
+ pass # table is rendered once, after the search completes
66
+
67
+ try:
68
+ outcome = run_bisect(
69
+ repo=repo,
70
+ good=good,
71
+ bad=bad,
72
+ test_cmd=test,
73
+ runs=runs,
74
+ threshold=threshold,
75
+ on_measure=on_measure,
76
+ )
77
+ except ValueError as e:
78
+ report.print_error(str(e))
79
+ raise typer.Exit(1)
80
+
81
+ report.print_progress_line(outcome.total_commits, outcome.runs, outcome.workers)
82
+ report.print_table(outcome.measurements)
83
+
84
+ explanation: Optional[str] = None
85
+ if explain:
86
+ provider_cfg = config_store.resolve(provider=provider, api_key=api_key, model=model, base_url=base_url)
87
+ diff = git_ops.commit_diff(repo, outcome.culprit.sha)
88
+ try:
89
+ explanation = ai_explain.explain(outcome, diff, threshold, provider_cfg)
90
+ except ProviderError as e:
91
+ report.print_error(str(e))
92
+
93
+ report.print_result(outcome, threshold, explanation)
94
+ report.print_footer(outcome.elapsed_seconds, len(outcome.measurements), outcome.workers)
95
+
96
+
97
+ @config_app.callback(invoke_without_command=True)
98
+ def config_main(ctx: typer.Context) -> None:
99
+ """Manage stored AI provider settings. Run with no subcommand for a guided setup."""
100
+ if ctx.invoked_subcommand is None:
101
+ config_wizard()
102
+
103
+
104
+ def config_wizard() -> None:
105
+ console = report.console
106
+ console.print()
107
+ console.print(Panel.fit("[bold]flarebisect setup[/bold] — pick an AI provider for root-cause explanations", border_style="grey50"))
108
+
109
+ console.print()
110
+ console.print("[bold]step 1/4[/bold] — provider")
111
+ for i, name in enumerate(PROVIDERS, start=1):
112
+ console.print(f" [cyan]{i}[/cyan]) {name:<10} [dim]{PROVIDER_BLURBS[name]}[/dim]")
113
+ choice = Prompt.ask(" choose", choices=[str(i) for i in range(1, len(PROVIDERS) + 1)], default="1")
114
+ provider = PROVIDERS[int(choice) - 1]
115
+
116
+ base_url = None
117
+ api_key = None
118
+
119
+ if provider in ("ollama", "custom"):
120
+ console.print()
121
+ console.print("[bold]step 2/4[/bold] — endpoint")
122
+ default_url = DEFAULT_BASE_URLS.get(provider, "http://localhost:11434/v1")
123
+ base_url = Prompt.ask(" base URL", default=default_url)
124
+
125
+ if provider in NO_KEY_REQUIRED:
126
+ needs_key = Confirm.ask(" does this endpoint require an API key?", default=False)
127
+ else:
128
+ needs_key = True
129
+ if needs_key:
130
+ api_key = Prompt.ask(" API key", password=True)
131
+ else:
132
+ console.print()
133
+ console.print("[bold]step 2/4[/bold] — API key")
134
+ console.print(f" [dim]leave blank to fall back to the {config_store.ENV_KEYS.get(provider)} env var[/dim]")
135
+ entered = Prompt.ask(" API key", password=True, default="", show_default=False)
136
+ api_key = entered or None
137
+
138
+ console.print()
139
+ console.print("[bold]step 3/4[/bold] — model")
140
+ default_model = DEFAULT_MODELS.get(provider, "")
141
+ if provider == "ollama":
142
+ gpu = hardware.detect_gpu()
143
+ recommended, desc = hardware.recommend_model_with_desc(gpu.vram_gb)
144
+ if gpu.vendor == "none":
145
+ console.print(" [dim]no dedicated GPU detected — recommending a small CPU-friendly model[/dim]")
146
+ else:
147
+ console.print(f" [dim]detected {gpu.name} (~{gpu.vram_gb} GB VRAM)[/dim]")
148
+ console.print(f" [dim]recommended: {recommended} — {desc}[/dim]")
149
+ default_model = recommended
150
+ model = Prompt.ask(" model", default=default_model)
151
+
152
+ console.print()
153
+ console.print("[bold]step 4/4[/bold] — confirm")
154
+ make_active = Confirm.ask(f" set '{provider}' as the active provider?", default=True)
155
+
156
+ if api_key:
157
+ config_store.set_key(provider, api_key)
158
+ if model:
159
+ config_store.set_model(provider, model)
160
+ if base_url:
161
+ config_store.set_base_url(provider, base_url)
162
+ if make_active:
163
+ config_store.use_provider(provider)
164
+
165
+ if provider == "ollama":
166
+ if hardware.ollama_available():
167
+ if Confirm.ask(f" pull '{model}' now via ollama?", default=True):
168
+ console.print()
169
+ try:
170
+ hardware.pull_model(model)
171
+ console.print(f"[green]pulled {model}[/green]")
172
+ except (RuntimeError, subprocess.CalledProcessError) as e:
173
+ report.print_error(f"pull failed: {e}")
174
+ else:
175
+ console.print(
176
+ " [yellow]ollama not found on PATH[/yellow] — install it from https://ollama.com, "
177
+ "then run `flarebisect models pull`"
178
+ )
179
+
180
+ masked_key = f"...{api_key[-4:]}" if api_key and len(api_key) > 4 else ("(none)" if not api_key else "(set)")
181
+ console.print()
182
+ console.print(
183
+ Panel.fit(
184
+ f"[bold green]saved[/bold green] — provider={provider} model={model}\n"
185
+ f"key={masked_key}"
186
+ + (f" base_url={base_url}" if base_url else "")
187
+ + f"\n\n[dim]config file: {config_store.config_path()}[/dim]\n"
188
+ f"[dim]next: flarebisect run --good <sha> --bad <sha> --test \"...\"[/dim]",
189
+ border_style="green",
190
+ )
191
+ )
192
+
193
+
194
+ @config_app.command("set-key")
195
+ def config_set_key(
196
+ provider: str = typer.Argument(..., help=f"one of {', '.join(PROVIDERS)}"),
197
+ api_key: str = typer.Argument(..., help="API key for this provider."),
198
+ ) -> None:
199
+ """Store an API key for a provider (local servers like Ollama usually don't need one)."""
200
+ if provider not in PROVIDERS:
201
+ report.print_error(f"unknown provider '{provider}' (expected one of {', '.join(PROVIDERS)})")
202
+ raise typer.Exit(1)
203
+ config_store.set_key(provider, api_key)
204
+ typer.echo(f"stored API key for {provider} in {config_store.config_path()}")
205
+
206
+
207
+ @config_app.command("set-model")
208
+ def config_set_model(
209
+ provider: str = typer.Argument(..., help=f"one of {', '.join(PROVIDERS)}"),
210
+ model: str = typer.Argument(...),
211
+ ) -> None:
212
+ """Set the default model used for a provider."""
213
+ if provider not in PROVIDERS:
214
+ report.print_error(f"unknown provider '{provider}' (expected one of {', '.join(PROVIDERS)})")
215
+ raise typer.Exit(1)
216
+ config_store.set_model(provider, model)
217
+ typer.echo(f"{provider} model set to {model}")
218
+
219
+
220
+ @config_app.command("set-base-url")
221
+ def config_set_base_url(
222
+ provider: str = typer.Argument(..., help=f"one of {', '.join(PROVIDERS)}"),
223
+ base_url: str = typer.Argument(..., help="e.g. http://localhost:11434/v1 for Ollama."),
224
+ ) -> None:
225
+ """Point a provider at a self-hosted / local endpoint."""
226
+ if provider not in PROVIDERS:
227
+ report.print_error(f"unknown provider '{provider}' (expected one of {', '.join(PROVIDERS)})")
228
+ raise typer.Exit(1)
229
+ config_store.set_base_url(provider, base_url)
230
+ typer.echo(f"{provider} base URL set to {base_url}")
231
+
232
+
233
+ @config_app.command("use")
234
+ def config_use(provider: str = typer.Argument(..., help=f"one of {', '.join(PROVIDERS)}")) -> None:
235
+ """Set the default provider used by `flarebisect run`."""
236
+ if provider not in PROVIDERS:
237
+ report.print_error(f"unknown provider '{provider}' (expected one of {', '.join(PROVIDERS)})")
238
+ raise typer.Exit(1)
239
+ config_store.use_provider(provider)
240
+ typer.echo(f"default provider set to {provider}")
241
+
242
+
243
+ @config_app.command("show")
244
+ def config_show() -> None:
245
+ """Show the current provider config (API keys masked)."""
246
+ data = config_store.load()
247
+ active = data.get("provider", config_store.DEFAULT_PROVIDER)
248
+ typer.echo(f"active provider: {active}")
249
+ typer.echo(f"config file: {config_store.config_path()}")
250
+ typer.echo()
251
+ for name in PROVIDERS:
252
+ stored = data.get("providers", {}).get(name, {})
253
+ key = stored.get("api_key")
254
+ masked = f"...{key[-4:]}" if key and len(key) > 4 else ("(set)" if key else "(none)")
255
+ marker = "*" if name == active else " "
256
+ typer.echo(
257
+ f"{marker} {name:<10} key={masked:<10} model={stored.get('model') or '(default)':<20} "
258
+ f"base_url={stored.get('base_url') or '(default)'}"
259
+ )
260
+
261
+
262
+ @models_app.command("detect")
263
+ def models_detect() -> None:
264
+ """Detect your GPU/VRAM and show the recommended local model."""
265
+ console = report.console
266
+ gpu = hardware.detect_gpu()
267
+ model, desc = hardware.recommend_model_with_desc(gpu.vram_gb)
268
+ console.print()
269
+ if gpu.vendor == "none":
270
+ console.print("[yellow]no dedicated GPU detected[/yellow] — falling back to a small CPU-friendly model")
271
+ else:
272
+ console.print(f"GPU: [bold]{gpu.name}[/bold] ({gpu.vendor}) — ~{gpu.vram_gb} GB VRAM")
273
+ console.print(f"recommended model: [bold cyan]{model}[/bold cyan] — {desc}")
274
+ console.print("[dim]run `flarebisect models pull` to download it[/dim]")
275
+
276
+
277
+ @models_app.command("pull")
278
+ def models_pull(
279
+ model: Optional[str] = typer.Argument(None, help="Model tag to pull (default: auto-detected from your GPU)."),
280
+ set_active: bool = typer.Option(
281
+ True, "--set-active/--no-set-active", help="Set ollama as the active provider with this model."
282
+ ),
283
+ ) -> None:
284
+ """Download a local model via Ollama, sized to your GPU's VRAM."""
285
+ console = report.console
286
+ gpu = hardware.detect_gpu()
287
+ chosen = model or hardware.recommend_model(gpu.vram_gb)
288
+ if not model:
289
+ console.print(f"detected [bold]{gpu.name}[/bold] (~{gpu.vram_gb} GB VRAM) — pulling [bold cyan]{chosen}[/bold cyan]")
290
+ else:
291
+ console.print(f"pulling [bold cyan]{chosen}[/bold cyan]")
292
+
293
+ try:
294
+ hardware.pull_model(chosen)
295
+ except RuntimeError as e:
296
+ report.print_error(str(e))
297
+ raise typer.Exit(1)
298
+ except subprocess.CalledProcessError as e:
299
+ report.print_error(f"ollama pull failed: {e}")
300
+ raise typer.Exit(1)
301
+
302
+ if set_active:
303
+ config_store.set_model("ollama", chosen)
304
+ config_store.use_provider("ollama")
305
+ console.print(f"[green]saved[/green] — active provider set to ollama / {chosen}")
306
+
307
+
308
+ if __name__ == "__main__":
309
+ app()
flarebisect/config.py ADDED
@@ -0,0 +1,89 @@
1
+ """Persisted CLI settings: default provider + per-provider api key/model/base_url.
2
+
3
+ Stored as JSON under the OS config dir so `flarebisect config set-key ...` sticks
4
+ across shells and reboots without env vars. Falls back to per-provider env vars
5
+ (ANTHROPIC_API_KEY, OPENAI_API_KEY, GOOGLE_API_KEY) when nothing is stored."""
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import os
11
+ from pathlib import Path
12
+
13
+ from .providers import ProviderConfig
14
+
15
+ ENV_KEYS = {
16
+ "anthropic": "ANTHROPIC_API_KEY",
17
+ "openai": "OPENAI_API_KEY",
18
+ "google": "GOOGLE_API_KEY",
19
+ }
20
+
21
+ DEFAULT_PROVIDER = "anthropic"
22
+
23
+
24
+ def config_dir() -> Path:
25
+ if os.name == "nt":
26
+ base = os.environ.get("APPDATA", str(Path.home()))
27
+ else:
28
+ base = os.environ.get("XDG_CONFIG_HOME", str(Path.home() / ".config"))
29
+ return Path(base) / "flarebisect"
30
+
31
+
32
+ def config_path() -> Path:
33
+ return config_dir() / "config.json"
34
+
35
+
36
+ def load() -> dict:
37
+ path = config_path()
38
+ if not path.exists():
39
+ return {"provider": DEFAULT_PROVIDER, "providers": {}}
40
+ return json.loads(path.read_text())
41
+
42
+
43
+ def save(data: dict) -> None:
44
+ path = config_path()
45
+ path.parent.mkdir(parents=True, exist_ok=True)
46
+ path.write_text(json.dumps(data, indent=2) + "\n")
47
+ if os.name != "nt":
48
+ path.chmod(0o600)
49
+
50
+
51
+ def set_key(provider: str, api_key: str) -> None:
52
+ data = load()
53
+ data.setdefault("providers", {}).setdefault(provider, {})["api_key"] = api_key
54
+ save(data)
55
+
56
+
57
+ def set_model(provider: str, model: str) -> None:
58
+ data = load()
59
+ data.setdefault("providers", {}).setdefault(provider, {})["model"] = model
60
+ save(data)
61
+
62
+
63
+ def set_base_url(provider: str, base_url: str) -> None:
64
+ data = load()
65
+ data.setdefault("providers", {}).setdefault(provider, {})["base_url"] = base_url
66
+ save(data)
67
+
68
+
69
+ def use_provider(provider: str) -> None:
70
+ data = load()
71
+ data["provider"] = provider
72
+ save(data)
73
+
74
+
75
+ def resolve(
76
+ provider: str | None = None,
77
+ api_key: str | None = None,
78
+ model: str | None = None,
79
+ base_url: str | None = None,
80
+ ) -> ProviderConfig:
81
+ data = load()
82
+ name = provider or data.get("provider", DEFAULT_PROVIDER)
83
+ stored = data.get("providers", {}).get(name, {})
84
+
85
+ resolved_key = api_key or stored.get("api_key") or os.environ.get(ENV_KEYS.get(name, ""), None)
86
+ resolved_model = model or stored.get("model")
87
+ resolved_base_url = base_url or stored.get("base_url")
88
+
89
+ return ProviderConfig(name=name, api_key=resolved_key, model=resolved_model, base_url=resolved_base_url)
flarebisect/git_ops.py ADDED
@@ -0,0 +1,70 @@
1
+ """Thin wrapper around git subprocess calls. Never touches the caller's
2
+ working tree — every candidate commit is checked out into its own
3
+ `git worktree`, which git guarantees is isolated from the main tree."""
4
+
5
+ from __future__ import annotations
6
+
7
+ import shutil
8
+ import subprocess
9
+ import tempfile
10
+ from dataclasses import dataclass
11
+ from pathlib import Path
12
+
13
+
14
+ class GitError(RuntimeError):
15
+ pass
16
+
17
+
18
+ def _run(args: list[str], cwd: Path) -> str:
19
+ try:
20
+ result = subprocess.run(
21
+ ["git", *args],
22
+ cwd=cwd,
23
+ capture_output=True,
24
+ text=True,
25
+ )
26
+ except FileNotFoundError as e:
27
+ raise GitError("git not found on PATH — install git and make sure it's on PATH") from e
28
+ if result.returncode != 0:
29
+ raise GitError(f"git {' '.join(args)} failed: {result.stderr.strip()}")
30
+ return result.stdout.strip()
31
+
32
+
33
+ def resolve_sha(repo: Path, ref: str) -> str:
34
+ return _run(["rev-parse", ref], repo)
35
+
36
+
37
+ def commit_list(repo: Path, good: str, bad: str) -> list[str]:
38
+ """Oldest -> newest commits strictly after `good`, up to and including `bad`."""
39
+ out = _run(["rev-list", "--reverse", f"{good}..{bad}"], repo)
40
+ return [line for line in out.splitlines() if line]
41
+
42
+
43
+ def commit_subject(repo: Path, sha: str) -> str:
44
+ return _run(["log", "-1", "--format=%s", sha], repo)
45
+
46
+
47
+ def commit_diff(repo: Path, sha: str) -> str:
48
+ return _run(["show", "--format=", sha], repo)
49
+
50
+
51
+ @dataclass
52
+ class Worktree:
53
+ path: Path
54
+ sha: str
55
+
56
+
57
+ def add_worktree(repo: Path, sha: str) -> Worktree:
58
+ tmp_root = Path(tempfile.mkdtemp(prefix="flarebisect-"))
59
+ wt_path = tmp_root / sha[:12]
60
+ _run(["worktree", "add", "--detach", str(wt_path), sha], repo)
61
+ return Worktree(path=wt_path, sha=sha)
62
+
63
+
64
+ def remove_worktree(repo: Path, wt: Worktree) -> None:
65
+ try:
66
+ _run(["worktree", "remove", "--force", str(wt.path)], repo)
67
+ except GitError:
68
+ # worktree metadata already gone; just clean the directory
69
+ pass
70
+ shutil.rmtree(wt.path.parent, ignore_errors=True)
@@ -0,0 +1,141 @@
1
+ """GPU/VRAM detection and local-model sizing.
2
+
3
+ Best-effort across NVIDIA (nvidia-smi), AMD (rocm-smi), Apple Silicon
4
+ (unified memory via sysctl), and Windows (WMI via PowerShell). Falls back to
5
+ "no GPU" if nothing is found, which still yields a usable (small) model
6
+ recommendation instead of an error.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import platform
13
+ import re
14
+ import shutil
15
+ import subprocess
16
+ from dataclasses import dataclass
17
+
18
+ # (vram_gb_ceiling, ollama_tag, description) - first tier whose ceiling exceeds
19
+ # the detected VRAM wins. Sizes assume 4-bit quantization with headroom for context.
20
+ MODEL_TIERS: list[tuple[float, str, str]] = [
21
+ (3.0, "qwen2.5:0.5b", "tiny CPU-friendly model (~0.4GB)"),
22
+ (6.0, "llama3.2:3b", "small, fast model (~2GB)"),
23
+ (10.0, "llama3.1:8b", "solid general-purpose model (~5GB)"),
24
+ (16.0, "qwen2.5:14b", "stronger reasoning (~9GB)"),
25
+ (40.0, "qwen2.5:32b", "large, high-quality model (~18GB)"),
26
+ ]
27
+ FALLBACK_MODEL: tuple[str, str] = ("llama3.1:70b", "flagship-scale model (~40GB), needs serious VRAM")
28
+
29
+
30
+ @dataclass
31
+ class GPUInfo:
32
+ vendor: str # "nvidia", "amd", "apple", "unknown", "none"
33
+ name: str
34
+ vram_gb: float
35
+
36
+
37
+ def _run(cmd: list[str]) -> str | None:
38
+ if not shutil.which(cmd[0]):
39
+ return None
40
+ try:
41
+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
42
+ except (OSError, subprocess.TimeoutExpired):
43
+ return None
44
+ if result.returncode != 0 or not result.stdout.strip():
45
+ return None
46
+ return result.stdout.strip()
47
+
48
+
49
+ def _detect_nvidia() -> GPUInfo | None:
50
+ out = _run(["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader,nounits"])
51
+ if not out:
52
+ return None
53
+ try:
54
+ name, mib = (part.strip() for part in out.splitlines()[0].split(","))
55
+ return GPUInfo(vendor="nvidia", name=name, vram_gb=round(float(mib) / 1024, 1))
56
+ except ValueError:
57
+ return None
58
+
59
+
60
+ def _detect_amd() -> GPUInfo | None:
61
+ out = _run(["rocm-smi", "--showproductname", "--showmeminfo", "vram"])
62
+ if not out:
63
+ return None
64
+ vram_match = re.search(r"VRAM Total Memory \(B\):\s*(\d+)", out)
65
+ if not vram_match:
66
+ return None
67
+ name_match = re.search(r"Card series:\s*(.+)", out)
68
+ vram_gb = round(int(vram_match.group(1)) / (1024 ** 3), 1)
69
+ name = name_match.group(1).strip() if name_match else "AMD GPU"
70
+ return GPUInfo(vendor="amd", name=name, vram_gb=vram_gb)
71
+
72
+
73
+ def _detect_apple_silicon() -> GPUInfo | None:
74
+ if platform.system() != "Darwin" or platform.machine() != "arm64":
75
+ return None
76
+ mem_out = _run(["sysctl", "-n", "hw.memsize"])
77
+ if not mem_out:
78
+ return None
79
+ try:
80
+ total_gb = int(mem_out) / (1024 ** 3)
81
+ except ValueError:
82
+ return None
83
+ # Unified memory is shared with the OS and other apps, so only a fraction
84
+ # is realistically usable for model weights.
85
+ usable_gb = round(total_gb * 0.7, 1)
86
+ name = _run(["sysctl", "-n", "machdep.cpu.brand_string"]) or "Apple Silicon"
87
+ return GPUInfo(vendor="apple", name=name, vram_gb=usable_gb)
88
+
89
+
90
+ def _detect_windows_wmi() -> GPUInfo | None:
91
+ if platform.system() != "Windows":
92
+ return None
93
+ out = _run(
94
+ [
95
+ "powershell",
96
+ "-NoProfile",
97
+ "-Command",
98
+ "Get-CimInstance Win32_VideoController | Select-Object -First 1 Name,AdapterRAM | ConvertTo-Json",
99
+ ]
100
+ )
101
+ if not out:
102
+ return None
103
+ try:
104
+ data = json.loads(out)
105
+ except ValueError:
106
+ return None
107
+ name = data.get("Name") or "GPU"
108
+ ram = data.get("AdapterRAM")
109
+ if not ram:
110
+ return None
111
+ vendor = "nvidia" if "nvidia" in name.lower() else "amd" if ("amd" in name.lower() or "radeon" in name.lower()) else "unknown"
112
+ return GPUInfo(vendor=vendor, name=name, vram_gb=round(int(ram) / (1024 ** 3), 1))
113
+
114
+
115
+ def detect_gpu() -> GPUInfo:
116
+ for detector in (_detect_nvidia, _detect_amd, _detect_apple_silicon, _detect_windows_wmi):
117
+ info = detector()
118
+ if info:
119
+ return info
120
+ return GPUInfo(vendor="none", name="no dedicated GPU detected", vram_gb=0.0)
121
+
122
+
123
+ def recommend_model_with_desc(vram_gb: float) -> tuple[str, str]:
124
+ for ceiling, model, desc in MODEL_TIERS:
125
+ if vram_gb < ceiling:
126
+ return model, desc
127
+ return FALLBACK_MODEL
128
+
129
+
130
+ def recommend_model(vram_gb: float) -> str:
131
+ return recommend_model_with_desc(vram_gb)[0]
132
+
133
+
134
+ def ollama_available() -> bool:
135
+ return shutil.which("ollama") is not None
136
+
137
+
138
+ def pull_model(model: str) -> None:
139
+ if not ollama_available():
140
+ raise RuntimeError("ollama not found on PATH - install it from https://ollama.com first")
141
+ subprocess.run(["ollama", "pull", model], check=True)
@@ -0,0 +1,109 @@
1
+ """LLM backend abstraction.
2
+
3
+ Claude talks over Anthropic's native SDK. Everything else - OpenAI, Gemini,
4
+ Ollama, LM Studio, llama.cpp server, vLLM, Groq, OpenRouter, whatever -
5
+ speaks the same OpenAI-style chat completions shape, so one client covers
6
+ all of them by swapping base_url."""
7
+
8
+ from __future__ import annotations
9
+
10
+ from dataclasses import dataclass
11
+
12
+ PROVIDERS = ("anthropic", "openai", "google", "ollama", "custom")
13
+
14
+ DEFAULT_BASE_URLS = {
15
+ "openai": "https://api.openai.com/v1",
16
+ "google": "https://generativelanguage.googleapis.com/v1beta/openai/",
17
+ "ollama": "http://localhost:11434/v1",
18
+ }
19
+
20
+ DEFAULT_MODELS = {
21
+ "anthropic": "claude-sonnet-5",
22
+ "openai": "gpt-4.1-mini",
23
+ "google": "gemini-2.5-flash",
24
+ "ollama": "llama3.1",
25
+ }
26
+
27
+ # local/self-hosted servers usually don't check the key at all
28
+ NO_KEY_REQUIRED = {"ollama", "custom"}
29
+
30
+
31
+ @dataclass
32
+ class ProviderConfig:
33
+ name: str
34
+ api_key: str | None = None
35
+ model: str | None = None
36
+ base_url: str | None = None
37
+
38
+ def resolved_model(self) -> str:
39
+ return self.model or DEFAULT_MODELS.get(self.name, "gpt-4o-mini")
40
+
41
+ def resolved_base_url(self) -> str | None:
42
+ return self.base_url or DEFAULT_BASE_URLS.get(self.name)
43
+
44
+
45
+ class ProviderError(RuntimeError):
46
+ pass
47
+
48
+
49
+ def complete(cfg: ProviderConfig, prompt: str, max_tokens: int = 300) -> str:
50
+ if cfg.name not in PROVIDERS:
51
+ raise ProviderError(f"unknown provider '{cfg.name}' (expected one of {', '.join(PROVIDERS)})")
52
+
53
+ if cfg.name == "anthropic":
54
+ return _complete_anthropic(cfg, prompt, max_tokens)
55
+ return _complete_openai_compatible(cfg, prompt, max_tokens)
56
+
57
+
58
+ def _complete_anthropic(cfg: ProviderConfig, prompt: str, max_tokens: int) -> str:
59
+ try:
60
+ import anthropic
61
+ except ImportError as e:
62
+ raise ProviderError("anthropic package not installed (pip install anthropic)") from e
63
+
64
+ if not cfg.api_key:
65
+ raise ProviderError(
66
+ "no Anthropic API key configured. Run `flarebisect config set-key anthropic <key>` "
67
+ "or set ANTHROPIC_API_KEY"
68
+ )
69
+
70
+ client = anthropic.Anthropic(api_key=cfg.api_key)
71
+ try:
72
+ message = client.messages.create(
73
+ model=cfg.resolved_model(),
74
+ max_tokens=max_tokens,
75
+ messages=[{"role": "user", "content": prompt}],
76
+ )
77
+ except anthropic.APIError as e:
78
+ raise ProviderError(str(e)) from e
79
+
80
+ return "".join(block.text for block in message.content if hasattr(block, "text")).strip()
81
+
82
+
83
+ def _complete_openai_compatible(cfg: ProviderConfig, prompt: str, max_tokens: int) -> str:
84
+ try:
85
+ import openai
86
+ except ImportError as e:
87
+ raise ProviderError("openai package not installed (pip install openai)") from e
88
+
89
+ api_key = cfg.api_key
90
+ if not api_key:
91
+ if cfg.name in NO_KEY_REQUIRED:
92
+ api_key = "not-needed"
93
+ else:
94
+ raise ProviderError(
95
+ f"no API key configured for '{cfg.name}'. Run "
96
+ f"`flarebisect config set-key {cfg.name} <key>` or set the matching env var"
97
+ )
98
+
99
+ client = openai.OpenAI(api_key=api_key, base_url=cfg.resolved_base_url())
100
+ try:
101
+ response = client.chat.completions.create(
102
+ model=cfg.resolved_model(),
103
+ max_tokens=max_tokens,
104
+ messages=[{"role": "user", "content": prompt}],
105
+ )
106
+ except openai.APIError as e:
107
+ raise ProviderError(str(e)) from e
108
+
109
+ return (response.choices[0].message.content or "").strip()
flarebisect/py.typed ADDED
File without changes
flarebisect/report.py ADDED
@@ -0,0 +1,117 @@
1
+ from __future__ import annotations
2
+
3
+ from rich import box
4
+ from rich.bar import Bar
5
+ from rich.console import Console, Group
6
+ from rich.panel import Panel
7
+ from rich.table import Table
8
+ from rich.text import Text
9
+
10
+ from .bisect import BisectOutcome, CommitMeasurement
11
+
12
+ console = Console()
13
+
14
+ BAR_WIDTH = 42
15
+ TRACK_COLOR = "grey19"
16
+
17
+ STATUS_STYLE = {
18
+ "good": "green3",
19
+ "stable": "green3",
20
+ "wobbling": "gold3",
21
+ "flare": "bold dark_orange",
22
+ "bad": "red3",
23
+ }
24
+
25
+ BAR_COLOR = {
26
+ "good": "grey19",
27
+ "stable": "grey19",
28
+ "wobbling": "gold3",
29
+ "flare": "dark_orange",
30
+ "bad": "red3",
31
+ }
32
+
33
+
34
+ def print_progress_line(total_commits: int, runs: int, workers: int) -> None:
35
+ console.print()
36
+ console.print(
37
+ f"[dim]{total_commits} commits in range · {runs} runs per commit · "
38
+ f"parallel worktrees ({workers} workers)[/dim]"
39
+ )
40
+ console.print()
41
+
42
+
43
+ def _row(m: CommitMeasurement) -> tuple:
44
+ short_sha = m.sha[:7]
45
+ is_flare = m.status == "flare"
46
+ commit_text = Text(short_sha, style="bold white" if is_flare else "grey70")
47
+
48
+ bar = Bar(
49
+ size=1.0,
50
+ begin=0,
51
+ end=m.result.flake_rate,
52
+ width=BAR_WIDTH,
53
+ color=BAR_COLOR.get(m.status, "grey50"),
54
+ bgcolor=TRACK_COLOR,
55
+ )
56
+
57
+ result_text = Text(f"{m.result.passed}/{m.result.runs}", style="bold white" if is_flare else "grey70")
58
+ status_text = Text(m.status, style=STATUS_STYLE.get(m.status, "grey70"))
59
+
60
+ return commit_text, bar, result_text, status_text
61
+
62
+
63
+ def print_table(measurements: list[CommitMeasurement]) -> None:
64
+ table = Table(box=None, show_header=True, header_style="dim", padding=(0, 2, 0, 0))
65
+ table.add_column("commit")
66
+ table.add_column("flake rate")
67
+ table.add_column("result", justify="right")
68
+ table.add_column("status")
69
+
70
+ for m in measurements:
71
+ table.add_row(*_row(m))
72
+
73
+ console.print(table)
74
+
75
+
76
+ def print_result(outcome: BisectOutcome, threshold: float, explanation: str | None = None) -> None:
77
+ before_pct = outcome.before.result.flake_rate
78
+ after_pct = outcome.culprit.result.flake_rate
79
+
80
+ header = Text.from_markup(
81
+ f"[bold]✦ culprit found[/bold] · commit [bold]{outcome.culprit.sha[:7]}[/bold]"
82
+ )
83
+ body_lines = [
84
+ header,
85
+ Text(""),
86
+ Text.from_markup(
87
+ f"flake rate jumped [bold]{before_pct:.0%} → {after_pct:.0%}[/bold] at this commit"
88
+ ),
89
+ Text(f'"{outcome.culprit.subject}"', style="italic grey70"),
90
+ ]
91
+
92
+ renderables = [*body_lines]
93
+ if explanation:
94
+ renderables.append(Text(""))
95
+ renderables.append(
96
+ Panel(
97
+ Text(f"💡 likely cause — {explanation}"),
98
+ border_style="grey35",
99
+ box=box.MINIMAL,
100
+ style="on grey11",
101
+ padding=(0, 2),
102
+ )
103
+ )
104
+
105
+ console.print()
106
+ console.print(Panel(Group(*renderables), border_style="red3", padding=(1, 2)))
107
+
108
+
109
+ def print_footer(elapsed_seconds: float, checked: int, workers: int) -> None:
110
+ console.print(
111
+ f"[dim]⏱ {elapsed_seconds:.1f}s ⚡ {checked} checked ⚙ {workers} parallel workers[/dim]"
112
+ )
113
+ console.print()
114
+
115
+
116
+ def print_error(message: str) -> None:
117
+ console.print(f"[bold red]error:[/bold red] {message}")
flarebisect/runner.py ADDED
@@ -0,0 +1,45 @@
1
+ """Run a test command N times against a worktree and compute a flake rate."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import subprocess
7
+ from concurrent.futures import ThreadPoolExecutor
8
+ from dataclasses import dataclass
9
+ from pathlib import Path
10
+
11
+
12
+ @dataclass
13
+ class RunResult:
14
+ passed: int
15
+ failed: int
16
+ runs: int
17
+
18
+ @property
19
+ def flake_rate(self) -> float:
20
+ return self.failed / self.runs if self.runs else 0.0
21
+
22
+
23
+ def worker_count_for(runs: int) -> int:
24
+ """Don't oversubscribe more parallel test runs than the machine has cores."""
25
+ cores = os.cpu_count() or 4
26
+ return max(1, min(runs, cores))
27
+
28
+
29
+ def _one_run(test_cmd: str, cwd: Path) -> bool:
30
+ result = subprocess.run(
31
+ test_cmd,
32
+ shell=True,
33
+ cwd=cwd,
34
+ capture_output=True,
35
+ text=True,
36
+ )
37
+ return result.returncode == 0
38
+
39
+
40
+ def measure_flake_rate(test_cmd: str, cwd: Path, runs: int) -> RunResult:
41
+ with ThreadPoolExecutor(max_workers=worker_count_for(runs)) as pool:
42
+ outcomes = list(pool.map(lambda _: _one_run(test_cmd, cwd), range(runs)))
43
+ passed = sum(outcomes)
44
+ failed = runs - passed
45
+ return RunResult(passed=passed, failed=failed, runs=runs)
@@ -0,0 +1,192 @@
1
+ Metadata-Version: 2.4
2
+ Name: flarebisect
3
+ Version: 0.3.1
4
+ Summary: git bisect that treats flakiness as a signal, not noise
5
+ Project-URL: Homepage, https://github.com/kaorii-ako/FlareBisect
6
+ Project-URL: Repository, https://github.com/kaorii-ako/FlareBisect
7
+ Project-URL: Issues, https://github.com/kaorii-ako/FlareBisect/issues
8
+ Author: Tawin Tangsukson
9
+ License: MIT
10
+ License-File: LICENSE
11
+ Keywords: bisect,cli,debugging,flaky-tests,git,testing
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Environment :: Console
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Topic :: Software Development :: Debuggers
23
+ Classifier: Topic :: Software Development :: Testing
24
+ Classifier: Topic :: Software Development :: Version Control :: Git
25
+ Requires-Python: >=3.10
26
+ Requires-Dist: anthropic>=0.40
27
+ Requires-Dist: openai>=1.40
28
+ Requires-Dist: rich>=13.7
29
+ Requires-Dist: typer>=0.12
30
+ Provides-Extra: dev
31
+ Requires-Dist: build>=1.2; extra == 'dev'
32
+ Requires-Dist: pytest>=8.0; extra == 'dev'
33
+ Requires-Dist: twine>=5.0; extra == 'dev'
34
+ Description-Content-Type: text/markdown
35
+
36
+ # FlareBisect 🔥
37
+
38
+ [![PyPI](https://img.shields.io/pypi/v/flarebisect.svg)](https://pypi.org/project/flarebisect/)
39
+ [![CI](https://github.com/kaorii-ako/FlareBisect/actions/workflows/ci.yml/badge.svg)](https://github.com/kaorii-ako/FlareBisect/actions/workflows/ci.yml)
40
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
41
+
42
+ FlareBisect finds the commit that made your tests flaky, not just the one
43
+ that broke them: it bisects on flake-rate drift instead of pass/fail, then
44
+ flares the culprit with an AI-written root cause.
45
+
46
+ `git bisect` gives you confidently wrong answers when the test you're
47
+ bisecting on is flaky — it treats every run as a clean pass/fail signal, so
48
+ a test that was already unstable *before* you started bisecting can point
49
+ straight at an innocent commit.
50
+
51
+ `flarebisect` bisects on **flake rate**, not pass/fail. It runs each
52
+ candidate commit's test N times in parallel, computes a failure rate, and
53
+ finds the commit where that rate jumps — then asks an LLM to explain, in
54
+ plain English, whether the commit broke the test cleanly or made it flakier.
55
+
56
+ ```
57
+ 12 commits in range · 20 runs per commit · parallel worktrees
58
+
59
+ commit flake rate result status
60
+ a1b2c3 5/5 good
61
+ 4f9e21 5/5 stable
62
+ 7d3aa0 ███████ 4/5 wobbling
63
+ 9c1f88 ████████████████████████████ 1/5 flare
64
+ HEAD ████████████████████████████ 1/5 bad
65
+
66
+ ╭────────────────────────────────────────────────────────────────╮
67
+ │ ✦ culprit found · commit 9c1f88       │
68
+ │           │
69
+ │ flake rate jumped 20% → 80% at this commit           │
70
+ │ "add request counter for rate limiting"           │
71
+ │           │
72
+ │ 💡 likely cause — shared counter incremented without a lock.        │
73
+ │ concurrent test workers race on the same variable.           │
74
+ ╰────────────────────────────────────────────────────────────────╯
75
+ ⏱ 8.4s ⚡ 5 checked ⚙ 4 parallel workers
76
+ ```
77
+
78
+ ## Install
79
+
80
+ ```bash
81
+ pip install flarebisect
82
+ ```
83
+
84
+ For local development:
85
+
86
+ ```bash
87
+ pip install -e ".[dev]"
88
+ pytest -q
89
+ ```
90
+
91
+ Works on Linux, macOS, and Windows — the only external dependency is `git`
92
+ on your `PATH`. Every candidate commit runs in its own `git worktree` under a
93
+ temp directory; your actual working tree and index are never touched.
94
+
95
+ ## AI provider setup
96
+
97
+ The root-cause explanation step works with Claude, OpenAI, Gemini, or a
98
+ local model — anything speaking an OpenAI-compatible chat API (Ollama, LM
99
+ Studio, llama.cpp server, vLLM, and similar all qualify).
100
+
101
+ ```bash
102
+ # cloud
103
+ flarebisect config set-key anthropic sk-ant-...
104
+ flarebisect config set-key openai sk-...
105
+ flarebisect config set-key google AI...
106
+
107
+ # local, no key needed - just have Ollama running
108
+ flarebisect config use ollama
109
+ flarebisect config set-model ollama llama3.1
110
+
111
+ # or let flarebisect pick a model sized to your GPU and pull it for you
112
+ flarebisect models detect # shows detected GPU/VRAM + recommended model
113
+ flarebisect models pull # downloads it and sets it active (run with no args)
114
+
115
+ # any other OpenAI-compatible endpoint
116
+ flarebisect config use custom
117
+ flarebisect config set-base-url custom http://localhost:8080/v1
118
+ flarebisect config set-key custom sk-local
119
+ ```
120
+
121
+ Run `flarebisect config` with no arguments for a guided setup wizard — for
122
+ Ollama it detects your GPU/VRAM, prefills a right-sized model, and offers to
123
+ pull it on the spot.
124
+
125
+ `flarebisect config show` lists the active provider and stored settings
126
+ (keys are masked). Config lives in a JSON file under the OS config dir
127
+ (`~/.config/flarebisect/config.json` on Linux/macOS, `%APPDATA%` on Windows),
128
+ written with owner-only permissions.
129
+
130
+ Per-run overrides skip the config file entirely:
131
+
132
+ ```bash
133
+ flarebisect run ... --provider openai --model gpt-4o-mini --api-key sk-...
134
+ flarebisect run ... --provider ollama --base-url http://localhost:11434/v1
135
+ ```
136
+
137
+ Env vars (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GOOGLE_API_KEY`) work too,
138
+ as a fallback under whatever's in the config file.
139
+
140
+ ## Usage
141
+
142
+ ```bash
143
+ flarebisect run \
144
+ --repo /path/to/repo \
145
+ --good <known-good-sha> \
146
+ --bad <known-bad-sha> \
147
+ --test "pytest -k my_flaky_test" \
148
+ --runs 20 \
149
+ --threshold 0.3
150
+ ```
151
+
152
+ - `--runs` — test executions per commit (default 20), run in parallel
153
+ (capped at your core count so it doesn't oversubscribe). Lower counts are
154
+ faster but noisier — a low sample can misattribute the culprit near the
155
+ threshold boundary.
156
+ - `--threshold` — flake-rate jump (0-1) vs. the `good` baseline required to
157
+ call a commit the culprit (default 0.3).
158
+ - `--no-explain` — skip the LLM root-cause call entirely (fully offline).
159
+
160
+ ## Demo
161
+
162
+ See [`demo/`](demo/) for a self-contained, seeded-flaky-bug repo you can
163
+ bisect against with no network access required for the bisection itself.
164
+
165
+ ## How it works
166
+
167
+ 1. Measure the flake rate at `good` and `bad` as baselines.
168
+ 2. Binary-search the commit range between them.
169
+ 3. At each candidate, run the test `N` times in an isolated worktree and
170
+ compute `failed / N`.
171
+ 4. The culprit is the first commit whose flake rate jumps by `>= threshold`
172
+ over the `good` baseline.
173
+ 5. Verdict: **clean break** (rate goes ~0% → ~100%) vs. **flakiness
174
+ regression** (rate goes ~0% → somewhere in between).
175
+ 6. The configured LLM gets the culprit's diff + the flake-rate evidence and
176
+ returns a short root-cause read (race condition, shared mutable state,
177
+ timing assumption, etc.).
178
+
179
+ Binary search assumes the flake rate is roughly monotonic across the range,
180
+ same as ordinary `git bisect` assumes pass/fail is. At low `--runs` counts
181
+ a noisy sample can occasionally violate that near the threshold boundary —
182
+ bump `--runs` if a result looks off.
183
+
184
+ ## Releasing
185
+
186
+ Version is single-sourced from `src/flarebisect/__init__.py`. Bump it, commit,
187
+ tag (`vX.Y.Z`), and cut a GitHub Release — `.github/workflows/publish.yml`
188
+ builds and publishes to PyPI via trusted publishing (OIDC, no stored token).
189
+
190
+ ## License
191
+
192
+ MIT
@@ -0,0 +1,16 @@
1
+ flarebisect/__init__.py,sha256=r4xAFihOf72W9TD-lpMi6ntWSTKTP2SlzKP1ytkjRbI,22
2
+ flarebisect/ai_explain.py,sha256=O0xW49W6pFQOrP3DB0EO7cd1AYtLxIdWGOQP2cTl4a0,1184
3
+ flarebisect/bisect.py,sha256=SCgOcAJZ7U0o87anYm_ZdMO4d7Hz7uodv722nkZHdVA,4330
4
+ flarebisect/cli.py,sha256=H-4DLj_F7BE-FSx8fqPPWZlKyMSQ8yKSKCTNhr6q9wo,12753
5
+ flarebisect/config.py,sha256=JpI7vPpfTPTPkNGIa_4GZM0AtbSZE-5Dsw3SM1WzIbg,2527
6
+ flarebisect/git_ops.py,sha256=XOq5LH9AaZXpfSIvcDxWxlZ0iP3Z_NczMysh7YY-U7c,2050
7
+ flarebisect/hardware.py,sha256=SkiLOM1yMH1pl4-L595bzU27LN3hgG1W2Fkb5e6bE-g,4755
8
+ flarebisect/providers.py,sha256=57P_Tu598TNpiNt6PBNVnqcg8rKR9Z0hMtK2Aj50BTA,3520
9
+ flarebisect/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
10
+ flarebisect/report.py,sha256=Chr28w7MGNCNn01gRXOGG5H5wNKZgnwkUjNmZ4uVkfE,3275
11
+ flarebisect/runner.py,sha256=EriLrFVH3dx9h4R95hqzUqjfg5QYsu-U5s6HwPT3d0g,1200
12
+ flarebisect-0.3.1.dist-info/METADATA,sha256=G5sGUSY1_FSbXKqCqBL9D3hgJBWnBcu3l2yQihaAXuc,8228
13
+ flarebisect-0.3.1.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
14
+ flarebisect-0.3.1.dist-info/entry_points.txt,sha256=cWg78Ga7VgIQD9vE_lrVlD4GmWzVTsi1Xd3DuceiZOI,52
15
+ flarebisect-0.3.1.dist-info/licenses/LICENSE,sha256=PHi-k4j6Jr3rg_5SrM1w_x5iqcJhk3fx2bPQlB6MAJE,1073
16
+ flarebisect-0.3.1.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.31.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ flarebisect = flarebisect.cli:app
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Tawin Tangsukson
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.