promptdiff-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
promptdiff/__init__.py ADDED
@@ -0,0 +1 @@
1
+ # PromptDiff package
@@ -0,0 +1 @@
1
+ # API package
@@ -0,0 +1 @@
1
+ # CLI package
@@ -0,0 +1,52 @@
1
+ import os
2
+ import json
3
+ from pathlib import Path
4
+ from contextlib import contextmanager
5
+ from typing import Generator
6
+ from sqlalchemy.orm import Session
7
+ import typer
8
+
9
+ from promptdiff.db.session import get_db
10
+
11
+ def get_config_dir() -> Path:
12
+ """Get the configuration directory path, allowing environment override for tests."""
13
+ env_home = os.environ.get("PROMPTDIFF_HOME")
14
+ if env_home:
15
+ return Path(env_home)
16
+ return Path.home() / ".promptdiff"
17
+
18
+ def get_config_path() -> Path:
19
+ """Get the path to the config.json file."""
20
+ return get_config_dir() / "config.json"
21
+
22
+ def resolve_project(explicit: str | None) -> str:
23
+ """Resolve the project name using explicit override or current project config.
24
+
25
+ Raises typer.Exit(code=1) if no project name can be resolved.
26
+ """
27
+ if explicit:
28
+ return explicit
29
+
30
+ config_file = get_config_path()
31
+ if config_file.is_file():
32
+ try:
33
+ with open(config_file, "r") as f:
34
+ data = json.load(f)
35
+ current = data.get("current_project")
36
+ if current:
37
+ return current
38
+ except Exception:
39
+ pass
40
+
41
+ typer.secho(
42
+ "Error: No project specified. Use --project or run `promptdiff use <project>` first.",
43
+ fg=typer.colors.RED,
44
+ err=True
45
+ )
46
+ raise typer.Exit(code=1)
47
+
48
+ @contextmanager
49
+ def get_cli_db() -> Generator[Session, None, None]:
50
+ """Yield a database session and guarantee it closes correctly."""
51
+ with get_db() as db:
52
+ yield db
promptdiff/cli/main.py ADDED
@@ -0,0 +1,338 @@
1
+ import json
2
+ from pathlib import Path
3
+ from typing import List
4
+ import typer
5
+ from rich.console import Console
6
+ from rich.table import Table
7
+ from rich.panel import Panel
8
+ from rich.text import Text
9
+ import sys
10
+
11
+ def get_symbol(char: str, fallback: str) -> str:
12
+ try:
13
+ encoding = sys.stdout.encoding or "utf-8"
14
+ char.encode(encoding)
15
+ return char
16
+ except Exception:
17
+ return fallback
18
+
19
+ CHECKMARK = get_symbol("✓", "+")
20
+ CROSSMARK = get_symbol("✗", "x")
21
+
22
+ from promptdiff.cli.context import resolve_project, get_cli_db, get_config_dir, get_config_path
23
+ import promptdiff.core.projects as projects_core
24
+ import promptdiff.core.prompts as prompts_core
25
+ import promptdiff.core.versions as versions_core
26
+ import promptdiff.core.diff as diff_core
27
+ import promptdiff.core.runner as runner_core
28
+ from promptdiff.core.exceptions import (
29
+ ProjectAlreadyExists,
30
+ ProjectNotFound,
31
+ PromptAlreadyExists,
32
+ PromptNotFound,
33
+ VersionNotFound,
34
+ PromptDiffError
35
+ )
36
+
37
+ app = typer.Typer(no_args_is_help=True, help="PromptDiff — Git-like prompt versioning and LLM testing CLI.")
38
+ project_app = typer.Typer(no_args_is_help=True, help="Manage PromptDiff projects.")
39
+ app.add_typer(project_app, name="project")
40
+
41
+ console = Console()
42
+ err_console = Console(stderr=True)
43
+
44
+ # --- Project Commands ---
45
+
46
+ @project_app.command("create")
47
+ def project_create(
48
+ name: str = typer.Argument(..., help="Name of the project to create."),
49
+ description: str = typer.Option(None, "--description", "-d", help="Description of the project.")
50
+ ):
51
+ """Create a new project."""
52
+ with get_cli_db() as db:
53
+ try:
54
+ projects_core.create_project(db, name, description)
55
+ console.print(f"[bold green]{CHECKMARK}[/bold green] Created project '[bold]{name}[/bold]'")
56
+ except ProjectAlreadyExists as e:
57
+ console.print(f"[bold red]Error:[/bold red] {e}")
58
+ raise typer.Exit(code=1)
59
+
60
+ @project_app.command("list")
61
+ def project_list():
62
+ """List all projects."""
63
+ with get_cli_db() as db:
64
+ projects = projects_core.list_projects(db)
65
+ if not projects:
66
+ console.print("[yellow]No projects found.[/yellow]")
67
+ return
68
+
69
+ table = Table(title="PromptDiff Projects", border_style="cyan")
70
+ table.add_column("Name", style="bold magenta")
71
+ table.add_column("Description")
72
+ table.add_column("Created At", style="dim")
73
+
74
+ for p in projects:
75
+ created_str = p.created_at.strftime("%Y-%m-%d %H:%M:%S") if p.created_at else ""
76
+ table.add_row(p.name, p.description or "", created_str)
77
+
78
+ console.print(table)
79
+
80
+ # --- Workflow Commands ---
81
+
82
+ @app.command("use")
83
+ def use_project(
84
+ name: str = typer.Argument(..., help="Name of the project to use by default.")
85
+ ):
86
+ """Set the current project context."""
87
+ with get_cli_db() as db:
88
+ project = projects_core.get_project(db, name)
89
+ if not project:
90
+ err_console.print(f"[bold red]Error:[/bold red] Project '{name}' does not exist.")
91
+ err_console.print(f"[yellow]Suggest running:[/yellow] promptdiff project create {name}")
92
+ raise typer.Exit(code=1)
93
+
94
+ config_dir = get_config_dir()
95
+ config_dir.mkdir(parents=True, exist_ok=True)
96
+ config_file = get_config_path()
97
+ try:
98
+ with open(config_file, "w") as f:
99
+ json.dump({"current_project": name}, f)
100
+ console.print(f"[bold green]{CHECKMARK}[/bold green] Using project '[bold]{name}[/bold]'")
101
+ except Exception as e:
102
+ err_console.print(f"[bold red]Error saving config:[/bold red] {e}")
103
+ raise typer.Exit(code=1)
104
+
105
+ @app.command("add")
106
+ def add_prompt(
107
+ prompt_name: str = typer.Argument(..., help="Name of the prompt to add."),
108
+ project: str = typer.Option(None, "--project", "-p", help="Project name override.")
109
+ ):
110
+ """Add a new prompt to the project."""
111
+ project_name = resolve_project(project)
112
+ with get_cli_db() as db:
113
+ try:
114
+ prompts_core.create_prompt(db, project_name, prompt_name)
115
+ console.print(f"[bold green]{CHECKMARK}[/bold green] Added prompt '[bold]{prompt_name}[/bold]' to project '[bold]{project_name}[/bold]'")
116
+ except (ProjectNotFound, PromptAlreadyExists) as e:
117
+ console.print(f"[bold red]Error:[/bold red] {e}")
118
+ raise typer.Exit(code=1)
119
+
120
+ @app.command("commit")
121
+ def commit_prompt(
122
+ prompt_name: str = typer.Argument(..., help="Name of the prompt to commit to."),
123
+ content_or_file: str = typer.Argument(
124
+ ...,
125
+ help="Literal prompt content OR path to a text file containing the prompt content."
126
+ ),
127
+ message: str = typer.Option(None, "--message", "-m", help="Commit message describing the changes."),
128
+ project: str = typer.Option(None, "--project", "-p", help="Project name override.")
129
+ ):
130
+ """Commit a new version of a prompt's content."""
131
+ project_name = resolve_project(project)
132
+
133
+ # Smart check: file vs literal content
134
+ path = Path(content_or_file)
135
+ if path.is_file():
136
+ try:
137
+ content = path.read_text(encoding="utf-8")
138
+ except Exception as e:
139
+ console.print(f"[bold red]Error reading file '{content_or_file}':[/bold red] {e}")
140
+ raise typer.Exit(code=1)
141
+ else:
142
+ content = content_or_file
143
+
144
+ with get_cli_db() as db:
145
+ try:
146
+ version = versions_core.commit_version(db, project_name, prompt_name, content, message)
147
+ console.print(f"[bold green]{CHECKMARK}[/bold green] Committed prompt '[bold]{prompt_name}[/bold]' as v{version.version_number}")
148
+ except PromptNotFound as e:
149
+ console.print(f"[bold red]Error:[/bold red] {e}")
150
+ raise typer.Exit(code=1)
151
+
152
+ @app.command("log")
153
+ def log_prompt(
154
+ prompt_name: str = typer.Argument(..., help="Name of the prompt to show logs for."),
155
+ project: str = typer.Option(None, "--project", "-p", help="Project name override.")
156
+ ):
157
+ """Show the version commit history of a prompt."""
158
+ project_name = resolve_project(project)
159
+ with get_cli_db() as db:
160
+ try:
161
+ versions = versions_core.list_versions(db, project_name, prompt_name)
162
+ if not versions:
163
+ console.print(f"[yellow]No commits found for prompt '{prompt_name}'.[/yellow]")
164
+ return
165
+
166
+ table = Table(title=f"Commit Log: {prompt_name} (Project: {project_name})", border_style="cyan")
167
+ table.add_column("Version", style="bold magenta")
168
+ table.add_column("Commit Message")
169
+ table.add_column("Created At", style="dim")
170
+ table.add_column("Content Preview", style="italic dim")
171
+
172
+ for v in versions:
173
+ created_str = v.created_at.strftime("%Y-%m-%d %H:%M:%S") if v.created_at else ""
174
+ preview = v.content[:50].replace("\n", " ")
175
+ if len(v.content) > 50:
176
+ preview += "..."
177
+ table.add_row(f"v{v.version_number}", v.commit_message or "", created_str, preview)
178
+
179
+ console.print(table)
180
+ except PromptNotFound as e:
181
+ console.print(f"[bold red]Error:[/bold red] {e}")
182
+ raise typer.Exit(code=1)
183
+
184
+ @app.command("diff")
185
+ def diff_prompt(
186
+ prompt_name: str = typer.Argument(..., help="Name of the prompt to compare."),
187
+ version_a: int = typer.Argument(..., help="Source version number."),
188
+ version_b: int = typer.Argument(..., help="Target version number."),
189
+ project: str = typer.Option(None, "--project", "-p", help="Project name override.")
190
+ ):
191
+ """Compare two versions of a prompt."""
192
+ project_name = resolve_project(project)
193
+ with get_cli_db() as db:
194
+ v_a = versions_core.get_version(db, project_name, prompt_name, version_a)
195
+ if not v_a:
196
+ console.print(f"[bold red]Error:[/bold red] Version {version_a} of prompt '{prompt_name}' in project '{project_name}' not found.")
197
+ raise typer.Exit(code=1)
198
+ v_b = versions_core.get_version(db, project_name, prompt_name, version_b)
199
+ if not v_b:
200
+ console.print(f"[bold red]Error:[/bold red] Version {version_b} of prompt '{prompt_name}' in project '{project_name}' not found.")
201
+ raise typer.Exit(code=1)
202
+
203
+ diff_result = diff_core.diff_versions(v_a, v_b)
204
+ console.print(f"[bold]Diffing prompt '{prompt_name}' (Project: {project_name}) between v{version_a} and v{version_b}:[/bold]\n")
205
+
206
+ for line in diff_result.lines:
207
+ if line.line_type == "unchanged":
208
+ console.print(f" {line.new_content or ''}")
209
+ elif line.line_type == "added":
210
+ console.print(f"+ {line.new_content or ''}", style="green")
211
+ elif line.line_type == "removed":
212
+ console.print(f"- {line.old_content or ''}", style="red")
213
+ elif line.line_type == "modified":
214
+ text = Text()
215
+ text.append("~ ")
216
+ for wd in line.word_diffs or []:
217
+ if wd.change_type == "unchanged":
218
+ text.append(wd.text)
219
+ elif wd.change_type == "added":
220
+ text.append(wd.text, style="bold green")
221
+ elif wd.change_type == "removed":
222
+ text.append(wd.text, style="red strike")
223
+ console.print(text)
224
+
225
+ @app.command("run")
226
+ def run_prompt_versions(
227
+ prompt_name: str = typer.Argument(..., help="Name of the prompt to execute."),
228
+ version_number: int = typer.Argument(..., help="Version number to execute."),
229
+ model: List[str] = typer.Option(
230
+ ["gemini/gemini-2.5-flash"],
231
+ "--model",
232
+ help="LLM model(s) to run the prompt on. Can be specified multiple times."
233
+ ),
234
+ project: str = typer.Option(None, "--project", "-p", help="Project name override.")
235
+ ):
236
+ """Run a prompt version against one or more models."""
237
+ project_name = resolve_project(project)
238
+ with get_cli_db() as db:
239
+ # Check if version exists first
240
+ version = versions_core.get_version(db, project_name, prompt_name, version_number)
241
+ if not version:
242
+ console.print(
243
+ f"[bold red]Error:[/bold red] Version {version_number} of prompt '{prompt_name}' "
244
+ f"in project '{project_name}' not found."
245
+ )
246
+ raise typer.Exit(code=1)
247
+
248
+ for m in model:
249
+ with console.status(f"Running '{prompt_name}' v{version_number} on {m}..."):
250
+ run_res = runner_core.run_prompt(db, project_name, prompt_name, version_number, model=m)
251
+
252
+ if run_res.status == "success":
253
+ output_text = f"[bold green]{CHECKMARK}[/bold green] {m}: success in {run_res.latency_ms}ms"
254
+ token_info = []
255
+ if run_res.output:
256
+ if run_res.output.prompt_tokens is not None:
257
+ token_info.append(f"prompt: {run_res.output.prompt_tokens}")
258
+ if run_res.output.completion_tokens is not None:
259
+ token_info.append(f"completion: {run_res.output.completion_tokens}")
260
+ if token_info:
261
+ output_text += f" ({', '.join(token_info)})"
262
+ if run_res.output and run_res.output.cost_usd is not None:
263
+ output_text += f" | Cost: ${run_res.output.cost_usd:.5f}"
264
+ console.print(output_text)
265
+
266
+ content = run_res.output.content if run_res.output else ""
267
+ console.print(Panel(content or "", title=f"Output ({m})", border_style="green"))
268
+ else:
269
+ console.print(f"[bold red]{CROSSMARK}[/bold red] {m}: failed in {run_res.latency_ms}ms. Error: {run_res.error_message}")
270
+
271
+ @app.command("diff-output")
272
+ def diff_output(
273
+ prompt_name: str = typer.Argument(..., help="Name of the prompt."),
274
+ version_a: int = typer.Argument(..., help="First version number."),
275
+ version_b: int = typer.Argument(..., help="Second version number."),
276
+ model: str = typer.Option(None, "--model", help="Specific model to compare outputs for."),
277
+ project: str = typer.Option(None, "--project", "-p", help="Project name override.")
278
+ ):
279
+ """Compare the generated outputs of two prompt versions side by side."""
280
+ project_name = resolve_project(project)
281
+ with get_cli_db() as db:
282
+ try:
283
+ if model:
284
+ run_a = runner_core.get_latest_successful_run(db, project_name, prompt_name, version_a, model=model)
285
+ run_b = runner_core.get_latest_successful_run(db, project_name, prompt_name, version_b, model=model)
286
+
287
+ if not run_a:
288
+ err_console.print(f"[bold red]Error:[/bold red] No successful run found for version {version_a} with model '{model}'.")
289
+ raise typer.Exit(code=1)
290
+ if not run_b:
291
+ err_console.print(f"[bold red]Error:[/bold red] No successful run found for version {version_b} with model '{model}'.")
292
+ raise typer.Exit(code=1)
293
+ else:
294
+ models_a = runner_core.list_distinct_models_for_version(db, project_name, prompt_name, version_a)
295
+ models_b = runner_core.list_distinct_models_for_version(db, project_name, prompt_name, version_b)
296
+
297
+ if not models_a:
298
+ err_console.print(f"[bold red]Error:[/bold red] No successful execution run found for version {version_a}. Run it first using 'run'.")
299
+ raise typer.Exit(code=1)
300
+ if not models_b:
301
+ err_console.print(f"[bold red]Error:[/bold red] No successful execution run found for version {version_b}. Run it first using 'run'.")
302
+ raise typer.Exit(code=1)
303
+
304
+ run_a = runner_core.get_latest_successful_run(db, project_name, prompt_name, version_a)
305
+ run_b = runner_core.get_latest_successful_run(db, project_name, prompt_name, version_b)
306
+
307
+ has_ambiguity = (
308
+ len(models_a) > 1 or
309
+ len(models_b) > 1 or
310
+ run_a.model != run_b.model
311
+ )
312
+
313
+ if has_ambiguity:
314
+ err_console.print(f"[bold yellow]Ambiguity detected:[/bold yellow] Multiple models exist across versions.")
315
+ err_console.print(f"Version {version_a} models: {', '.join(models_a)}")
316
+ err_console.print(f"Version {version_b} models: {', '.join(models_b)}")
317
+ err_console.print("\nPlease specify [bold]--model[/bold] to choose which model to compare.")
318
+ raise typer.Exit(code=1)
319
+
320
+ table = Table.grid(expand=True, padding=2)
321
+ table.add_column(ratio=1)
322
+ table.add_column(ratio=1)
323
+
324
+ content_a = run_a.output.content if run_a.output else ""
325
+ content_b = run_b.output.content if run_b.output else ""
326
+
327
+ table.add_row(
328
+ Panel(content_a or "", title=f"v{version_a} Output ({run_a.model})", border_style="cyan"),
329
+ Panel(content_b or "", title=f"v{version_b} Output ({run_b.model})", border_style="magenta")
330
+ )
331
+ console.print(table)
332
+
333
+ except VersionNotFound as e:
334
+ err_console.print(f"[bold red]Error:[/bold red] {e}")
335
+ raise typer.Exit(code=1)
336
+
337
+ if __name__ == "__main__":
338
+ app()
@@ -0,0 +1 @@
1
+ # Core package
@@ -0,0 +1,126 @@
1
+ import re
2
+ from dataclasses import dataclass
3
+ from typing import Literal, List, Optional
4
+ from difflib import SequenceMatcher
5
+ from promptdiff.db.models import Version
6
+
7
+ @dataclass
8
+ class WordDiff:
9
+ text: str
10
+ change_type: Literal["unchanged", "added", "removed"]
11
+
12
+ @dataclass
13
+ class DiffLine:
14
+ line_type: Literal["unchanged", "added", "removed", "modified"]
15
+ old_content: Optional[str]
16
+ new_content: Optional[str]
17
+ word_diffs: Optional[List[WordDiff]] = None # only populated if line_type == "modified"
18
+
19
+ @dataclass
20
+ class DiffResult:
21
+ from_version: int
22
+ to_version: int
23
+ lines: List[DiffLine]
24
+
25
+ def _split_words(line: str) -> List[str]:
26
+ # Match alphanumeric words, whitespace sequences, or individual punctuation marks
27
+ return re.findall(r'\w+|\s+|[^\w\s]', line)
28
+
29
+ def _diff_words(old_line: str, new_line: str) -> List[WordDiff]:
30
+ old_words = _split_words(old_line)
31
+ new_words = _split_words(new_line)
32
+
33
+ matcher = SequenceMatcher(None, old_words, new_words)
34
+ word_diffs = []
35
+
36
+ for tag, i1, i2, j1, j2 in matcher.get_opcodes():
37
+ if tag == 'equal':
38
+ for w in old_words[i1:i2]:
39
+ word_diffs.append(WordDiff(text=w, change_type="unchanged"))
40
+ elif tag == 'delete':
41
+ for w in old_words[i1:i2]:
42
+ word_diffs.append(WordDiff(text=w, change_type="removed"))
43
+ elif tag == 'insert':
44
+ for w in new_words[j1:j2]:
45
+ word_diffs.append(WordDiff(text=w, change_type="added"))
46
+ elif tag == 'replace':
47
+ for w in old_words[i1:i2]:
48
+ word_diffs.append(WordDiff(text=w, change_type="removed"))
49
+ for w in new_words[j1:j2]:
50
+ word_diffs.append(WordDiff(text=w, change_type="added"))
51
+
52
+ return word_diffs
53
+
54
+ def diff_versions(version_a: Version, version_b: Version) -> DiffResult:
55
+ """Compare two versions line-by-line, and compute word-level diffs for modified lines.
56
+
57
+ Returns a DiffResult structure detailing the changes.
58
+ """
59
+ lines_a = version_a.content.splitlines() if version_a.content else []
60
+ lines_b = version_b.content.splitlines() if version_b.content else []
61
+
62
+ matcher = SequenceMatcher(None, lines_a, lines_b)
63
+ diff_lines = []
64
+
65
+ for tag, i1, i2, j1, j2 in matcher.get_opcodes():
66
+ if tag == 'equal':
67
+ for line in lines_a[i1:i2]:
68
+ diff_lines.append(DiffLine(
69
+ line_type="unchanged",
70
+ old_content=line,
71
+ new_content=line
72
+ ))
73
+ elif tag == 'delete':
74
+ for line in lines_a[i1:i2]:
75
+ diff_lines.append(DiffLine(
76
+ line_type="removed",
77
+ old_content=line,
78
+ new_content=None
79
+ ))
80
+ elif tag == 'insert':
81
+ for line in lines_b[j1:j2]:
82
+ diff_lines.append(DiffLine(
83
+ line_type="added",
84
+ old_content=None,
85
+ new_content=line
86
+ ))
87
+ elif tag == 'replace':
88
+ del_block = lines_a[i1:i2]
89
+ ins_block = lines_b[j1:j2]
90
+
91
+ # Pair them up where possible
92
+ min_len = min(len(del_block), len(ins_block))
93
+ for idx in range(min_len):
94
+ old_l = del_block[idx]
95
+ new_l = ins_block[idx]
96
+ w_diff = _diff_words(old_l, new_l)
97
+ diff_lines.append(DiffLine(
98
+ line_type="modified",
99
+ old_content=old_l,
100
+ new_content=new_l,
101
+ word_diffs=w_diff
102
+ ))
103
+
104
+ # Remaining deletions (treated as pure removed)
105
+ if len(del_block) > min_len:
106
+ for line in del_block[min_len:]:
107
+ diff_lines.append(DiffLine(
108
+ line_type="removed",
109
+ old_content=line,
110
+ new_content=None
111
+ ))
112
+
113
+ # Remaining insertions (treated as pure added)
114
+ if len(ins_block) > min_len:
115
+ for line in ins_block[min_len:]:
116
+ diff_lines.append(DiffLine(
117
+ line_type="added",
118
+ old_content=None,
119
+ new_content=line
120
+ ))
121
+
122
+ return DiffResult(
123
+ from_version=version_a.version_number,
124
+ to_version=version_b.version_number,
125
+ lines=diff_lines
126
+ )
@@ -0,0 +1,23 @@
1
+ class PromptDiffError(Exception):
2
+ """Base exception for all PromptDiff errors."""
3
+ pass
4
+
5
+ class ProjectNotFound(PromptDiffError):
6
+ """Raised when a requested project is not found."""
7
+ pass
8
+
9
+ class ProjectAlreadyExists(PromptDiffError):
10
+ """Raised when trying to create a project that already exists."""
11
+ pass
12
+
13
+ class PromptNotFound(PromptDiffError):
14
+ """Raised when a requested prompt is not found."""
15
+ pass
16
+
17
+ class PromptAlreadyExists(PromptDiffError):
18
+ """Raised when trying to create a prompt that already exists."""
19
+ pass
20
+
21
+ class VersionNotFound(PromptDiffError):
22
+ """Raised when a requested prompt version is not found."""
23
+ pass
@@ -0,0 +1,27 @@
1
+ from sqlalchemy.orm import Session
2
+ from sqlalchemy.exc import IntegrityError
3
+ from promptdiff.db.models import Project
4
+ from promptdiff.core.exceptions import ProjectAlreadyExists
5
+
6
+ def create_project(db: Session, name: str, description: str | None = None) -> Project:
7
+ """Create a new project.
8
+
9
+ Raises ProjectAlreadyExists if a project with the same name exists.
10
+ """
11
+ project = Project(name=name, description=description)
12
+ db.add(project)
13
+ try:
14
+ db.commit()
15
+ db.refresh(project)
16
+ except IntegrityError as e:
17
+ db.rollback()
18
+ raise ProjectAlreadyExists(f"Project with name '{name}' already exists.") from e
19
+ return project
20
+
21
+ def get_project(db: Session, name: str) -> Project | None:
22
+ """Retrieve a project by its name."""
23
+ return db.query(Project).filter(Project.name == name).first()
24
+
25
+ def list_projects(db: Session) -> list[Project]:
26
+ """List all projects in the database."""
27
+ return db.query(Project).all()
@@ -0,0 +1,45 @@
1
+ from sqlalchemy.orm import Session
2
+ from sqlalchemy.exc import IntegrityError
3
+ from promptdiff.db.models import Prompt, Project
4
+ from promptdiff.core.exceptions import ProjectNotFound, PromptAlreadyExists
5
+
6
+ def create_prompt(db: Session, project_name: str, prompt_name: str) -> Prompt:
7
+ """Create a new prompt within a specified project.
8
+
9
+ Raises ProjectNotFound if the project doesn't exist.
10
+ Raises PromptAlreadyExists if a prompt with the same name exists in the project.
11
+ """
12
+ project = db.query(Project).filter(Project.name == project_name).first()
13
+ if not project:
14
+ raise ProjectNotFound(f"Project '{project_name}' not found.")
15
+
16
+ prompt = Prompt(project_id=project.id, name=prompt_name)
17
+ db.add(prompt)
18
+ try:
19
+ db.commit()
20
+ db.refresh(prompt)
21
+ except IntegrityError as e:
22
+ db.rollback()
23
+ raise PromptAlreadyExists(
24
+ f"Prompt '{prompt_name}' already exists in project '{project_name}'."
25
+ ) from e
26
+ return prompt
27
+
28
+ def get_prompt(db: Session, project_name: str, prompt_name: str) -> Prompt | None:
29
+ """Retrieve a prompt by project name and prompt name."""
30
+ return (
31
+ db.query(Prompt)
32
+ .join(Project)
33
+ .filter(Project.name == project_name, Prompt.name == prompt_name)
34
+ .first()
35
+ )
36
+
37
+ def list_prompts(db: Session, project_name: str) -> list[Prompt]:
38
+ """List all prompts under a specified project.
39
+
40
+ Raises ProjectNotFound if the project doesn't exist.
41
+ """
42
+ project = db.query(Project).filter(Project.name == project_name).first()
43
+ if not project:
44
+ raise ProjectNotFound(f"Project '{project_name}' not found.")
45
+ return project.prompts
@@ -0,0 +1,213 @@
1
+ import time
2
+ import litellm
3
+ from sqlalchemy.orm import Session
4
+ from dotenv import load_dotenv
5
+
6
+ from promptdiff.db.models import Version, Run, Output, Prompt, Project
7
+ from promptdiff.core.exceptions import VersionNotFound
8
+
9
+ # Load environment variables (such as API keys)
10
+ load_dotenv()
11
+
12
+ def run_prompt(
13
+ db: Session,
14
+ project_name: str,
15
+ prompt_name: str,
16
+ version_number: int,
17
+ model: str = "gemini/gemini-2.5-flash"
18
+ ) -> Run:
19
+ """Run a prompt version using LiteLLM and store results.
20
+
21
+ Creates a pending run, executes it, measures latency, and registers the outcome.
22
+ On success, creates an Output record with response metadata.
23
+ On failure, logs the error details.
24
+ Raises VersionNotFound if the version does not exist.
25
+ """
26
+ version = (
27
+ db.query(Version)
28
+ .join(Prompt)
29
+ .join(Project)
30
+ .filter(
31
+ Project.name == project_name,
32
+ Prompt.name == prompt_name,
33
+ Version.version_number == version_number
34
+ )
35
+ .first()
36
+ )
37
+ if not version:
38
+ raise VersionNotFound(
39
+ f"Version {version_number} of prompt '{prompt_name}' in project '{project_name}' not found."
40
+ )
41
+
42
+ # Insert pending Run record
43
+ run = Run(
44
+ version_id=version.id,
45
+ model=model,
46
+ status="pending"
47
+ )
48
+ db.add(run)
49
+ db.commit()
50
+ db.refresh(run)
51
+
52
+ start_time = time.monotonic()
53
+ try:
54
+ # Call LiteLLM completion
55
+ response = litellm.completion(
56
+ model=model,
57
+ messages=[{"role": "user", "content": version.content}]
58
+ )
59
+ latency_ms = int((time.monotonic() - start_time) * 1000)
60
+
61
+ # Extract content, token counts, and cost
62
+ resp_content = None
63
+ prompt_tokens = None
64
+ completion_tokens = None
65
+
66
+ if hasattr(response, "choices") and response.choices:
67
+ resp_content = response.choices[0].message.content
68
+ elif isinstance(response, dict) and "choices" in response and response["choices"]:
69
+ resp_content = response["choices"][0]["message"]["content"]
70
+
71
+ if hasattr(response, "usage") and response.usage:
72
+ prompt_tokens = getattr(response.usage, "prompt_tokens", None)
73
+ completion_tokens = getattr(response.usage, "completion_tokens", None)
74
+ elif isinstance(response, dict) and "usage" in response:
75
+ prompt_tokens = response["usage"].get("prompt_tokens")
76
+ completion_tokens = response["usage"].get("completion_tokens")
77
+
78
+ cost_usd = None
79
+ try:
80
+ cost_usd = litellm.completion_cost(completion_response=response)
81
+ except Exception:
82
+ pass
83
+
84
+ # Update run to success
85
+ run.status = "success"
86
+ run.latency_ms = latency_ms
87
+ db.commit()
88
+
89
+ # Save run output details
90
+ output = Output(
91
+ run_id=run.id,
92
+ content=resp_content,
93
+ prompt_tokens=prompt_tokens,
94
+ completion_tokens=completion_tokens,
95
+ cost_usd=cost_usd
96
+ )
97
+ db.add(output)
98
+ db.commit()
99
+ db.refresh(run)
100
+
101
+ except Exception as e:
102
+ latency_ms = int((time.monotonic() - start_time) * 1000)
103
+ db.rollback()
104
+
105
+ # Clean error string representation
106
+ error_msg = str(e)[:1000]
107
+
108
+ # Update run as failed
109
+ run.status = "failed"
110
+ run.error_message = error_msg
111
+ run.latency_ms = latency_ms
112
+ db.commit()
113
+ db.refresh(run)
114
+
115
+ return run
116
+
117
+ def list_runs_for_version(
118
+ db: Session,
119
+ project_name: str,
120
+ prompt_name: str,
121
+ version_number: int
122
+ ) -> list[Run]:
123
+ """List all execution runs for a specific version.
124
+
125
+ Raises VersionNotFound if the version does not exist.
126
+ """
127
+ version = (
128
+ db.query(Version)
129
+ .join(Prompt)
130
+ .join(Project)
131
+ .filter(
132
+ Project.name == project_name,
133
+ Prompt.name == prompt_name,
134
+ Version.version_number == version_number
135
+ )
136
+ .first()
137
+ )
138
+ if not version:
139
+ raise VersionNotFound(
140
+ f"Version {version_number} of prompt '{prompt_name}' in project '{project_name}' not found."
141
+ )
142
+
143
+ return (
144
+ db.query(Run)
145
+ .filter(Run.version_id == version.id)
146
+ .order_by(Run.created_at.desc())
147
+ .all()
148
+ )
149
+
150
+ def get_run_output(db: Session, run_id: str) -> Output | None:
151
+ """Retrieve output details of a specific run."""
152
+ return db.query(Output).filter(Output.run_id == run_id).first()
153
+
154
+ def get_latest_successful_run(
155
+ db: Session,
156
+ project_name: str,
157
+ prompt_name: str,
158
+ version_number: int,
159
+ model: str | None = None
160
+ ) -> Run | None:
161
+ """Get the latest successful execution run for a specific version, optionally filtered by model."""
162
+ version = (
163
+ db.query(Version)
164
+ .join(Prompt)
165
+ .join(Project)
166
+ .filter(
167
+ Project.name == project_name,
168
+ Prompt.name == prompt_name,
169
+ Version.version_number == version_number
170
+ )
171
+ .first()
172
+ )
173
+ if not version:
174
+ raise VersionNotFound(
175
+ f"Version {version_number} of prompt '{prompt_name}' in project '{project_name}' not found."
176
+ )
177
+
178
+ query = db.query(Run).filter(Run.version_id == version.id, Run.status == "success")
179
+ if model:
180
+ query = query.filter(Run.model == model)
181
+
182
+ return query.order_by(Run.created_at.desc()).first()
183
+
184
+ def list_distinct_models_for_version(
185
+ db: Session,
186
+ project_name: str,
187
+ prompt_name: str,
188
+ version_number: int
189
+ ) -> list[str]:
190
+ """Get list of distinct models that have successful runs for a specific version."""
191
+ version = (
192
+ db.query(Version)
193
+ .join(Prompt)
194
+ .join(Project)
195
+ .filter(
196
+ Project.name == project_name,
197
+ Prompt.name == prompt_name,
198
+ Version.version_number == version_number
199
+ )
200
+ .first()
201
+ )
202
+ if not version:
203
+ raise VersionNotFound(
204
+ f"Version {version_number} of prompt '{prompt_name}' in project '{project_name}' not found."
205
+ )
206
+
207
+ results = (
208
+ db.query(Run.model)
209
+ .filter(Run.version_id == version.id, Run.status == "success")
210
+ .distinct()
211
+ .all()
212
+ )
213
+ return [r[0] for r in results]
@@ -0,0 +1,113 @@
1
+ from sqlalchemy import func
2
+ from sqlalchemy.orm import Session
3
+ from promptdiff.db.models import Prompt, Project, Version
4
+ from promptdiff.core.exceptions import PromptNotFound
5
+
6
+ def commit_version(
7
+ db: Session,
8
+ project_name: str,
9
+ prompt_name: str,
10
+ content: str,
11
+ message: str | None = None
12
+ ) -> Version:
13
+ """Commit a new version of the prompt content.
14
+
15
+ Correctly auto-increments version_number scoped per-prompt.
16
+ Raises PromptNotFound if the prompt doesn't exist.
17
+ """
18
+ prompt = (
19
+ db.query(Prompt)
20
+ .join(Project)
21
+ .filter(Project.name == project_name, Prompt.name == prompt_name)
22
+ .first()
23
+ )
24
+ if not prompt:
25
+ raise PromptNotFound(f"Prompt '{prompt_name}' in project '{project_name}' not found.")
26
+
27
+ # Auto-increment version number per prompt
28
+ max_version = (
29
+ db.query(func.max(Version.version_number))
30
+ .filter(Version.prompt_id == prompt.id)
31
+ .scalar()
32
+ )
33
+ new_version_number = (max_version or 0) + 1
34
+
35
+ version = Version(
36
+ prompt_id=prompt.id,
37
+ version_number=new_version_number,
38
+ content=content,
39
+ commit_message=message
40
+ )
41
+ db.add(version)
42
+ db.commit()
43
+ db.refresh(version)
44
+ return version
45
+
46
+ def get_version(
47
+ db: Session,
48
+ project_name: str,
49
+ prompt_name: str,
50
+ version_number: int
51
+ ) -> Version | None:
52
+ """Retrieve a specific version of a prompt by its number."""
53
+ return (
54
+ db.query(Version)
55
+ .join(Prompt)
56
+ .join(Project)
57
+ .filter(
58
+ Project.name == project_name,
59
+ Prompt.name == prompt_name,
60
+ Version.version_number == version_number
61
+ )
62
+ .first()
63
+ )
64
+
65
+ def list_versions(
66
+ db: Session,
67
+ project_name: str,
68
+ prompt_name: str
69
+ ) -> list[Version]:
70
+ """List all versions of a prompt, ordered by version_number ascending.
71
+
72
+ Raises PromptNotFound if the prompt doesn't exist.
73
+ """
74
+ prompt = (
75
+ db.query(Prompt)
76
+ .join(Project)
77
+ .filter(Project.name == project_name, Prompt.name == prompt_name)
78
+ .first()
79
+ )
80
+ if not prompt:
81
+ raise PromptNotFound(f"Prompt '{prompt_name}' in project '{project_name}' not found.")
82
+
83
+ return (
84
+ db.query(Version)
85
+ .filter(Version.prompt_id == prompt.id)
86
+ .order_by(Version.version_number.asc())
87
+ .all()
88
+ )
89
+
90
+ def get_latest_version(
91
+ db: Session,
92
+ project_name: str,
93
+ prompt_name: str
94
+ ) -> Version | None:
95
+ """Retrieve the most recent version of a prompt.
96
+
97
+ Raises PromptNotFound if the prompt doesn't exist.
98
+ """
99
+ prompt = (
100
+ db.query(Prompt)
101
+ .join(Project)
102
+ .filter(Project.name == project_name, Prompt.name == prompt_name)
103
+ .first()
104
+ )
105
+ if not prompt:
106
+ raise PromptNotFound(f"Prompt '{prompt_name}' in project '{project_name}' not found.")
107
+
108
+ return (
109
+ db.query(Version)
110
+ .filter(Version.prompt_id == prompt.id)
111
+ .order_by(Version.version_number.desc())
112
+ .first()
113
+ )
@@ -0,0 +1 @@
1
+ # DB package
promptdiff/db/base.py ADDED
@@ -0,0 +1,4 @@
1
+ from sqlalchemy.orm import DeclarativeBase
2
+
3
+ class Base(DeclarativeBase):
4
+ pass
@@ -0,0 +1,105 @@
1
+ import uuid
2
+ from datetime import datetime, timezone
3
+ from typing import List, Optional
4
+ from sqlalchemy import ForeignKey, String, Text, DateTime, UniqueConstraint, Uuid
5
+ from sqlalchemy.orm import Mapped, mapped_column, relationship
6
+
7
+ from promptdiff.db.base import Base
8
+
9
+ def get_utc_now() -> datetime:
10
+ return datetime.now(timezone.utc)
11
+
12
+ class Project(Base):
13
+ __tablename__ = "projects"
14
+
15
+ id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4)
16
+ name: Mapped[str] = mapped_column(String, unique=True, nullable=False)
17
+ description: Mapped[Optional[str]] = mapped_column(String, nullable=True)
18
+ created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=get_utc_now)
19
+
20
+ # Relationship to Prompt
21
+ prompts: Mapped[List["Prompt"]] = relationship(
22
+ back_populates="project",
23
+ cascade="all, delete-orphan"
24
+ )
25
+
26
+ class Prompt(Base):
27
+ __tablename__ = "prompts"
28
+
29
+ id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4)
30
+ project_id: Mapped[uuid.UUID] = mapped_column(
31
+ Uuid, ForeignKey("projects.id", ondelete="CASCADE"), nullable=False
32
+ )
33
+ name: Mapped[str] = mapped_column(String, nullable=False)
34
+ created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=get_utc_now)
35
+
36
+ # Relationships
37
+ project: Mapped["Project"] = relationship(back_populates="prompts")
38
+ versions: Mapped[List["Version"]] = relationship(
39
+ back_populates="prompt",
40
+ cascade="all, delete-orphan"
41
+ )
42
+
43
+ __table_args__ = (
44
+ UniqueConstraint("project_id", "name", name="uq_prompt_project_name"),
45
+ )
46
+
47
+ class Version(Base):
48
+ __tablename__ = "versions"
49
+
50
+ id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4)
51
+ prompt_id: Mapped[uuid.UUID] = mapped_column(
52
+ Uuid, ForeignKey("prompts.id", ondelete="CASCADE"), nullable=False
53
+ )
54
+ version_number: Mapped[int] = mapped_column(nullable=False)
55
+ content: Mapped[str] = mapped_column(Text, nullable=False)
56
+ commit_message: Mapped[Optional[str]] = mapped_column(String, nullable=True)
57
+ created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=get_utc_now)
58
+
59
+ # Relationships
60
+ prompt: Mapped["Prompt"] = relationship(back_populates="versions")
61
+ runs: Mapped[List["Run"]] = relationship(
62
+ back_populates="version",
63
+ cascade="all, delete-orphan"
64
+ )
65
+
66
+ __table_args__ = (
67
+ UniqueConstraint("prompt_id", "version_number", name="uq_version_prompt_number"),
68
+ )
69
+
70
+ class Run(Base):
71
+ __tablename__ = "runs"
72
+
73
+ id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4)
74
+ version_id: Mapped[uuid.UUID] = mapped_column(
75
+ Uuid, ForeignKey("versions.id", ondelete="CASCADE"), nullable=False
76
+ )
77
+ model: Mapped[str] = mapped_column(String, nullable=False)
78
+ status: Mapped[str] = mapped_column(String, nullable=False) # pending, success, failed
79
+ error_message: Mapped[Optional[str]] = mapped_column(String, nullable=True)
80
+ latency_ms: Mapped[Optional[int]] = mapped_column(nullable=True)
81
+ created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=get_utc_now)
82
+
83
+ # Relationships
84
+ version: Mapped["Version"] = relationship(back_populates="runs")
85
+ output: Mapped[Optional["Output"]] = relationship(
86
+ back_populates="run",
87
+ cascade="all, delete-orphan",
88
+ uselist=False
89
+ )
90
+
91
+ class Output(Base):
92
+ __tablename__ = "outputs"
93
+
94
+ id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4)
95
+ run_id: Mapped[uuid.UUID] = mapped_column(
96
+ Uuid, ForeignKey("runs.id", ondelete="CASCADE"), unique=True, nullable=False
97
+ )
98
+ content: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
99
+ prompt_tokens: Mapped[Optional[int]] = mapped_column(nullable=True)
100
+ completion_tokens: Mapped[Optional[int]] = mapped_column(nullable=True)
101
+ cost_usd: Mapped[Optional[float]] = mapped_column(nullable=True)
102
+ created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=get_utc_now)
103
+
104
+ # Relationships
105
+ run: Mapped["Run"] = relationship(back_populates="output")
@@ -0,0 +1,24 @@
1
+ import os
2
+ from contextlib import contextmanager
3
+ from typing import Generator
4
+ from sqlalchemy import create_engine
5
+ from sqlalchemy.orm import sessionmaker, Session
6
+
7
+ # Resolve absolute path to project root to keep DB file location consistent
8
+ BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
9
+ DATABASE_URL = f"sqlite:///{os.path.join(BASE_DIR, 'promptdiff.db')}"
10
+
11
+ engine = create_engine(
12
+ DATABASE_URL,
13
+ connect_args={"check_same_thread": False}
14
+ )
15
+
16
+ SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
17
+
18
+ @contextmanager
19
+ def get_db() -> Generator[Session, None, None]:
20
+ db = SessionLocal()
21
+ try:
22
+ yield db
23
+ finally:
24
+ db.close()
@@ -0,0 +1,166 @@
1
+ Metadata-Version: 2.4
2
+ Name: promptdiff-cli
3
+ Version: 0.1.0
4
+ Summary: Git for prompts - version, diff, and test your LLM prompts across any model
5
+ Author: Raguram R
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/Ragu3175/promptdiff
8
+ Project-URL: Repository, https://github.com/Ragu3175/promptdiff
9
+ Project-URL: Issues, https://github.com/Ragu3175/promptdiff/issues
10
+ Keywords: llm,prompt-engineering,prompt-versioning,cli,litellm
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Topic :: Software Development :: Version Control
17
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
18
+ Requires-Python: >=3.11
19
+ Description-Content-Type: text/markdown
20
+ License-File: LICENSE
21
+ Requires-Dist: fastapi>=0.110.0
22
+ Requires-Dist: uvicorn>=0.28.0
23
+ Requires-Dist: sqlalchemy>=2.0.0
24
+ Requires-Dist: alembic>=1.13.0
25
+ Requires-Dist: typer>=0.9.0
26
+ Requires-Dist: rich>=13.7.0
27
+ Requires-Dist: litellm>=1.30.0
28
+ Requires-Dist: python-dotenv>=1.0.0
29
+ Provides-Extra: dev
30
+ Requires-Dist: pytest>=8.0.0; extra == "dev"
31
+ Requires-Dist: pytest-mock; extra == "dev"
32
+ Dynamic: license-file
33
+
34
+ # PromptDiff
35
+
36
+ **Git for prompts.** Version your LLM prompts, diff them word-by-word, and run any version against Gemini, Groq, or any other LiteLLM-supported model — all from the terminal.
37
+
38
+ ```bash
39
+ promptdiff commit summarizer "Summarize: {text}" -m "v1: baseline"
40
+ promptdiff commit summarizer "Summarize concisely in one sentence: {text}" -m "v2: tighter constraint"
41
+ promptdiff diff summarizer 1 2
42
+ promptdiff run summarizer 2 --model gemini/gemini-2.5-flash
43
+ ```
44
+
45
+ ## Why PromptDiff
46
+
47
+ If you've ever overwritten a prompt and lost the version that actually worked, or pasted two prompt drafts into a doc just to eyeball what changed — PromptDiff is the tool for that. It treats prompts as versioned, diffable artifacts the same way Git treats code, and adds one thing Git can't: running any version against a real LLM and recording exactly what it produced, so you can trace output back to the exact prompt and model that generated it.
48
+
49
+ ## How it's different from other prompt tools
50
+
51
+ The prompt-tooling space has grown a lot — [Promptfoo](https://github.com/promptfoo/promptfoo) is a mature, YAML-config-based open-source tool with strong CI/eval integration, and platforms like PromptLayer, Langfuse, and Braintrust offer hosted, full-lifecycle prompt management with branching, RBAC, and observability.
52
+
53
+ PromptDiff isn't trying to compete with those. It's deliberately smaller: a local-first CLI for a single developer iterating on prompts, with zero config files, zero hosted accounts, and zero cost beyond free-tier LLM API usage. If you outgrow it — multiple collaborators, non-engineer prompt editors, compliance requirements — those other tools are the right next step. If you're a solo developer who just wants `git commit`-style version control for prompts without standing up infrastructure, PromptDiff is built for exactly that.
54
+
55
+ ## Features
56
+
57
+ - **Version control for prompts** — every `commit` is an immutable, numbered version (v1, v2, v3...) scoped per prompt
58
+ - **Word-level diffs** — see exactly which words changed between versions, rendered with color in your terminal (green additions, red strikethrough removals)
59
+ - **Multi-model runs** — execute any prompt version against multiple LLMs in a single command and compare cost, latency, and output side by side
60
+ - **Output comparison** — `diff-output` shows what two versions actually *produced*, not just what their text looks like
61
+ - **Free-tier friendly** — built and tested against Gemini and Groq's free tiers; rate-limit and quota failures are recorded gracefully, never crash the tool
62
+ - **Local-first** — everything lives in a local SQLite database, no account or server required
63
+
64
+ ## Installation
65
+
66
+ ```bash
67
+ pip install promptdiff-cli
68
+ ```
69
+
70
+ (Or, to run from source — see [Development](#development) below.)
71
+
72
+ ## Quick start
73
+
74
+ ```bash
75
+ # Create a project and switch to it
76
+ promptdiff project create my-app
77
+ promptdiff use my-app
78
+
79
+ # Create a prompt and commit your first version
80
+ promptdiff add summarizer
81
+ promptdiff commit summarizer "Summarize this in one sentence: {text}" -m "v1: baseline"
82
+
83
+ # Edit and commit a new version
84
+ promptdiff commit summarizer "Summarize concisely, max 20 words: {text}" -m "v2: tighter constraint"
85
+
86
+ # See version history
87
+ promptdiff log summarizer
88
+
89
+ # See exactly what changed
90
+ promptdiff diff summarizer 1 2
91
+
92
+ # Run a version against a model (needs an API key — see below)
93
+ promptdiff run summarizer 2 --model gemini/gemini-2.5-flash
94
+
95
+ # Run the same version against multiple models at once
96
+ promptdiff run summarizer 2 --model gemini/gemini-2.5-flash --model groq/llama-3.3-70b-versatile
97
+
98
+ # Compare what two versions actually produced
99
+ promptdiff diff-output summarizer 1 2 --model gemini/gemini-2.5-flash
100
+ ```
101
+
102
+ ## API keys
103
+
104
+ PromptDiff uses [LiteLLM](https://github.com/BerriAI/litellm) under the hood, so it works with any LiteLLM-supported provider. It's built and tested primarily against free tiers:
105
+
106
+ - **Gemini** — get a free key at [aistudio.google.com/apikey](https://aistudio.google.com/apikey)
107
+ - **Groq** — get a free key at [console.groq.com/keys](https://console.groq.com/keys)
108
+
109
+ Copy `.env.example` to `.env` in your project directory and fill in the keys you have:
110
+
111
+ ```
112
+ GEMINI_API_KEY=your_key_here
113
+ GROQ_API_KEY=your_key_here
114
+ ```
115
+
116
+ ## Commands
117
+
118
+ | Command | What it does |
119
+ |---|---|
120
+ | `promptdiff project create <name>` | Create a new project |
121
+ | `promptdiff project list` | List all projects |
122
+ | `promptdiff use <project>` | Set the current project (so you don't need `--project` on every command) |
123
+ | `promptdiff add <prompt>` | Add a new prompt to the current project |
124
+ | `promptdiff commit <prompt> <content_or_file> -m "<message>"` | Commit a new version. Accepts a literal string or a path to a file |
125
+ | `promptdiff log <prompt>` | Show version history |
126
+ | `promptdiff diff <prompt> <v1> <v2>` | Word-level diff between two versions |
127
+ | `promptdiff run <prompt> <version> --model <model>` | Run a version against one or more models (repeat `--model` to run against several at once) |
128
+ | `promptdiff diff-output <prompt> <v1> <v2> [--model <model>]` | Compare what two versions actually produced. If a version was run against multiple models, `--model` is required to avoid an ambiguous comparison |
129
+
130
+ Every command accepts `-p` / `--project` to override the current project for that one call.
131
+
132
+ ## Architecture
133
+
134
+ - **Database**: SQLite, 5 tables (`projects`, `prompts`, `versions`, `runs`, `outputs`) via SQLAlchemy + Alembic migrations
135
+ - **Diff engine**: Python's `difflib`, line-level structure with word-level precision inside changed lines
136
+ - **Runner**: [LiteLLM](https://github.com/BerriAI/litellm) for provider-agnostic model calls; failures (rate limits, timeouts, bad keys) are recorded as failed runs rather than crashing the CLI
137
+ - **CLI**: [Typer](https://typer.tiangolo.com/) + [Rich](https://github.com/Textualize/rich)
138
+
139
+ ## Development
140
+
141
+ ```bash
142
+ git clone https://github.com/Ragu3175/promptdiff.git
143
+ cd promptdiff
144
+ python -m venv venv
145
+ venv\Scripts\activate # Windows
146
+ # source venv/bin/activate # macOS/Linux
147
+ pip install -r requirements.txt
148
+ pip install -e .
149
+ pytest -v
150
+ ```
151
+
152
+ ## Status
153
+
154
+ Core CLI is complete and tested: `commit`, `log`, `diff`, `run`, and `diff-output` all work end-to-end against real Gemini and Groq calls. A REST API layer and web UI are planned but not yet built — the CLI is fully usable on its own today.
155
+
156
+ ## Contributing
157
+
158
+ Issues and PRs welcome. This is an early-stage solo project — if you run into something broken or have an idea, please open an issue.
159
+
160
+ ## License
161
+
162
+ MIT — see [LICENSE](LICENSE).
163
+
164
+ ## Author
165
+
166
+ Built by [Raguram R](https://github.com/Ragu3175).
@@ -0,0 +1,22 @@
1
+ promptdiff/__init__.py,sha256=-ElmcZBCJkYwpgk1LAVKt8XFCpUlV3f2Jf69tV9QiZQ,21
2
+ promptdiff/api/__init__.py,sha256=-HTwAJ8O401KTSPm8ZYja2d0RRn6oj1_SOFX75w-mT0,14
3
+ promptdiff/cli/__init__.py,sha256=23Mz_sVEtHKRAxopQaFKt6FYDaYj4ZG3S9ag1JIlIbA,14
4
+ promptdiff/cli/context.py,sha256=2ROnd8QOpXWUNrlG1-zApjHhO2A45AZ6tE4c7pk9-hc,1534
5
+ promptdiff/cli/main.py,sha256=0j7UhSj-janrZ0X4QQWU4UMHrZ20VEdrfrFvhQ1eK7k,15535
6
+ promptdiff/core/__init__.py,sha256=TETStgToTe7QSsCZgRHDk2oSErlLJoeGN0sFg4Yx2_c,15
7
+ promptdiff/core/diff.py,sha256=8i-wQI4T2hp4x3hP7jfRojR7HCvEz0LIkgsDjqatGhI,4566
8
+ promptdiff/core/exceptions.py,sha256=R2HaS5BgEEadEbSoRg0U7eTD9EYOxg8cdJVAt1WnEHQ,666
9
+ promptdiff/core/projects.py,sha256=glWvok10qPD6jqHOW3_3IvquA-34pPw-J8yYFxoWIXM,975
10
+ promptdiff/core/prompts.py,sha256=KNhAn1Og7yZshzpInkGDCJ4vsTjfjzDnaXwm3BKhsKw,1691
11
+ promptdiff/core/runner.py,sha256=YlHu5pe6gO0EFwIe0C5Ex0SO7vS9XmVZywclULwdH3k,6327
12
+ promptdiff/core/versions.py,sha256=i_L4oCQtcnYzBvMk1Ks4YlLhHYT8FXKoYw82GC-4M20,3066
13
+ promptdiff/db/__init__.py,sha256=k1h6E6ObbcwbVkENlopOZaprvznmy7zTFRjFmddK7Hk,13
14
+ promptdiff/db/base.py,sha256=aCH8LR0m4jZ-wIKkAf1qXAaWt4lckE0hmqWSiFiB7a0,82
15
+ promptdiff/db/models.py,sha256=W5JEYwAt24rPg5PWvlLlMRBswctAzMMCehOf9bS21Vo,4203
16
+ promptdiff/db/session.py,sha256=yRFiET_2lF7Zl1sViCSH8U5lEneyLgQLcDEhs5xeEVs,713
17
+ promptdiff_cli-0.1.0.dist-info/licenses/LICENSE,sha256=iEDhVNZPUT5wbGJybB3oMUeKO_H5cRukGFTLWvZcnL4,1066
18
+ promptdiff_cli-0.1.0.dist-info/METADATA,sha256=Hco7p37LFPKnG7MgIA0UxMQF47WKwbn9zJXulPWu0K8,8050
19
+ promptdiff_cli-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
20
+ promptdiff_cli-0.1.0.dist-info/entry_points.txt,sha256=dOnK6W9jqJAFXWkFO6nf0oThPxVDVnViSaBWMZ-5JjM,55
21
+ promptdiff_cli-0.1.0.dist-info/top_level.txt,sha256=o21R8uUq_ASJoZ2e4vALVjGHzLsBznMuj9YgR7mJeLU,11
22
+ promptdiff_cli-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (82.0.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ promptdiff = promptdiff.cli.main:app
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Raguram R
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ promptdiff