promptdiff-ai 1.0.0__py3-none-any.whl

promptdiff/__init__.py

@@ -0,0 +1,52 @@
+"""promptdiff: Semantic diff for LLM prompt changes.
+
+Public API exports for convenience imports::
+
+    from promptdiff import diff_files, PromptDiff, format_text
+"""
+
+from promptdiff.models import (
+    BreakingChange,
+    ChangeStatus,
+    MessageDiff,
+    MetadataDiff,
+    PromptDiff,
+    TokenDelta,
+    VariableDiff,
+)
+from promptdiff.differ import (
+    compute_token_delta,
+    diff_files,
+    diff_messages,
+    diff_metadata,
+    diff_variables,
+)
+from promptdiff.analyzer import analyze_breaking_changes
+from promptdiff.reporter import format_json, format_markdown, format_text
+
+__version__ = "1.0.0"
+
+__all__ = [
+    # Models
+    "BreakingChange",
+    "ChangeStatus",
+    "MessageDiff",
+    "MetadataDiff",
+    "PromptDiff",
+    "TokenDelta",
+    "VariableDiff",
+    # Differ
+    "compute_token_delta",
+    "diff_files",
+    "diff_messages",
+    "diff_metadata",
+    "diff_variables",
+    # Analyzer
+    "analyze_breaking_changes",
+    # Reporter
+    "format_json",
+    "format_markdown",
+    "format_text",
+    # Version
+    "__version__",
+]

promptdiff/analyzer.py

@@ -0,0 +1,163 @@
+"""Breaking change detection for promptdiff.
+
+Analyzes a PromptDiff and classifies changes by severity.
+
+Breaking changes:
+  - New required variable (no default) -- high severity
+  - Removed variable -- high severity
+  - Removed message -- high severity
+  - Changed role ordering -- medium severity
+  - Model change (in metadata) -- medium severity
+
+Non-breaking changes:
+  - Added variable with default
+  - Added messages
+  - Content modifications
+  - Metadata changes (except model)
+"""
+
+from __future__ import annotations
+
+from promptdiff.models import (
+    BreakingChange,
+    ChangeStatus,
+    PromptDiff,
+)
+
+
+def analyze_breaking_changes(diff: PromptDiff) -> list[BreakingChange]:
+    """Analyze a diff for breaking changes.
+
+    Parameters
+    ----------
+    diff:
+        The prompt diff to analyze.
+
+    Returns
+    -------
+    list[BreakingChange]
+        All detected breaking changes sorted by severity.
+    """
+    changes: list[BreakingChange] = []
+    changes.extend(_check_variable_changes(diff))
+    changes.extend(_check_message_changes(diff))
+    changes.extend(_check_metadata_changes(diff))
+    changes.extend(_check_role_ordering(diff))
+
+    # Sort by severity: high first, then medium, then low.
+    severity_order = {"high": 0, "medium": 1, "low": 2}
+    changes.sort(key=lambda c: severity_order.get(c.severity, 3))
+
+    return changes
+
+
+def _check_variable_changes(diff: PromptDiff) -> list[BreakingChange]:
+    """Detect breaking variable changes."""
+    changes: list[BreakingChange] = []
+
+    for vd in diff.variable_diffs:
+        if vd.status == ChangeStatus.REMOVED:
+            changes.append(
+                BreakingChange(
+                    category="variable",
+                    description=f"Variable '{vd.name}' was removed",
+                    severity="high",
+                )
+            )
+        elif vd.status == ChangeStatus.ADDED and vd.is_breaking:
+            changes.append(
+                BreakingChange(
+                    category="variable",
+                    description=(
+                        f"New required variable '{vd.name}' added without a default value"
+                    ),
+                    severity="high",
+                )
+            )
+
+    return changes
+
+
+def _check_message_changes(diff: PromptDiff) -> list[BreakingChange]:
+    """Detect breaking message changes."""
+    changes: list[BreakingChange] = []
+
+    for md in diff.message_diffs:
+        if md.status == ChangeStatus.REMOVED:
+            changes.append(
+                BreakingChange(
+                    category="message",
+                    description=f"{md.role.capitalize()} message was removed",
+                    severity="high",
+                )
+            )
+
+    return changes
+
+
+def _check_metadata_changes(diff: PromptDiff) -> list[BreakingChange]:
+    """Detect breaking metadata changes (model changes)."""
+    changes: list[BreakingChange] = []
+
+    for md in diff.metadata_diffs:
+        if md.key == "model" and md.status == ChangeStatus.MODIFIED:
+            changes.append(
+                BreakingChange(
+                    category="model",
+                    description=(
+                        f"Model changed from '{md.old_value}' to '{md.new_value}'"
+                    ),
+                    severity="medium",
+                )
+            )
+        elif md.key == "model" and md.status == ChangeStatus.REMOVED:
+            changes.append(
+                BreakingChange(
+                    category="model",
+                    description="Model specification was removed",
+                    severity="medium",
+                )
+            )
+
+    return changes
+
+
+def _check_role_ordering(diff: PromptDiff) -> list[BreakingChange]:
+    """Detect role ordering changes.
+
+    If the sequence of roles (ignoring unchanged content) changes between
+    old and new, this is potentially breaking.
+    """
+    changes: list[BreakingChange] = []
+
+    # Reconstruct old and new role sequences from the message diffs.
+    old_roles: list[str] = []
+    new_roles: list[str] = []
+
+    for md in diff.message_diffs:
+        if md.status == ChangeStatus.REMOVED:
+            old_roles.append(md.role)
+        elif md.status == ChangeStatus.ADDED:
+            new_roles.append(md.role)
+        elif md.status in (ChangeStatus.MODIFIED, ChangeStatus.UNCHANGED):
+            old_roles.append(md.role)
+            new_roles.append(md.role)
+
+    if old_roles != new_roles and len(old_roles) > 0 and len(new_roles) > 0:
+        # Only flag if neither list is a subset scenario (pure additions/removals
+        # are already caught above). Check if the common elements changed order.
+        common_old = [r for r in old_roles if r in new_roles]
+        common_new = [r for r in new_roles if r in old_roles]
+        if common_old != common_new:
+            changes.append(
+                BreakingChange(
+                    category="role",
+                    description=(
+                        f"Message role ordering changed: "
+                        f"{' -> '.join(old_roles)} to {' -> '.join(new_roles)}"
+                    ),
+                    severity="medium",
+                )
+            )
+
+    return changes

promptdiff/cli.py

@@ -0,0 +1,123 @@
+"""Typer CLI application for promptdiff.
+
+Provides the ``promptdiff`` entry point for comparing two prompt files
+and reporting differences in text, JSON, or Markdown format.
+"""
+
+from __future__ import annotations
+
+import sys
+from enum import Enum
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+
+from promptdiff.differ import diff_files
+from promptdiff.reporter import format_json, format_markdown, format_text
+
+# ---------------------------------------------------------------------------
+# Console
+# ---------------------------------------------------------------------------
+
+_console = Console()
+_err_console = Console(stderr=True)
+
+
+class OutputFormat(str, Enum):
+    """Supported output formats."""
+
+    TEXT = "text"
+    JSON = "json"
+    MARKDOWN = "markdown"
+
+
+# ---------------------------------------------------------------------------
+# Version callback
+# ---------------------------------------------------------------------------
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        from promptdiff import __version__
+
+        _console.print(f"promptdiff {__version__}")
+        raise typer.Exit()
+
+
+# ---------------------------------------------------------------------------
+# Main command — single-command app (no subcommands needed)
+# ---------------------------------------------------------------------------
+
+
+def main(
+    file_a: Path = typer.Argument(..., help="Path to the old prompt file"),
+    file_b: Path = typer.Argument(..., help="Path to the new prompt file"),
+    output_format: OutputFormat = typer.Option(
+        OutputFormat.TEXT,
+        "--format",
+        "-f",
+        help="Output format: text, json, markdown",
+    ),
+    exit_on_breaking: bool = typer.Option(
+        False,
+        "--exit-on-breaking",
+        help="Exit with code 1 if breaking changes are found",
+    ),
+    token_detail: bool = typer.Option(
+        False,
+        "--token-detail",
+        help="Show per-message token breakdowns",
+    ),
+    encoding: str = typer.Option(
+        "cl100k_base",
+        "--encoding",
+        "-e",
+        help="tiktoken encoding for token counting",
+    ),
+    version: bool = typer.Option(
+        False,
+        "--version",
+        "-V",
+        callback=_version_callback,
+        is_eager=True,
+        help="Show version and exit",
+    ),
+) -> None:
+    """Semantic diff for LLM prompt changes.
+
+    Compare two prompt files and show structured diff with message-level
+    changes, variable changes, token deltas, and breaking change classification.
+    """
+    try:
+        result = diff_files(file_a, file_b, encoding=encoding)
+    except FileNotFoundError as exc:
+        _err_console.print(f"[red]Error:[/red] {exc}")
+        raise typer.Exit(2)
+    except Exception as exc:
+        _err_console.print(f"[red]Error:[/red] {exc}")
+        raise typer.Exit(2)
+
+    if output_format == OutputFormat.TEXT:
+        output = format_text(result, show_token_detail=token_detail)
+        _console.print(output)
+    elif output_format == OutputFormat.JSON:
+        output = format_json(result)
+        # Use print() to avoid Rich adding ANSI codes to raw JSON.
+        print(output)
+    elif output_format == OutputFormat.MARKDOWN:
+        output = format_markdown(result)
+        print(output)
+
+    if exit_on_breaking and result.is_breaking:
+        raise typer.Exit(1)
+
+
+app = typer.Typer(
+    name="promptdiff",
+    help="Semantic diff for LLM prompt files.",
+    add_completion=False,
+    no_args_is_help=True,
+)
+app.command()(main)

promptdiff/differ.py

@@ -0,0 +1,393 @@
+"""Core diff engine for promptdiff.
+
+Compares two prompt files parsed via prompttools_core and produces a
+structured PromptDiff with message-level, variable-level, and metadata-level
+change information.
+"""
+
+from __future__ import annotations
+
+import difflib
+from pathlib import Path
+from typing import Any, Optional
+
+from prompttools_core import PromptFile, Tokenizer, count_tokens, parse_file
+from prompttools_core.models import Message
+
+from promptdiff.analyzer import analyze_breaking_changes
+from promptdiff.models import (
+    ChangeStatus,
+    MessageDiff,
+    MetadataDiff,
+    PromptDiff,
+    TokenDelta,
+    VariableDiff,
+)
+
+# Default encoding used for token counting when no model is specified.
+_DEFAULT_ENCODING = "cl100k_base"
+
+
+def diff_files(
+    old_path: Path,
+    new_path: Path,
+    encoding: str = _DEFAULT_ENCODING,
+) -> PromptDiff:
+    """Parse and diff two prompt files.
+
+    Parameters
+    ----------
+    old_path:
+        Path to the old prompt file.
+    new_path:
+        Path to the new prompt file.
+    encoding:
+        tiktoken encoding name for token counting.
+
+    Returns
+    -------
+    PromptDiff
+        A structured diff between the two files.
+    """
+    old_file = parse_file(Path(old_path))
+    new_file = parse_file(Path(new_path))
+
+    msg_diffs = diff_messages(old_file.messages, new_file.messages, encoding)
+    var_diffs = diff_variables(
+        old_file.variables,
+        new_file.variables,
+        old_file.variable_defaults,
+        new_file.variable_defaults,
+    )
+    meta_diffs = diff_metadata(old_file.metadata, new_file.metadata)
+    token_d = compute_token_delta(old_file, new_file, encoding)
+
+    result = PromptDiff(
+        file_path=new_path,
+        old_hash=old_file.content_hash,
+        new_hash=new_file.content_hash,
+        message_diffs=msg_diffs,
+        variable_diffs=var_diffs,
+        metadata_diffs=meta_diffs,
+        token_delta=token_d,
+        breaking_changes=[],
+    )
+
+    # Run breaking change analysis and attach results.
+    result.breaking_changes = analyze_breaking_changes(result)
+
+    return result
+
+
+def diff_messages(
+    old_msgs: list[Message],
+    new_msgs: list[Message],
+    encoding: str = _DEFAULT_ENCODING,
+) -> list[MessageDiff]:
+    """Align and diff two ordered lists of messages.
+
+    Alignment is done by (role, position-within-role). Messages are matched
+    by role in the order they appear -- the first system message in old is
+    compared with the first system message in new, etc.
+
+    Parameters
+    ----------
+    old_msgs:
+        Messages from the old prompt file.
+    new_msgs:
+        Messages from the new prompt file.
+    encoding:
+        tiktoken encoding name for token counting.
+
+    Returns
+    -------
+    list[MessageDiff]
+        Ordered list of message diffs.
+    """
+    # Group messages by role, preserving order within each role.
+    old_by_role: dict[str, list[Message]] = {}
+    new_by_role: dict[str, list[Message]] = {}
+
+    for msg in old_msgs:
+        old_by_role.setdefault(msg.role, []).append(msg)
+    for msg in new_msgs:
+        new_by_role.setdefault(msg.role, []).append(msg)
+
+    all_roles_ordered: list[str] = []
+    seen: set[str] = set()
+    for msg in old_msgs:
+        if msg.role not in seen:
+            all_roles_ordered.append(msg.role)
+            seen.add(msg.role)
+    for msg in new_msgs:
+        if msg.role not in seen:
+            all_roles_ordered.append(msg.role)
+            seen.add(msg.role)
+
+    diffs: list[MessageDiff] = []
+
+    for role in all_roles_ordered:
+        old_list = old_by_role.get(role, [])
+        new_list = new_by_role.get(role, [])
+
+        max_len = max(len(old_list), len(new_list))
+        for i in range(max_len):
+            old_msg = old_list[i] if i < len(old_list) else None
+            new_msg = new_list[i] if i < len(new_list) else None
+            diffs.append(_diff_single_message(old_msg, new_msg, role, encoding))
+
+    return diffs
+
+
+def _diff_single_message(
+    old_msg: Optional[Message],
+    new_msg: Optional[Message],
+    role: str,
+    encoding: str,
+) -> MessageDiff:
+    """Diff a single aligned pair of messages."""
+    if old_msg is None and new_msg is not None:
+        # Added
+        new_tokens = count_tokens(new_msg.content, encoding)
+        return MessageDiff(
+            status=ChangeStatus.ADDED,
+            role=role,
+            new_content=new_msg.content,
+            token_delta=new_tokens,
+            changes=[f"Added {role} message ({new_tokens} tokens)"],
+        )
+
+    if old_msg is not None and new_msg is None:
+        # Removed
+        old_tokens = count_tokens(old_msg.content, encoding)
+        return MessageDiff(
+            status=ChangeStatus.REMOVED,
+            role=role,
+            old_content=old_msg.content,
+            token_delta=-old_tokens,
+            changes=[f"Removed {role} message ({old_tokens} tokens)"],
+        )
+
+    # Both exist -- compare content.
+    assert old_msg is not None and new_msg is not None
+
+    if old_msg.content == new_msg.content:
+        return MessageDiff(
+            status=ChangeStatus.UNCHANGED,
+            role=role,
+            old_content=old_msg.content,
+            new_content=new_msg.content,
+            token_delta=0,
+        )
+
+    # Modified
+    old_tokens = count_tokens(old_msg.content, encoding)
+    new_tokens = count_tokens(new_msg.content, encoding)
+    delta = new_tokens - old_tokens
+
+    content_diff = "\n".join(
+        difflib.unified_diff(
+            old_msg.content.splitlines(),
+            new_msg.content.splitlines(),
+            lineterm="",
+            fromfile="old",
+            tofile="new",
+        )
+    )
+
+    changes: list[str] = []
+    if delta != 0:
+        direction = "increased" if delta > 0 else "decreased"
+        changes.append(
+            f"{role.capitalize()} message {direction} by {abs(delta)} tokens"
+        )
+    changes.append(f"{role.capitalize()} message content modified")
+
+    return MessageDiff(
+        status=ChangeStatus.MODIFIED,
+        role=role,
+        old_content=old_msg.content,
+        new_content=new_msg.content,
+        content_diff=content_diff,
+        token_delta=delta,
+        changes=changes,
+    )
+
+
+def diff_variables(
+    old_vars: dict[str, str],
+    new_vars: dict[str, str],
+    old_defaults: dict[str, str],
+    new_defaults: dict[str, str],
+) -> list[VariableDiff]:
+    """Compare variable sets between old and new prompt versions.
+
+    Parameters
+    ----------
+    old_vars:
+        Variables found in the old version (name -> syntax style).
+    new_vars:
+        Variables found in the new version (name -> syntax style).
+    old_defaults:
+        Default values from the old version metadata.
+    new_defaults:
+        Default values from the new version metadata.
+
+    Returns
+    -------
+    list[VariableDiff]
+        List of variable diffs.
+    """
+    all_names = sorted(set(old_vars) | set(new_vars))
+    diffs: list[VariableDiff] = []
+
+    for name in all_names:
+        in_old = name in old_vars
+        in_new = name in new_vars
+
+        if in_old and not in_new:
+            diffs.append(
+                VariableDiff(
+                    name=name,
+                    status=ChangeStatus.REMOVED,
+                    old_default=old_defaults.get(name),
+                    is_breaking=True,
+                )
+            )
+        elif not in_old and in_new:
+            new_default = new_defaults.get(name)
+            # Breaking if no default is provided for the new variable.
+            diffs.append(
+                VariableDiff(
+                    name=name,
+                    status=ChangeStatus.ADDED,
+                    new_default=new_default,
+                    is_breaking=new_default is None,
+                )
+            )
+        else:
+            # Present in both -- check if default changed.
+            old_def = old_defaults.get(name)
+            new_def = new_defaults.get(name)
+            if old_def == new_def:
+                diffs.append(
+                    VariableDiff(
+                        name=name,
+                        status=ChangeStatus.UNCHANGED,
+                        old_default=old_def,
+                        new_default=new_def,
+                    )
+                )
+            else:
+                diffs.append(
+                    VariableDiff(
+                        name=name,
+                        status=ChangeStatus.MODIFIED,
+                        old_default=old_def,
+                        new_default=new_def,
+                    )
+                )
+
+    return diffs
+
+
+def diff_metadata(
+    old_meta: dict[str, Any],
+    new_meta: dict[str, Any],
+) -> list[MetadataDiff]:
+    """Compare metadata dictionaries.
+
+    Parameters
+    ----------
+    old_meta:
+        Metadata from the old prompt file.
+    new_meta:
+        Metadata from the new prompt file.
+
+    Returns
+    -------
+    list[MetadataDiff]
+        List of metadata diffs.
+    """
+    all_keys = sorted(set(old_meta) | set(new_meta))
+    diffs: list[MetadataDiff] = []
+
+    for key in all_keys:
+        in_old = key in old_meta
+        in_new = key in new_meta
+
+        if in_old and not in_new:
+            diffs.append(
+                MetadataDiff(
+                    key=key,
+                    status=ChangeStatus.REMOVED,
+                    old_value=old_meta[key],
+                )
+            )
+        elif not in_old and in_new:
+            diffs.append(
+                MetadataDiff(
+                    key=key,
+                    status=ChangeStatus.ADDED,
+                    new_value=new_meta[key],
+                )
+            )
+        elif old_meta[key] == new_meta[key]:
+            diffs.append(
+                MetadataDiff(
+                    key=key,
+                    status=ChangeStatus.UNCHANGED,
+                    old_value=old_meta[key],
+                    new_value=new_meta[key],
+                )
+            )
+        else:
+            diffs.append(
+                MetadataDiff(
+                    key=key,
+                    status=ChangeStatus.MODIFIED,
+                    old_value=old_meta[key],
+                    new_value=new_meta[key],
+                )
+            )
+
+    return diffs
+
+
+def compute_token_delta(
+    old_file: PromptFile,
+    new_file: PromptFile,
+    encoding: str = _DEFAULT_ENCODING,
+) -> TokenDelta:
+    """Compute token count comparison between two prompt files.
+
+    Parameters
+    ----------
+    old_file:
+        The old parsed prompt file.
+    new_file:
+        The new parsed prompt file.
+    encoding:
+        tiktoken encoding name for token counting.
+
+    Returns
+    -------
+    TokenDelta
+        Token count delta.
+    """
+    tokenizer = Tokenizer(encoding=encoding)
+    old_total = tokenizer.count_file(old_file)
+    new_total = tokenizer.count_file(new_file)
+
+    delta = new_total - old_total
+    if old_total == 0:
+        percent_change = 100.0 if new_total > 0 else 0.0
+    else:
+        percent_change = round((delta / old_total) * 100, 2)
+
+    return TokenDelta(
+        old_total=old_total,
+        new_total=new_total,
+        delta=delta,
+        percent_change=percent_change,
+    )

promptdiff/models.py

@@ -0,0 +1,135 @@
+"""Data models for promptdiff.
+
+All models use Pydantic v2 syntax. These define the structured representation
+of prompt diffs, including message-level changes, variable changes, token
+deltas, and breaking change classification.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from pathlib import Path
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field, computed_field
+
+
+class ChangeStatus(str, Enum):
+    """Status of a change between two versions."""
+
+    ADDED = "added"
+    REMOVED = "removed"
+    MODIFIED = "modified"
+    UNCHANGED = "unchanged"
+
+
+class MessageDiff(BaseModel):
+    """Diff of a single message between old and new prompt versions."""
+
+    status: ChangeStatus = Field(..., description="Change status for this message")
+    role: str = Field(..., description="Message role (system, user, assistant, tool)")
+    old_content: Optional[str] = Field(
+        default=None, description="Content in the old version"
+    )
+    new_content: Optional[str] = Field(
+        default=None, description="Content in the new version"
+    )
+    content_diff: Optional[str] = Field(
+        default=None, description="Unified diff of the content"
+    )
+    token_delta: int = Field(
+        default=0, description="Change in token count for this message"
+    )
+    changes: list[str] = Field(
+        default_factory=list,
+        description="Human-readable descriptions of changes",
+    )
+
+
+class VariableDiff(BaseModel):
+    """Diff of a template variable between old and new prompt versions."""
+
+    name: str = Field(..., description="Variable name")
+    status: ChangeStatus = Field(..., description="Change status for this variable")
+    old_default: Optional[str] = Field(
+        default=None, description="Default value in the old version"
+    )
+    new_default: Optional[str] = Field(
+        default=None, description="Default value in the new version"
+    )
+    is_breaking: bool = Field(
+        default=False,
+        description="True if this is a breaking change",
+    )
+
+
+class MetadataDiff(BaseModel):
+    """Diff of a metadata key between old and new prompt versions."""
+
+    key: str = Field(..., description="Metadata key")
+    status: ChangeStatus = Field(..., description="Change status for this key")
+    old_value: Optional[Any] = Field(
+        default=None, description="Value in the old version"
+    )
+    new_value: Optional[Any] = Field(
+        default=None, description="Value in the new version"
+    )
+
+
+class TokenDelta(BaseModel):
+    """Token count comparison between old and new prompt versions."""
+
+    old_total: int = Field(..., description="Total tokens in old version")
+    new_total: int = Field(..., description="Total tokens in new version")
+    delta: int = Field(..., description="Change in token count (new - old)")
+    percent_change: float = Field(
+        ..., description="Percentage change in token count"
+    )
+
+
+class BreakingChange(BaseModel):
+    """A detected breaking change between prompt versions."""
+
+    category: str = Field(
+        ...,
+        description="Category: variable, message, model, role",
+    )
+    description: str = Field(
+        ..., description="Human-readable description of the breaking change"
+    )
+    severity: str = Field(
+        ..., description="Severity: high, medium, low"
+    )
+
+
+class PromptDiff(BaseModel):
+    """Complete structured diff between two prompt file versions."""
+
+    file_path: Path = Field(..., description="Path to the diffed file")
+    old_hash: str = Field(..., description="SHA256 hash of the old content")
+    new_hash: str = Field(..., description="SHA256 hash of the new content")
+    message_diffs: list[MessageDiff] = Field(
+        default_factory=list,
+        description="Per-message diffs",
+    )
+    variable_diffs: list[VariableDiff] = Field(
+        default_factory=list,
+        description="Per-variable diffs",
+    )
+    metadata_diffs: list[MetadataDiff] = Field(
+        default_factory=list,
+        description="Per-metadata-key diffs",
+    )
+    token_delta: TokenDelta = Field(
+        ..., description="Token count comparison"
+    )
+    breaking_changes: list[BreakingChange] = Field(
+        default_factory=list,
+        description="Detected breaking changes",
+    )
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def is_breaking(self) -> bool:
+        """True if any breaking changes were detected."""
+        return len(self.breaking_changes) > 0

promptdiff/reporter.py

@@ -0,0 +1,266 @@
+"""Output formatters for promptdiff.
+
+Supports text (Rich terminal), JSON, and Markdown output formats.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from promptdiff.models import (
+    ChangeStatus,
+    PromptDiff,
+)
+
+
+# ---------------------------------------------------------------------------
+# JSON output
+# ---------------------------------------------------------------------------
+
+
+def format_json(diff: PromptDiff) -> str:
+    """Format a PromptDiff as JSON.
+
+    Parameters
+    ----------
+    diff:
+        The prompt diff to format.
+
+    Returns
+    -------
+    str
+        JSON string.
+    """
+    return diff.model_dump_json(indent=2)
+
+
+# ---------------------------------------------------------------------------
+# Text output (Rich-ready, but returns plain strings with markup)
+# ---------------------------------------------------------------------------
+
+_STATUS_STYLE: dict[ChangeStatus, tuple[str, str]] = {
+    ChangeStatus.ADDED: ("[green]+[/green]", "green"),
+    ChangeStatus.REMOVED: ("[red]-[/red]", "red"),
+    ChangeStatus.MODIFIED: ("[yellow]~[/yellow]", "yellow"),
+    ChangeStatus.UNCHANGED: (" ", "dim"),
+}
+
+
+def format_text(diff: PromptDiff, show_token_detail: bool = False) -> str:
+    """Format a PromptDiff as rich-markup terminal text.
+
+    Parameters
+    ----------
+    diff:
+        The prompt diff to format.
+    show_token_detail:
+        If True, include per-message token breakdowns.
+
+    Returns
+    -------
+    str
+        Text with Rich markup suitable for Console.print().
+    """
+    lines: list[str] = []
+
+    # Header
+    lines.append("")
+    lines.append(f"[bold]Prompt Diff:[/bold] {diff.file_path}")
+    lines.append(f"  old: {diff.old_hash[:12]}  new: {diff.new_hash[:12]}")
+    lines.append("")
+
+    # Breaking changes summary
+    if diff.is_breaking:
+        lines.append(
+            f"[bold red]BREAKING CHANGES ({len(diff.breaking_changes)}):[/bold red]"
+        )
+        for bc in diff.breaking_changes:
+            severity_color = {"high": "red", "medium": "yellow", "low": "blue"}.get(
+                bc.severity, "white"
+            )
+            lines.append(
+                f"  [{severity_color}]{bc.severity.upper()}[/{severity_color}] "
+                f"[{bc.category}] {bc.description}"
+            )
+        lines.append("")
+    else:
+        lines.append("[green]No breaking changes detected.[/green]")
+        lines.append("")
+
+    # Token summary
+    td = diff.token_delta
+    delta_str = f"+{td.delta}" if td.delta > 0 else str(td.delta)
+    pct_str = f"+{td.percent_change}%" if td.percent_change > 0 else f"{td.percent_change}%"
+    delta_color = "red" if td.delta > 0 else "green" if td.delta < 0 else "dim"
+    lines.append("[bold]Token Delta:[/bold]")
+    lines.append(
+        f"  {td.old_total} -> {td.new_total} "
+        f"([{delta_color}]{delta_str}[/{delta_color}], {pct_str})"
+    )
+    lines.append("")
+
+    # Message diffs
+    msg_changes = [m for m in diff.message_diffs if m.status != ChangeStatus.UNCHANGED]
+    if msg_changes:
+        lines.append(f"[bold]Messages ({len(msg_changes)} changed):[/bold]")
+        for md in diff.message_diffs:
+            marker, color = _STATUS_STYLE[md.status]
+            if md.status == ChangeStatus.UNCHANGED and not show_token_detail:
+                continue
+            lines.append(f"  {marker} [{color}]{md.role}[/{color}]")
+            for change in md.changes:
+                lines.append(f"      {change}")
+            if show_token_detail and md.token_delta != 0:
+                sign = "+" if md.token_delta > 0 else ""
+                lines.append(f"      tokens: {sign}{md.token_delta}")
+            if md.content_diff and md.status == ChangeStatus.MODIFIED:
+                for dl in md.content_diff.splitlines()[:10]:
+                    if dl.startswith("+") and not dl.startswith("+++"):
+                        lines.append(f"      [green]{dl}[/green]")
+                    elif dl.startswith("-") and not dl.startswith("---"):
+                        lines.append(f"      [red]{dl}[/red]")
+                    else:
+                        lines.append(f"      {dl}")
+        lines.append("")
+
+    # Variable diffs
+    var_changes = [v for v in diff.variable_diffs if v.status != ChangeStatus.UNCHANGED]
+    if var_changes:
+        lines.append(f"[bold]Variables ({len(var_changes)} changed):[/bold]")
+        for vd in var_changes:
+            marker, color = _STATUS_STYLE[vd.status]
+            breaking_tag = " [red](BREAKING)[/red]" if vd.is_breaking else ""
+            lines.append(f"  {marker} [{color}]{vd.name}[/{color}]{breaking_tag}")
+            if vd.status == ChangeStatus.ADDED and vd.new_default is not None:
+                lines.append(f"      default: {vd.new_default!r}")
+            elif vd.status == ChangeStatus.MODIFIED:
+                lines.append(
+                    f"      default: {vd.old_default!r} -> {vd.new_default!r}"
+                )
+        lines.append("")
+
+    # Metadata diffs
+    meta_changes = [m for m in diff.metadata_diffs if m.status != ChangeStatus.UNCHANGED]
+    if meta_changes:
+        lines.append(f"[bold]Metadata ({len(meta_changes)} changed):[/bold]")
+        for md in meta_changes:
+            marker, color = _STATUS_STYLE[md.status]
+            lines.append(f"  {marker} [{color}]{md.key}[/{color}]")
+            if md.status == ChangeStatus.ADDED:
+                lines.append(f"      value: {md.new_value!r}")
+            elif md.status == ChangeStatus.REMOVED:
+                lines.append(f"      was: {md.old_value!r}")
+            elif md.status == ChangeStatus.MODIFIED:
+                lines.append(
+                    f"      {md.old_value!r} -> {md.new_value!r}"
+                )
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+# Markdown output (for GitHub PR comments)
+# ---------------------------------------------------------------------------
+
+_STATUS_EMOJI: dict[ChangeStatus, str] = {
+    ChangeStatus.ADDED: "+",
+    ChangeStatus.REMOVED: "-",
+    ChangeStatus.MODIFIED: "~",
+    ChangeStatus.UNCHANGED: " ",
+}
+
+
+def format_markdown(diff: PromptDiff) -> str:
+    """Format a PromptDiff as Markdown.
+
+    Parameters
+    ----------
+    diff:
+        The prompt diff to format.
+
+    Returns
+    -------
+    str
+        Markdown string suitable for GitHub PR comments.
+    """
+    lines: list[str] = []
+
+    # Header
+    lines.append(f"## Prompt Diff: `{diff.file_path}`")
+    lines.append("")
+    lines.append(f"**Old:** `{diff.old_hash[:12]}` | **New:** `{diff.new_hash[:12]}`")
+    lines.append("")
+
+    # Breaking changes
+    if diff.is_breaking:
+        lines.append(
+            f"### Breaking Changes ({len(diff.breaking_changes)})"
+        )
+        lines.append("")
+        for bc in diff.breaking_changes:
+            severity_badge = {
+                "high": "**HIGH**",
+                "medium": "MEDIUM",
+                "low": "low",
+            }.get(bc.severity, bc.severity)
+            lines.append(
+                f"- {severity_badge} [{bc.category}]: {bc.description}"
+            )
+        lines.append("")
+    else:
+        lines.append("No breaking changes detected.")
+        lines.append("")
+
+    # Token delta
+    td = diff.token_delta
+    delta_str = f"+{td.delta}" if td.delta > 0 else str(td.delta)
+    pct_str = f"+{td.percent_change}%" if td.percent_change > 0 else f"{td.percent_change}%"
+    lines.append("### Token Delta")
+    lines.append("")
+    lines.append(f"| Old | New | Delta | Change |")
+    lines.append(f"|-----|-----|-------|--------|")
+    lines.append(f"| {td.old_total} | {td.new_total} | {delta_str} | {pct_str} |")
+    lines.append("")
+
+    # Messages
+    msg_changes = [m for m in diff.message_diffs if m.status != ChangeStatus.UNCHANGED]
+    if msg_changes:
+        lines.append(f"### Messages ({len(msg_changes)} changed)")
+        lines.append("")
+        for md in msg_changes:
+            status_marker = _STATUS_EMOJI[md.status]
+            lines.append(f"- `{status_marker}` **{md.role}**: {md.status.value}")
+            for change in md.changes:
+                lines.append(f"  - {change}")
+            if md.content_diff:
+                lines.append("")
+                lines.append("  ```diff")
+                for dl in md.content_diff.splitlines()[:15]:
+                    lines.append(f"  {dl}")
+                lines.append("  ```")
+        lines.append("")
+
+    # Variables
+    var_changes = [v for v in diff.variable_diffs if v.status != ChangeStatus.UNCHANGED]
+    if var_changes:
+        lines.append(f"### Variables ({len(var_changes)} changed)")
+        lines.append("")
+        for vd in var_changes:
+            status_marker = _STATUS_EMOJI[vd.status]
+            breaking = " **(BREAKING)**" if vd.is_breaking else ""
+            lines.append(f"- `{status_marker}` `{vd.name}`: {vd.status.value}{breaking}")
+        lines.append("")
+
+    # Metadata
+    meta_changes = [m for m in diff.metadata_diffs if m.status != ChangeStatus.UNCHANGED]
+    if meta_changes:
+        lines.append(f"### Metadata ({len(meta_changes)} changed)")
+        lines.append("")
+        for md in meta_changes:
+            status_marker = _STATUS_EMOJI[md.status]
+            lines.append(f"- `{status_marker}` `{md.key}`: {md.status.value}")
+        lines.append("")
+
+    return "\n".join(lines)

promptdiff_ai-1.0.0.dist-info/METADATA

@@ -0,0 +1,183 @@
+Metadata-Version: 2.4
+Name: promptdiff-ai
+Version: 1.0.0
+Summary: Semantic diff for LLM prompt changes
+Project-URL: Homepage, https://github.com/scottconverse/promptdiff
+Project-URL: Repository, https://github.com/scottconverse/promptdiff
+Project-URL: Issues, https://github.com/scottconverse/promptdiff/issues
+Author: Scott Converse
+License: MIT
+Keywords: ai,breaking-changes,diff,llm,prompt,semantic
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.9
+Requires-Dist: prompttools-core-ai<2.0,>=1.0
+Requires-Dist: rich>=13.0
+Requires-Dist: typer[all]>=0.12
+Provides-Extra: dev
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# promptdiff
+
+Semantic diff for LLM prompt changes. Part of the [prompttools](https://github.com/scottconverse/prompttools) suite.
+
+Unlike generic text diffing, promptdiff understands prompt structure: messages, variables, metadata, and token counts. It classifies changes as breaking or non-breaking and outputs structured reports suitable for CI/CD pipelines and GitHub PR comments.
+
+## Installation
+
+```bash
+pip install promptdiff-ai
+```
+
+## Quick Start
+
+```bash
+# Compare two prompt files
+promptdiff old_prompt.yaml new_prompt.yaml
+
+# JSON output for CI pipelines
+promptdiff old.yaml new.yaml --format json
+
+# Markdown output for GitHub PR comments
+promptdiff old.yaml new.yaml --format markdown
+
+# Exit with code 1 if breaking changes are found (CI gate)
+promptdiff old.yaml new.yaml --exit-on-breaking
+
+# Show per-message token breakdowns
+promptdiff old.yaml new.yaml --token-detail
+```
+
+## What It Detects
+
+### Message-Level Changes
+- Added, removed, or modified messages
+- Per-message token deltas
+- Unified content diffs within modified messages
+
+### Variable Changes
+- New variables (with or without defaults)
+- Removed variables
+- Modified default values
+
+### Metadata Changes
+- Model changes
+- Added/removed/modified metadata keys
+
+### Token Deltas
+- Total token count comparison
+- Percentage change
+- Per-message breakdowns (with `--token-detail`)
+
+## Breaking Change Classification
+
+### Breaking (High Severity)
+- **New required variable** -- a variable added without a default value; existing callers will fail
+- **Removed variable** -- callers referencing this variable will break
+- **Removed message** -- changes the prompt structure
+
+### Breaking (Medium Severity)
+- **Model change** -- may affect behavior, pricing, and capabilities
+- **Role ordering change** -- may affect model behavior
+
+### Non-Breaking
+- Added variable with a default value
+- Added messages (extends the prompt)
+- Content modifications within existing messages
+- Metadata changes (except model)
+
+## Output Formats
+
+### Text (default)
+Rich terminal output with color-coded diffs:
+
+```
+Prompt Diff: new_prompt.yaml
+  old: a1b2c3d4e5f6  new: f6e5d4c3b2a1
+
+BREAKING CHANGES (2):
+  HIGH [variable] Variable 'tone' was removed
+  MEDIUM [model] Model changed from 'gpt-4' to 'gpt-4o'
+
+Token Delta:
+  150 -> 165 (+15, +10.0%)
+
+Messages (1 changed):
+  ~ system
+      System message content modified
+```
+
+### JSON
+Structured JSON for programmatic consumption:
+
+```bash
+promptdiff old.yaml new.yaml --format json
+```
+
+### Markdown
+GitHub-flavored Markdown for PR comments:
+
+```bash
+promptdiff old.yaml new.yaml --format markdown
+```
+
+## Python API
+
+```python
+from promptdiff import diff_files, format_text, format_json
+
+# Compare two files
+result = diff_files("prompts/v1.yaml", "prompts/v2.yaml")
+
+# Check for breaking changes
+if result.is_breaking:
+    for bc in result.breaking_changes:
+        print(f"[{bc.severity}] {bc.description}")
+
+# Get token delta
+print(f"Tokens: {result.token_delta.old_total} -> {result.token_delta.new_total}")
+
+# Format output
+print(format_text(result))
+```
+
+## CLI Reference
+
+```
+Usage: promptdiff [OPTIONS] FILE_A FILE_B
+
+Arguments:
+  FILE_A  Path to the old prompt file
+  FILE_B  Path to the new prompt file
+
+Options:
+  -f, --format [text|json|markdown]  Output format (default: text)
+  --exit-on-breaking                 Exit with code 1 if breaking changes found
+  --token-detail                     Show per-message token breakdowns
+  -e, --encoding TEXT                tiktoken encoding (default: cl100k_base)
+  -V, --version                      Show version and exit
+  --help                             Show this message and exit
+```
+
+## Supported File Formats
+
+All formats supported by prompttools-core:
+- YAML (`.yaml`, `.yml`)
+- JSON (`.json`)
+- Markdown (`.md`)
+- Text (`.txt`)
+
+## License
+
+MIT

promptdiff_ai-1.0.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+promptdiff/__init__.py,sha256=PPiJkrvfsrjdZonlq3KqxSyfF8KfSq9Zkgvn_QjS0ZE,1055
+promptdiff/analyzer.py,sha256=kP4gsNsKmi0kpleJghs9tJQeh2EvIjN7z-JrBy9TxHs,5203
+promptdiff/cli.py,sha256=hvgO0cMD8F1rnUraPtJ9I_kfQXEhZ5jcyYZI160OvWo,3562
+promptdiff/differ.py,sha256=WqNa9_mmqps3lVU4FePXTODSdFDBWVktDUdwmqzGqhc,11125
+promptdiff/models.py,sha256=Amx-Ux2WPQzSjksiF4X-hMe5ljwsbIlRQghG3nlaeBw,4446
+promptdiff/reporter.py,sha256=EfdKoVUEreik-LDXSU_Y_6vt-VxIt7iWY18yNfrq1zk,9401
+promptdiff_ai-1.0.0.dist-info/METADATA,sha256=4uF2wQ7mKwGbVeQKhAt5VXsxPvGwhLw9y-ZnpNqmz3I,5020
+promptdiff_ai-1.0.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+promptdiff_ai-1.0.0.dist-info/entry_points.txt,sha256=y1t43_RBs-tH4P2yHMEJJGskSOl5fQ25Kb5Fyz7a8RU,50
+promptdiff_ai-1.0.0.dist-info/RECORD,,

promptdiff_ai-1.0.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

promptdiff_ai-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+promptdiff = promptdiff.cli:app