promptdiff-ai 1.0.0__tar.gz

promptdiff_ai-1.0.0/.gitignore

@@ -0,0 +1,43 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+*.egg-info/
+*.egg
+dist/
+build/
+.eggs/
+*.whl
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.mypy_cache/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Secrets - NEVER commit these
+.env
+.env.*
+secrets/
+*.key
+*.pem
+
+# Cache
+.prompttools-cache/

promptdiff_ai-1.0.0/PKG-INFO

@@ -0,0 +1,183 @@
+Metadata-Version: 2.4
+Name: promptdiff-ai
+Version: 1.0.0
+Summary: Semantic diff for LLM prompt changes
+Project-URL: Homepage, https://github.com/scottconverse/promptdiff
+Project-URL: Repository, https://github.com/scottconverse/promptdiff
+Project-URL: Issues, https://github.com/scottconverse/promptdiff/issues
+Author: Scott Converse
+License: MIT
+Keywords: ai,breaking-changes,diff,llm,prompt,semantic
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.9
+Requires-Dist: prompttools-core-ai<2.0,>=1.0
+Requires-Dist: rich>=13.0
+Requires-Dist: typer[all]>=0.12
+Provides-Extra: dev
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# promptdiff
+
+Semantic diff for LLM prompt changes. Part of the [prompttools](https://github.com/scottconverse/prompttools) suite.
+
+Unlike generic text diffing, promptdiff understands prompt structure: messages, variables, metadata, and token counts. It classifies changes as breaking or non-breaking and outputs structured reports suitable for CI/CD pipelines and GitHub PR comments.
+
+## Installation
+
+```bash
+pip install promptdiff-ai
+```
+
+## Quick Start
+
+```bash
+# Compare two prompt files
+promptdiff old_prompt.yaml new_prompt.yaml
+
+# JSON output for CI pipelines
+promptdiff old.yaml new.yaml --format json
+
+# Markdown output for GitHub PR comments
+promptdiff old.yaml new.yaml --format markdown
+
+# Exit with code 1 if breaking changes are found (CI gate)
+promptdiff old.yaml new.yaml --exit-on-breaking
+
+# Show per-message token breakdowns
+promptdiff old.yaml new.yaml --token-detail
+```
+
+## What It Detects
+
+### Message-Level Changes
+- Added, removed, or modified messages
+- Per-message token deltas
+- Unified content diffs within modified messages
+
+### Variable Changes
+- New variables (with or without defaults)
+- Removed variables
+- Modified default values
+
+### Metadata Changes
+- Model changes
+- Added/removed/modified metadata keys
+
+### Token Deltas
+- Total token count comparison
+- Percentage change
+- Per-message breakdowns (with `--token-detail`)
+
+## Breaking Change Classification
+
+### Breaking (High Severity)
+- **New required variable** -- a variable added without a default value; existing callers will fail
+- **Removed variable** -- callers referencing this variable will break
+- **Removed message** -- changes the prompt structure
+
+### Breaking (Medium Severity)
+- **Model change** -- may affect behavior, pricing, and capabilities
+- **Role ordering change** -- may affect model behavior
+
+### Non-Breaking
+- Added variable with a default value
+- Added messages (extends the prompt)
+- Content modifications within existing messages
+- Metadata changes (except model)
+
+## Output Formats
+
+### Text (default)
+Rich terminal output with color-coded diffs:
+
+```
+Prompt Diff: new_prompt.yaml
+  old: a1b2c3d4e5f6  new: f6e5d4c3b2a1
+
+BREAKING CHANGES (2):
+  HIGH [variable] Variable 'tone' was removed
+  MEDIUM [model] Model changed from 'gpt-4' to 'gpt-4o'
+
+Token Delta:
+  150 -> 165 (+15, +10.0%)
+
+Messages (1 changed):
+  ~ system
+      System message content modified
+```
+
+### JSON
+Structured JSON for programmatic consumption:
+
+```bash
+promptdiff old.yaml new.yaml --format json
+```
+
+### Markdown
+GitHub-flavored Markdown for PR comments:
+
+```bash
+promptdiff old.yaml new.yaml --format markdown
+```
+
+## Python API
+
+```python
+from promptdiff import diff_files, format_text, format_json
+
+# Compare two files
+result = diff_files("prompts/v1.yaml", "prompts/v2.yaml")
+
+# Check for breaking changes
+if result.is_breaking:
+    for bc in result.breaking_changes:
+        print(f"[{bc.severity}] {bc.description}")
+
+# Get token delta
+print(f"Tokens: {result.token_delta.old_total} -> {result.token_delta.new_total}")
+
+# Format output
+print(format_text(result))
+```
+
+## CLI Reference
+
+```
+Usage: promptdiff [OPTIONS] FILE_A FILE_B
+
+Arguments:
+  FILE_A  Path to the old prompt file
+  FILE_B  Path to the new prompt file
+
+Options:
+  -f, --format [text|json|markdown]  Output format (default: text)
+  --exit-on-breaking                 Exit with code 1 if breaking changes found
+  --token-detail                     Show per-message token breakdowns
+  -e, --encoding TEXT                tiktoken encoding (default: cl100k_base)
+  -V, --version                      Show version and exit
+  --help                             Show this message and exit
+```
+
+## Supported File Formats
+
+All formats supported by prompttools-core:
+- YAML (`.yaml`, `.yml`)
+- JSON (`.json`)
+- Markdown (`.md`)
+- Text (`.txt`)
+
+## License
+
+MIT

promptdiff_ai-1.0.0/README.md

@@ -0,0 +1,153 @@
+# promptdiff
+
+Semantic diff for LLM prompt changes. Part of the [prompttools](https://github.com/scottconverse/prompttools) suite.
+
+Unlike generic text diffing, promptdiff understands prompt structure: messages, variables, metadata, and token counts. It classifies changes as breaking or non-breaking and outputs structured reports suitable for CI/CD pipelines and GitHub PR comments.
+
+## Installation
+
+```bash
+pip install promptdiff-ai
+```
+
+## Quick Start
+
+```bash
+# Compare two prompt files
+promptdiff old_prompt.yaml new_prompt.yaml
+
+# JSON output for CI pipelines
+promptdiff old.yaml new.yaml --format json
+
+# Markdown output for GitHub PR comments
+promptdiff old.yaml new.yaml --format markdown
+
+# Exit with code 1 if breaking changes are found (CI gate)
+promptdiff old.yaml new.yaml --exit-on-breaking
+
+# Show per-message token breakdowns
+promptdiff old.yaml new.yaml --token-detail
+```
+
+## What It Detects
+
+### Message-Level Changes
+- Added, removed, or modified messages
+- Per-message token deltas
+- Unified content diffs within modified messages
+
+### Variable Changes
+- New variables (with or without defaults)
+- Removed variables
+- Modified default values
+
+### Metadata Changes
+- Model changes
+- Added/removed/modified metadata keys
+
+### Token Deltas
+- Total token count comparison
+- Percentage change
+- Per-message breakdowns (with `--token-detail`)
+
+## Breaking Change Classification
+
+### Breaking (High Severity)
+- **New required variable** -- a variable added without a default value; existing callers will fail
+- **Removed variable** -- callers referencing this variable will break
+- **Removed message** -- changes the prompt structure
+
+### Breaking (Medium Severity)
+- **Model change** -- may affect behavior, pricing, and capabilities
+- **Role ordering change** -- may affect model behavior
+
+### Non-Breaking
+- Added variable with a default value
+- Added messages (extends the prompt)
+- Content modifications within existing messages
+- Metadata changes (except model)
+
+## Output Formats
+
+### Text (default)
+Rich terminal output with color-coded diffs:
+
+```
+Prompt Diff: new_prompt.yaml
+  old: a1b2c3d4e5f6  new: f6e5d4c3b2a1
+
+BREAKING CHANGES (2):
+  HIGH [variable] Variable 'tone' was removed
+  MEDIUM [model] Model changed from 'gpt-4' to 'gpt-4o'
+
+Token Delta:
+  150 -> 165 (+15, +10.0%)
+
+Messages (1 changed):
+  ~ system
+      System message content modified
+```
+
+### JSON
+Structured JSON for programmatic consumption:
+
+```bash
+promptdiff old.yaml new.yaml --format json
+```
+
+### Markdown
+GitHub-flavored Markdown for PR comments:
+
+```bash
+promptdiff old.yaml new.yaml --format markdown
+```
+
+## Python API
+
+```python
+from promptdiff import diff_files, format_text, format_json
+
+# Compare two files
+result = diff_files("prompts/v1.yaml", "prompts/v2.yaml")
+
+# Check for breaking changes
+if result.is_breaking:
+    for bc in result.breaking_changes:
+        print(f"[{bc.severity}] {bc.description}")
+
+# Get token delta
+print(f"Tokens: {result.token_delta.old_total} -> {result.token_delta.new_total}")
+
+# Format output
+print(format_text(result))
+```
+
+## CLI Reference
+
+```
+Usage: promptdiff [OPTIONS] FILE_A FILE_B
+
+Arguments:
+  FILE_A  Path to the old prompt file
+  FILE_B  Path to the new prompt file
+
+Options:
+  -f, --format [text|json|markdown]  Output format (default: text)
+  --exit-on-breaking                 Exit with code 1 if breaking changes found
+  --token-detail                     Show per-message token breakdowns
+  -e, --encoding TEXT                tiktoken encoding (default: cl100k_base)
+  -V, --version                      Show version and exit
+  --help                             Show this message and exit
+```
+
+## Supported File Formats
+
+All formats supported by prompttools-core:
+- YAML (`.yaml`, `.yml`)
+- JSON (`.json`)
+- Markdown (`.md`)
+- Text (`.txt`)
+
+## License
+
+MIT

promptdiff_ai-1.0.0/pyproject.toml

@@ -0,0 +1,63 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "promptdiff-ai"
+version = "1.0.0"
+description = "Semantic diff for LLM prompt changes"
+readme = "README.md"
+license = {text = "MIT"}
+requires-python = ">=3.9"
+authors = [
+    {name = "Scott Converse"},
+]
+keywords = ["llm", "prompt", "diff", "semantic", "ai", "breaking-changes"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Quality Assurance",
+]
+dependencies = [
+    "prompttools-core-ai>=1.0,<2.0",
+    "typer[all]>=0.12",
+    "rich>=13.0",
+]
+
+[project.scripts]
+promptdiff = "promptdiff.cli:app"
+
+[project.urls]
+Homepage = "https://github.com/scottconverse/promptdiff"
+Repository = "https://github.com/scottconverse/promptdiff"
+Issues = "https://github.com/scottconverse/promptdiff/issues"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+    "pytest-cov",
+    "ruff",
+    "mypy",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/promptdiff"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-v --tb=short"
+
+[tool.ruff]
+target-version = "py39"
+line-length = 100
+
+[tool.mypy]
+python_version = "3.9"
+warn_return_any = true
+warn_unused_configs = true

promptdiff_ai-1.0.0/src/promptdiff/__init__.py

@@ -0,0 +1,52 @@
+"""promptdiff: Semantic diff for LLM prompt changes.
+
+Public API exports for convenience imports::
+
+    from promptdiff import diff_files, PromptDiff, format_text
+"""
+
+from promptdiff.models import (
+    BreakingChange,
+    ChangeStatus,
+    MessageDiff,
+    MetadataDiff,
+    PromptDiff,
+    TokenDelta,
+    VariableDiff,
+)
+from promptdiff.differ import (
+    compute_token_delta,
+    diff_files,
+    diff_messages,
+    diff_metadata,
+    diff_variables,
+)
+from promptdiff.analyzer import analyze_breaking_changes
+from promptdiff.reporter import format_json, format_markdown, format_text
+
+__version__ = "1.0.0"
+
+__all__ = [
+    # Models
+    "BreakingChange",
+    "ChangeStatus",
+    "MessageDiff",
+    "MetadataDiff",
+    "PromptDiff",
+    "TokenDelta",
+    "VariableDiff",
+    # Differ
+    "compute_token_delta",
+    "diff_files",
+    "diff_messages",
+    "diff_metadata",
+    "diff_variables",
+    # Analyzer
+    "analyze_breaking_changes",
+    # Reporter
+    "format_json",
+    "format_markdown",
+    "format_text",
+    # Version
+    "__version__",
+]

promptdiff_ai-1.0.0/src/promptdiff/analyzer.py

@@ -0,0 +1,163 @@
+"""Breaking change detection for promptdiff.
+
+Analyzes a PromptDiff and classifies changes by severity.
+
+Breaking changes:
+  - New required variable (no default) -- high severity
+  - Removed variable -- high severity
+  - Removed message -- high severity
+  - Changed role ordering -- medium severity
+  - Model change (in metadata) -- medium severity
+
+Non-breaking changes:
+  - Added variable with default
+  - Added messages
+  - Content modifications
+  - Metadata changes (except model)
+"""
+
+from __future__ import annotations
+
+from promptdiff.models import (
+    BreakingChange,
+    ChangeStatus,
+    PromptDiff,
+)
+
+
+def analyze_breaking_changes(diff: PromptDiff) -> list[BreakingChange]:
+    """Analyze a diff for breaking changes.
+
+    Parameters
+    ----------
+    diff:
+        The prompt diff to analyze.
+
+    Returns
+    -------
+    list[BreakingChange]
+        All detected breaking changes sorted by severity.
+    """
+    changes: list[BreakingChange] = []
+    changes.extend(_check_variable_changes(diff))
+    changes.extend(_check_message_changes(diff))
+    changes.extend(_check_metadata_changes(diff))
+    changes.extend(_check_role_ordering(diff))
+
+    # Sort by severity: high first, then medium, then low.
+    severity_order = {"high": 0, "medium": 1, "low": 2}
+    changes.sort(key=lambda c: severity_order.get(c.severity, 3))
+
+    return changes
+
+
+def _check_variable_changes(diff: PromptDiff) -> list[BreakingChange]:
+    """Detect breaking variable changes."""
+    changes: list[BreakingChange] = []
+
+    for vd in diff.variable_diffs:
+        if vd.status == ChangeStatus.REMOVED:
+            changes.append(
+                BreakingChange(
+                    category="variable",
+                    description=f"Variable '{vd.name}' was removed",
+                    severity="high",
+                )
+            )
+        elif vd.status == ChangeStatus.ADDED and vd.is_breaking:
+            changes.append(
+                BreakingChange(
+                    category="variable",
+                    description=(
+                        f"New required variable '{vd.name}' added without a default value"
+                    ),
+                    severity="high",
+                )
+            )
+
+    return changes
+
+
+def _check_message_changes(diff: PromptDiff) -> list[BreakingChange]:
+    """Detect breaking message changes."""
+    changes: list[BreakingChange] = []
+
+    for md in diff.message_diffs:
+        if md.status == ChangeStatus.REMOVED:
+            changes.append(
+                BreakingChange(
+                    category="message",
+                    description=f"{md.role.capitalize()} message was removed",
+                    severity="high",
+                )
+            )
+
+    return changes
+
+
+def _check_metadata_changes(diff: PromptDiff) -> list[BreakingChange]:
+    """Detect breaking metadata changes (model changes)."""
+    changes: list[BreakingChange] = []
+
+    for md in diff.metadata_diffs:
+        if md.key == "model" and md.status == ChangeStatus.MODIFIED:
+            changes.append(
+                BreakingChange(
+                    category="model",
+                    description=(
+                        f"Model changed from '{md.old_value}' to '{md.new_value}'"
+                    ),
+                    severity="medium",
+                )
+            )
+        elif md.key == "model" and md.status == ChangeStatus.REMOVED:
+            changes.append(
+                BreakingChange(
+                    category="model",
+                    description="Model specification was removed",
+                    severity="medium",
+                )
+            )
+
+    return changes
+
+
+def _check_role_ordering(diff: PromptDiff) -> list[BreakingChange]:
+    """Detect role ordering changes.
+
+    If the sequence of roles (ignoring unchanged content) changes between
+    old and new, this is potentially breaking.
+    """
+    changes: list[BreakingChange] = []
+
+    # Reconstruct old and new role sequences from the message diffs.
+    old_roles: list[str] = []
+    new_roles: list[str] = []
+
+    for md in diff.message_diffs:
+        if md.status == ChangeStatus.REMOVED:
+            old_roles.append(md.role)
+        elif md.status == ChangeStatus.ADDED:
+            new_roles.append(md.role)
+        elif md.status in (ChangeStatus.MODIFIED, ChangeStatus.UNCHANGED):
+            old_roles.append(md.role)
+            new_roles.append(md.role)
+
+    if old_roles != new_roles and len(old_roles) > 0 and len(new_roles) > 0:
+        # Only flag if neither list is a subset scenario (pure additions/removals
+        # are already caught above). Check if the common elements changed order.
+        common_old = [r for r in old_roles if r in new_roles]
+        common_new = [r for r in new_roles if r in old_roles]
+        if common_old != common_new:
+            changes.append(
+                BreakingChange(
+                    category="role",
+                    description=(
+                        f"Message role ordering changed: "
+                        f"{' -> '.join(old_roles)} to {' -> '.join(new_roles)}"
+                    ),
+                    severity="medium",
+                )
+            )
+
+    return changes