crewscore 0.2.0__py3-none-any.whl

crewscore/__init__.py

@@ -0,0 +1,3 @@
+"""CrewScore — offline structural production-readiness scorecard for AI agent prompts."""
+
+__version__ = "0.2.0"

crewscore/cli.py

@@ -0,0 +1,403 @@
+"""CrewScore CLI — structural production-readiness scoring for AI agents."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import click
+from rich.console import Console
+from rich.panel import Panel
+
+from crewscore import __version__
+from crewscore.report import render_badge_svg, render_html_report, share_text
+from crewscore.scoring import DIMENSIONS, build_result, tier_color
+from crewscore.scorers import structural_analysis
+from crewscore.vendor_scorecard import assess_vendor
+
+console = Console()
+
+BRAND = "CrewScore"
+HOMEPAGE = "https://crewscore.ai"
+REPO = "https://github.com/shmindmaster/crewscore"
+
+
+def render_score_bar(score: int) -> str:
+    filled = round(score / 10)
+    empty = 10 - filled
+    bar = "=" * filled + "-" * empty
+
+    if score >= 90:
+        color = "green"
+        status = "PASS"
+    elif score >= 70:
+        color = "yellow"
+        status = "MONITOR"
+    elif score >= 50:
+        color = "dark_orange"
+        status = "WEAK"
+    else:
+        color = "red"
+        status = "MISSING" if score == 0 else "FAILS"
+
+    return f"[{color}][{bar}] {score:>3}/100[/{color}]  {status}"
+
+
+@click.group()
+@click.version_option(version=__version__, prog_name="crewscore")
+def main():
+    """CrewScore — offline structural scorecard for AI agent system prompts."""
+    pass
+
+
+main.add_command(assess_vendor)
+
+
+@main.command()
+@click.option("--prompt", "-p", help="System prompt string to test")
+@click.option(
+    "--prompt-file",
+    "-f",
+    type=click.Path(exists=True),
+    help="Path to system prompt file",
+)
+@click.option(
+    "--json",
+    "as_json",
+    is_flag=True,
+    help="Emit machine-readable JSON (for CI)",
+)
+@click.option(
+    "--threshold",
+    type=click.IntRange(0, 100),
+    default=None,
+    help="Exit non-zero if overall score is below this threshold",
+)
+@click.option(
+    "--explain",
+    is_flag=True,
+    help="Show matched vs missing guardrail signals per dimension",
+)
+@click.option(
+    "--report",
+    type=click.Path(),
+    default=None,
+    help="Write a self-contained HTML scorecard to this path",
+)
+@click.option(
+    "--badge",
+    type=click.Path(),
+    default=None,
+    help="Write an SVG badge to this path",
+)
+def test(prompt, prompt_file, as_json, threshold, explain, report, badge):
+    """Run structural production-readiness analysis on an agent system prompt.
+
+    This mode is offline and free: it scans the prompt text for guardrail
+    signals. It does not run live LLM adversarial attacks.
+    """
+    system_prompt = None
+    source = "prompt"
+
+    if prompt:
+        system_prompt = prompt
+        source = "prompt"
+    elif prompt_file:
+        system_prompt = Path(prompt_file).read_text(encoding="utf-8")
+        source = str(prompt_file)
+    else:
+        console.print(
+            "[red]Error: Provide --prompt or --prompt-file[/red]",
+            err=True,
+        )
+        console.print(
+            '[dim]Example: crewscore test --prompt "You are a helpful assistant..."[/dim]',
+            err=True,
+        )
+        sys.exit(1)
+
+    findings = None
+    if explain:
+        dimensions, findings = structural_analysis.analyze_with_findings(system_prompt)
+    else:
+        dimensions = structural_analysis.analyze(system_prompt)
+    result = build_result(dimensions, mode="structural", source=source)
+
+    if report:
+        report_path = Path(report)
+        report_path.parent.mkdir(parents=True, exist_ok=True)
+        report_path.write_text(render_html_report(result), encoding="utf-8")
+    if badge:
+        badge_path = Path(badge)
+        badge_path.parent.mkdir(parents=True, exist_ok=True)
+        badge_path.write_text(render_badge_svg(result), encoding="utf-8")
+
+    if as_json:
+        payload = result.to_dict()
+        if findings is not None:
+            payload["findings"] = findings
+        click.echo(json.dumps(payload, indent=2, sort_keys=True))
+    else:
+        color = tier_color(result.overall)
+        console.print()
+        console.print(
+            Panel(
+                f"[bold]{BRAND.upper()} — Structural Production Readiness Report[/bold]",
+                border_style="blue",
+                expand=False,
+            )
+        )
+        console.print()
+        console.print(
+            "[dim]Mode: structural (offline prompt scan). "
+            "Not a substitute for live behavioral red-teaming.[/dim]"
+        )
+        console.print()
+
+        for label, key in DIMENSIONS:
+            score = result.dimensions.get(key, 0)
+            console.print(f"  {label:<32} {render_score_bar(score)}")
+
+        console.print()
+        console.print(f"  {'-' * 54}")
+        console.print(
+            f"  [{color}]OVERALL SCORE:  {result.overall}/100  {result.tier}[/{color}]"
+        )
+        console.print(f"  {'-' * 54}")
+
+        critical = [
+            (label, key)
+            for label, key in DIMENSIONS
+            if result.dimensions.get(key, 0) < 50
+        ]
+        if critical:
+            console.print()
+            for label, key in critical:
+                score = result.dimensions[key]
+                if score == 0:
+                    console.print(
+                        f"  [red]CRITICAL:[/red] No {label.lower()} detected in your agent."
+                    )
+                else:
+                    console.print(
+                        f"  [dark_orange]WEAK:[/dark_orange] {label} is below "
+                        f"production threshold ({score}/100)."
+                    )
+
+        if findings is not None:
+            _render_findings(findings)
+
+        console.print()
+        console.print(f"  Share: {share_text(result)}")
+        console.print()
+        console.print(
+            f"  -> Run [bold]crewscore fix[/bold] to apply recommended guardrail patterns."
+        )
+        console.print(
+            "  -> Re-run with [bold]--json[/bold] for CI. "
+            "Use [bold]--threshold N[/bold] to fail builds below N."
+        )
+        if report:
+            console.print(f"  -> HTML report written to [bold]{report}[/bold]")
+        if badge:
+            console.print(f"  -> SVG badge written to [bold]{badge}[/bold]")
+        console.print(f"  -> {HOMEPAGE}")
+        console.print()
+
+    if threshold is not None and result.overall < threshold:
+        if not as_json:
+            console.print(
+                f"  [red]Threshold failure: {result.overall} < {threshold}[/red]",
+                err=True,
+            )
+        sys.exit(2)
+
+
+def _render_findings(findings: list[dict]) -> None:
+    """Print matched/missing findings grouped by dimension."""
+    label_by_key = {key: label for label, key in DIMENSIONS}
+    by_dim: dict[str, list[dict]] = {}
+    for f in findings:
+        by_dim.setdefault(f["dimension"], []).append(f)
+
+    console.print()
+    console.print("[bold]Findings (matched vs missing signals)[/bold]")
+    for _, key in DIMENSIONS:
+        items = by_dim.get(key, [])
+        if not items:
+            continue
+        dim_label = label_by_key.get(key, key)
+        console.print()
+        console.print(f"  [bold]{dim_label}[/bold] ({key})")
+        for f in items:
+            status = f["status"]
+            reason = f.get("pattern_or_reason") or ""
+            snippet = f.get("snippet")
+            if status == "matched":
+                detail = snippet or reason
+                console.print(f"    [green]matched[/green]  {detail}")
+            else:
+                console.print(f"    [red]missing[/red]  {reason}")
+
+
+
+@main.command()
+@click.option("--prompt", "-p", help="System prompt string to fix")
+@click.option(
+    "--prompt-file",
+    "-f",
+    type=click.Path(exists=True),
+    help="Path to system prompt file",
+)
+@click.option(
+    "--apply",
+    is_flag=True,
+    help="Write the fixed prompt back to the file (in-place)",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(),
+    help="Write enhanced prompt to a new file",
+)
+@click.option(
+    "--json",
+    "as_json",
+    is_flag=True,
+    help="Emit JSON summary of applied dimensions and score delta",
+)
+def fix(prompt, prompt_file, apply, output, as_json):
+    """Append recommended guardrail patterns to a system prompt."""
+    from crewscore.scorers.fix_patterns import apply_fixes, explain_fixes, generate_fixes
+
+    system_prompt = None
+    source_path = None
+
+    if prompt:
+        system_prompt = prompt
+    elif prompt_file:
+        source_path = Path(prompt_file)
+        system_prompt = source_path.read_text(encoding="utf-8")
+    else:
+        console.print(
+            "[red]Error: Provide --prompt or --prompt-file[/red]",
+            err=True,
+        )
+        console.print(
+            "[dim]Example: crewscore fix --prompt-file ./system-prompt.md --apply[/dim]",
+            err=True,
+        )
+        sys.exit(1)
+
+    before = structural_analysis.analyze(system_prompt)
+    before_result = build_result(before)
+    fixes = generate_fixes(before)
+
+    honesty_note = (
+        "Templates must be paired with runtime gates "
+        "(tool allowlists, human approval hooks, logging, and policy enforcement)"
+    )
+
+    if not fixes:
+        if as_json:
+            click.echo(
+                json.dumps(
+                    {
+                        "fixes_applied": [],
+                        "before": before_result.to_dict(),
+                        "after": before_result.to_dict(),
+                        "message": "No fixes needed",
+                        "note": honesty_note,
+                    },
+                    indent=2,
+                    sort_keys=True,
+                )
+            )
+        else:
+            console.print()
+            console.print(
+                "  [green]No fixes needed — structural score is already strong.[/green]"
+            )
+            console.print()
+        return
+
+    enhanced = apply_fixes(system_prompt, fixes)
+    after = structural_analysis.analyze(enhanced)
+    after_result = build_result(after)
+
+    if apply and source_path:
+        source_path.write_text(enhanced, encoding="utf-8")
+    elif output:
+        Path(output).write_text(enhanced, encoding="utf-8")
+
+    if as_json:
+        click.echo(
+            json.dumps(
+                {
+                    "fixes_applied": list(fixes.keys()),
+                    "before": before_result.to_dict(),
+                    "after": after_result.to_dict(),
+                    "written": bool(apply and source_path) or bool(output),
+                    "path": str(source_path)
+                    if apply and source_path
+                    else (output or None),
+                    "note": honesty_note,
+                },
+                indent=2,
+                sort_keys=True,
+            )
+        )
+        return
+
+    console.print()
+    console.print(
+        Panel(
+            f"[bold]{BRAND.upper()} — Applying Fixes[/bold]",
+            border_style="green",
+            expand=False,
+        )
+    )
+    console.print()
+    console.print(explain_fixes(fixes))
+    console.print()
+    console.print(
+        "[yellow]Honesty note:[/yellow] These are prompt templates — "
+        "they must be paired with runtime gates (tool allowlists, human "
+        "approval hooks, logging, and policy enforcement) to have real effect."
+    )
+    console.print()
+
+    if apply and source_path:
+        console.print(f"  [green]Fixes applied in-place to {source_path}[/green]")
+        console.print()
+        console.print(
+            f"  Score: [red]{before_result.overall}/100[/red] -> "
+            f"[green]{after_result.overall}/100[/green] "
+            f"(+{after_result.overall - before_result.overall})"
+        )
+        console.print()
+    elif output:
+        console.print(f"  [green]Enhanced prompt written to {output}[/green]")
+        console.print()
+        console.print(
+            f"  Score: [red]{before_result.overall}/100[/red] -> "
+            f"[green]{after_result.overall}/100[/green] "
+            f"(+{after_result.overall - before_result.overall})"
+        )
+        console.print()
+    else:
+        console.print("[dim]--- Enhanced System Prompt ---[/dim]")
+        console.print()
+        console.print(enhanced)
+        console.print()
+        console.print("[dim]--- End ---[/dim]")
+        console.print()
+        console.print(
+            "[dim]Use --apply to write in-place, or --output <file> to save to a new file.[/dim]"
+        )
+        console.print()
+
+
+if __name__ == "__main__":
+    main()

crewscore/report.py

@@ -0,0 +1,171 @@
+"""Self-contained HTML report, SVG badge, and share text for CrewScore."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from html import escape
+from xml.sax.saxutils import escape as xml_escape
+
+from crewscore import __version__
+from crewscore.scoring import DIMENSIONS, ScoreResult
+
+HOMEPAGE = "https://crewscore.ai"
+
+# Tier colors aligned with index.html score classes
+_TIER_HEX = {
+    "green": "#10b981",
+    "yellow": "#eab308",
+    "dark_orange": "#f59e0b",
+    "orange": "#f59e0b",
+    "red": "#ef4444",
+}
+
+
+def _score_color_hex(overall: int) -> str:
+    if overall >= 90:
+        return _TIER_HEX["green"]
+    if overall >= 70:
+        return _TIER_HEX["yellow"]
+    if overall >= 50:
+        return _TIER_HEX["dark_orange"]
+    return _TIER_HEX["red"]
+
+
+def _bar_color(score: int) -> str:
+    return _score_color_hex(score)
+
+
+def share_text(result: ScoreResult) -> str:
+    """One-line share copy with overall score and product URL."""
+    return (
+        f"My AI agent scored {result.overall}/100 on CrewScore "
+        f"({result.tier}) — structural production-readiness scan. "
+        f"{HOMEPAGE}"
+    )
+
+
+def render_badge_svg(result: ScoreResult) -> str:
+    """Shields-style SVG badge: CrewScore | {score}/100 colored by tier."""
+    label = "CrewScore"
+    value = f"{result.overall}/100"
+    color = _score_color_hex(result.overall)
+
+    # Approximate widths for monospace-ish badge layout
+    label_w = 78
+    value_w = 54
+    total_w = label_w + value_w
+    label_x = label_w / 2
+    value_x = label_w + value_w / 2
+
+    label_esc = xml_escape(label)
+    value_esc = xml_escape(value)
+
+    return (
+        f'<svg xmlns="http://www.w3.org/2000/svg" width="{total_w}" height="20" '
+        f'role="img" aria-label="{label_esc}: {value_esc}">'
+        f"<title>{label_esc}: {value_esc}</title>"
+        f'<linearGradient id="s" x2="0" y2="100%">'
+        f'<stop offset="0" stop-color="#bbb" stop-opacity=".1"/>'
+        f'<stop offset="1" stop-opacity=".1"/>'
+        f"</linearGradient>"
+        f'<clipPath id="r"><rect width="{total_w}" height="20" rx="3" fill="#fff"/></clipPath>'
+        f'<g clip-path="url(#r)">'
+        f'<rect width="{label_w}" height="20" fill="#555"/>'
+        f'<rect x="{label_w}" width="{value_w}" height="20" fill="{color}"/>'
+        f'<rect width="{total_w}" height="20" fill="url(#s)"/>'
+        f"</g>"
+        f'<g fill="#fff" text-anchor="middle" '
+        f'font-family="Verdana,Geneva,DejaVu Sans,sans-serif" '
+        f'font-size="11">'
+        f'<text x="{label_x}" y="14">{label_esc}</text>'
+        f'<text x="{value_x}" y="14">{value_esc}</text>'
+        f"</g>"
+        f"</svg>"
+    )
+
+
+def render_html_report(
+    result: ScoreResult,
+    *,
+    generated_at: str | None = None,
+) -> str:
+    """Self-contained dark HTML scorecard (inline CSS, no scripts/CDN)."""
+    if generated_at is None:
+        generated_at = (
+            datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+        )
+
+    color = _score_color_hex(result.overall)
+    overall = result.overall
+    tier = escape(result.tier)
+    mode = escape(result.mode)
+    source = escape(result.source)
+    version = escape(__version__)
+    ts = escape(generated_at)
+
+    rows: list[str] = []
+    for label, key in DIMENSIONS:
+        score = result.dimensions.get(key, 0)
+        bar_color = _bar_color(score)
+        rows.append(
+            f'<div class="dim-row">'
+            f'<span class="dim-label">{escape(label)}</span>'
+            f'<div class="dim-bar">'
+            f'<div class="dim-fill" style="width:{score}%;background:{bar_color}"></div>'
+            f"</div>"
+            f'<span class="dim-score">{score}/100</span>'
+            f"</div>"
+        )
+    dim_html = "\n".join(rows)
+
+    return f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>CrewScore — {overall}/100</title>
+<style>
+*{{box-sizing:border-box;margin:0;padding:0}}
+body{{font-family:'SF Mono',SFMono-Regular,Consolas,'Liberation Mono',Menlo,monospace;
+background:#0f0f1a;color:#e2e8f0;min-height:100vh;padding:2rem 1rem;
+display:flex;flex-direction:column;align-items:center}}
+.container{{max-width:640px;width:100%}}
+h1{{font-size:1.75rem;color:#fff;text-align:center;margin-bottom:0.25rem}}
+.subtitle{{text-align:center;color:#94a3b8;font-size:0.85rem;margin-bottom:1.5rem}}
+.card{{background:#1a1f2e;border:1px solid #334155;border-radius:12px;padding:1.5rem}}
+.score-big{{font-size:3rem;font-weight:bold;text-align:center;color:{color};margin:0.5rem 0}}
+.tier{{text-align:center;font-size:1.05rem;font-weight:bold;color:{color};margin-bottom:1rem}}
+.meta{{text-align:center;font-size:0.75rem;color:#64748b;margin-bottom:1.25rem}}
+.dim-row{{display:flex;align-items:center;gap:0.5rem;margin:0.45rem 0;font-size:0.75rem}}
+.dim-label{{width:200px;color:#94a3b8;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}}
+.dim-bar{{flex:1;height:8px;background:#1e293b;border-radius:4px;overflow:hidden}}
+.dim-fill{{height:100%;border-radius:4px}}
+.dim-score{{width:55px;text-align:right;color:#64748b}}
+.disclaimer{{margin-top:1.25rem;padding:0.75rem;background:#0f0f1a;border:1px solid #334155;
+border-radius:8px;font-size:0.75rem;color:#94a3b8;line-height:1.45}}
+.footer{{margin-top:1.25rem;text-align:center;font-size:0.7rem;color:#475569;line-height:1.5}}
+.footer a{{color:#3b82f6;text-decoration:none}}
+</style>
+</head>
+<body>
+<div class="container">
+  <h1>CrewScore</h1>
+  <p class="subtitle">Structural Production Readiness Report</p>
+  <div class="card">
+    <div class="score-big">{overall}/100</div>
+    <div class="tier">{tier}</div>
+    <div class="meta">Mode: {mode} · Source: {source}</div>
+    {dim_html}
+    <div class="disclaimer">
+      <strong>Structural scan only</strong> — offline pattern match on prompt text.
+      Not a substitute for live behavioral red-teaming or runtime proof of safety.
+    </div>
+  </div>
+  <div class="footer">
+    CrewScore v{version} · Generated {ts}<br>
+    <a href="{HOMEPAGE}">crewscore.ai</a>
+  </div>
+</div>
+</body>
+</html>
+"""

crewscore/scorers/__init__.py

@@ -0,0 +1 @@
+"""Scorer modules for each evaluation dimension."""