specscore 0.1.4__py3-none-any.whl

clitic/analyzer.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+import re
+import shutil
+import subprocess
+import time
+from dataclasses import dataclass, field
+
+TIMEOUT = 5.0  # seconds per probe run
+
+
+@dataclass
+class RunResult:
+    args: list[str]
+    exit_code: int
+    stdout: str
+    stderr: str
+    elapsed: float
+    timed_out: bool = False
+    error: str | None = None
+
+
+@dataclass
+class ToolProbe:
+    tool_name: str
+    tool_path: str | None
+
+    help_result: RunResult | None = None       # tool --help
+    version_result: RunResult | None = None    # tool --version
+    no_args_result: RunResult | None = None    # tool (no args)
+    bad_args_result: RunResult | None = None   # tool --xxxxclitictest (unknown flag)
+    json_results: list[RunResult] = field(default_factory=list)  # --json / --output json / etc.
+    subcommand_names: list[str] = field(default_factory=list)
+    subcommand_help_results: list[tuple[str, RunResult]] = field(default_factory=list)
+
+
+def _run(cmd: list[str]) -> RunResult:
+    t0 = time.monotonic()
+    try:
+        proc = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            timeout=TIMEOUT,
+        )
+        elapsed = time.monotonic() - t0
+        return RunResult(
+            args=cmd,
+            exit_code=proc.returncode,
+            stdout=proc.stdout,
+            stderr=proc.stderr,
+            elapsed=elapsed,
+        )
+    except subprocess.TimeoutExpired:
+        elapsed = time.monotonic() - t0
+        return RunResult(args=cmd, exit_code=-1, stdout="", stderr="", elapsed=elapsed, timed_out=True)
+    except FileNotFoundError as exc:
+        elapsed = time.monotonic() - t0
+        return RunResult(args=cmd, exit_code=-1, stdout="", stderr="", elapsed=elapsed, error=str(exc))
+    except Exception as exc:
+        elapsed = time.monotonic() - t0
+        return RunResult(args=cmd, exit_code=-1, stdout="", stderr="", elapsed=elapsed, error=str(exc))
+
+
+def _extract_subcommands(help_text: str) -> list[str]:
+    """Heuristically extract subcommand names from help text."""
+    lines = help_text.splitlines()
+
+    # Strategy 1: Named section headers (Commands:, Available Commands:, Subcommands:)
+    in_commands_section = False
+    section_subcommands: list[str] = []
+    skip_words = {"usage", "options", "arguments", "flags", "help", "version", "global", "topics"}
+
+    for line in lines:
+        stripped = line.strip()
+        if re.match(
+            r"^(commands?|subcommands?|available commands?|sub-commands?)\s*:?\s*$",
+            stripped,
+            re.IGNORECASE,
+        ):
+            in_commands_section = True
+            continue
+
+        if in_commands_section:
+            if not stripped:
+                if section_subcommands:
+                    break
+                continue
+            # New section header ends the commands section
+            if line and not line[0].isspace() and re.search(r":\s*$", stripped):
+                break
+            match = re.match(r"^\s{1,8}([a-z][\w-]{0,30})\b", line)
+            if match:
+                candidate = match.group(1)
+                if candidate.lower() not in skip_words:
+                    section_subcommands.append(candidate)
+
+    if section_subcommands:
+        return section_subcommands[:8]
+
+    # Strategy 2: Indented "word  Description" lines (git-style)
+    pattern_subcommands: list[str] = []
+    for line in lines:
+        match = re.match(r"^\s{2,8}([a-z][\w-]{1,20})\s{2,}[A-Za-z]", line)
+        if match:
+            candidate = match.group(1)
+            if candidate.lower() not in skip_words:
+                pattern_subcommands.append(candidate)
+
+    return pattern_subcommands[:8]
+
+
+def probe(tool_name: str) -> ToolProbe:
+    """Run all probes for a CLI tool and return a ToolProbe."""
+    tool_path = shutil.which(tool_name)
+    result = ToolProbe(tool_name=tool_name, tool_path=tool_path)
+
+    if tool_path is None:
+        return result  # tool not found; all results will be None
+
+    cmd_base = [tool_path]
+
+    # help probe
+    result.help_result = _run(cmd_base + ["--help"])
+    if result.help_result.exit_code != 0 and not result.help_result.error:
+        alt = _run(cmd_base + ["-h"])
+        if alt.exit_code == 0:
+            result.help_result = alt
+
+    # version probe
+    result.version_result = _run(cmd_base + ["--version"])
+    if result.version_result.exit_code != 0 and not result.version_result.error:
+        alt = _run(cmd_base + ["-V"])
+        if alt.exit_code == 0:
+            result.version_result = alt
+
+    # no-args probe
+    result.no_args_result = _run(cmd_base)
+
+    # bad args probe — unique flag an agent would never pass intentionally
+    result.bad_args_result = _run(cmd_base + ["--xxxxclitictest-unknown"])
+
+    # JSON output probes (try common patterns)
+    for json_flag in [["--json"], ["--output", "json"], ["-o", "json"], ["--format", "json"]]:
+        r = _run(cmd_base + json_flag)
+        if not r.error:
+            result.json_results.append(r)
+
+    # Subcommand discovery
+    help_text = (result.help_result.stdout or "") + (result.help_result.stderr or "")
+    result.subcommand_names = _extract_subcommands(help_text)
+
+    # Probe first few subcommands' help
+    for sub in result.subcommand_names[:3]:
+        sub_help = _run(cmd_base + [sub, "--help"])
+        result.subcommand_help_results.append((sub, sub_help))
+
+    return result

clitic/cli.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import shutil
+
+import typer
+from typing_extensions import Annotated
+
+from .analyzer import probe
+from .scorer import run
+from . import report as reporter
+
+clitic_app = typer.Typer(
+    name="clitic",
+    help="Score a CLI tool for AI-agent readiness — the CLI Intelligence & Compliance Tester.",
+    add_completion=False,
+)
+
+_DEMO_TOOL = "git"
+
+
+def _score_tool(tool_name: str, as_json: bool) -> None:
+    if not as_json:
+        tool_probe = reporter.animate_probe(
+            tool_name,
+            probe,
+            tool_name,
+        )
+    else:
+        tool_probe = probe(tool_name)
+
+    if tool_probe is None or tool_probe.tool_path is None:
+        typer.echo(f"Error: '{tool_name}' not found in PATH", err=True)
+        raise typer.Exit(1)
+
+    result = run(tool_probe)
+
+    if as_json:
+        reporter.print_json(result)
+    else:
+        reporter.print_report(result)
+
+    if result.overall_score < 60:
+        raise typer.Exit(2)
+
+
+@clitic_app.command()
+def score(
+    tool: Annotated[str, typer.Argument(help="CLI tool name or path (e.g. 'git', 'gh', 'curl')")],
+    json: Annotated[bool, typer.Option("--json", help="Output results as JSON")] = False,
+) -> None:
+    """Score a CLI tool for AI-agent readiness across 5 dimensions."""
+    _score_tool(tool, json)
+
+
+@clitic_app.command()
+def demo(
+    json: Annotated[bool, typer.Option("--json", help="Output results as JSON")] = False,
+) -> None:
+    """Score 'git' as a demo — see how a real-world CLI tool fares for AI agents."""
+    if shutil.which(_DEMO_TOOL) is None:
+        typer.echo(f"Error: '{_DEMO_TOOL}' not found in PATH — install git to run the demo.", err=True)
+        raise typer.Exit(1)
+    _score_tool(_DEMO_TOOL, json)

clitic/dimensions/arg_design.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import re
+
+from ..models import DimensionResult, Issue
+from ..analyzer import ToolProbe
+
+NAME = "Argument & Interface Design"
+
+
+def _strip_ansi(text: str) -> str:
+    return re.sub(r"\x1b\[[0-9;]*m", "", text)
+
+
+def score(probe: ToolProbe) -> DimensionResult:
+    issues: list[Issue] = []
+    total_score = 0.0
+
+    hr = probe.help_result
+    if hr is None:
+        return DimensionResult(name=NAME, score=0, issues=[
+            Issue(severity="error", message="Tool not found — cannot assess argument design", location="tool")
+        ])
+
+    help_text = _strip_ansi((hr.stdout or "") + (hr.stderr or ""))
+    help_lower = help_text.lower()
+
+    long_flags = re.findall(r"--[\w-]+", help_text)
+    short_flags = re.findall(r"(?<!\-)(?<!\w)-[a-zA-Z]\b", help_text)
+
+    # 1. GNU-style long flags (--flag) present — 25 pts
+    if len(long_flags) >= 2:
+        total_score += 25
+    elif len(long_flags) == 1:
+        total_score += 12
+        issues.append(Issue(
+            severity="warning",
+            message="Only one long flag detected — prefer GNU-style --flags for agent-readable invocations",
+            location="flags",
+        ))
+    else:
+        issues.append(Issue(
+            severity="warning",
+            message="No GNU-style long flags (--flag) detected — agents benefit from descriptive flag names",
+            location="flags",
+        ))
+
+    # 2. Flags are kebab-case, not camelCase — 20 pts
+    camel_flags = [f for f in long_flags if re.search(r"--[a-z]+[A-Z]", f)]
+    if not camel_flags:
+        total_score += 20
+    else:
+        issues.append(Issue(
+            severity="warning",
+            message=f"camelCase flags detected ({', '.join(camel_flags[:3])}) — prefer kebab-case (--my-flag) by convention",
+            location="flags",
+        ))
+
+    # 3. Standard flags present (--verbose, --help, --quiet) — 25 pts
+    standard = {
+        "--verbose / -v": ["--verbose", "-v ", "--debug"],
+        "--help / -h":    ["--help", "-h "],
+        "--quiet / -q":   ["--quiet", "-q ", "--silent"],
+    }
+    found = sum(
+        1 for variants in standard.values()
+        if any(v in help_lower for v in variants)
+    )
+    total_score += (found / len(standard)) * 25
+    if found < 2:
+        missing = [k for k, variants in standard.items() if not any(v in help_lower for v in variants)]
+        issues.append(Issue(
+            severity="info",
+            message=f"Missing standard flags: {', '.join(missing)} — standard flags aid agent discoverability",
+            location="flags",
+        ))
+
+    # 4. Both short and long flag forms — 15 pts
+    if long_flags and short_flags:
+        total_score += 15
+    elif long_flags:
+        total_score += 8
+        issues.append(Issue(
+            severity="info",
+            message="No short flag aliases (-x) detected — short aliases improve usability in agent-generated commands",
+            location="flags",
+        ))
+
+    # 5. Config / environment variable documentation — 15 pts
+    config_keywords = ["config", "env", "environment", ".env", "configuration", "settings", "$ "]
+    if any(kw in help_lower for kw in config_keywords):
+        total_score += 15
+    else:
+        issues.append(Issue(
+            severity="info",
+            message="No mention of config files or environment variables — agents benefit from knowing all configuration methods",
+            location="--help output",
+        ))
+
+    return DimensionResult(name=NAME, score=round(min(total_score, 100), 1), issues=issues)

clitic/dimensions/discoverability.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import re
+
+from ..models import DimensionResult, Issue
+from ..analyzer import ToolProbe
+
+NAME = "Help & Discoverability"
+
+
+def score(probe: ToolProbe) -> DimensionResult:
+    issues: list[Issue] = []
+    total_score = 0.0
+
+    hr = probe.help_result
+    if hr is None:
+        issues.append(Issue(severity="error", message="Tool not found in PATH", location="tool"))
+        return DimensionResult(name=NAME, score=0, issues=issues)
+
+    # 1. --help works (exit 0) — 30 pts
+    if hr.timed_out:
+        issues.append(Issue(
+            severity="error",
+            message="--help timed out — tool hangs on --help, agents cannot introspect it",
+            location="--help",
+        ))
+    elif hr.exit_code == 0:
+        total_score += 30
+    else:
+        combined = (hr.stdout or "") + (hr.stderr or "")
+        if len(combined.strip()) > 50:
+            total_score += 15  # help text exists but exit code is wrong
+            issues.append(Issue(
+                severity="warning",
+                message=f"--help exits {hr.exit_code} instead of 0 — agents may interpret this as failure",
+                location="--help",
+            ))
+        else:
+            issues.append(Issue(
+                severity="error",
+                message="--help produced no useful output or failed entirely",
+                location="--help",
+            ))
+
+    # 2. Help text is substantial — 20 pts
+    help_text = (hr.stdout or "") + (hr.stderr or "")
+    help_clean = re.sub(r"\x1b\[[0-9;]*m", "", help_text).strip()
+    if len(help_clean) >= 100:
+        total_score += 20
+    elif len(help_clean) >= 40:
+        total_score += 10
+        issues.append(Issue(
+            severity="warning",
+            message=f"Help text is very short ({len(help_clean)} chars) — agents need context to invoke the tool correctly",
+            location="--help output",
+        ))
+    else:
+        issues.append(Issue(
+            severity="error",
+            message="Help text is minimal or absent",
+            location="--help output",
+        ))
+
+    # 3. Subcommands listed — 20 pts
+    if probe.subcommand_names:
+        total_score += 20
+    else:
+        if len(help_clean) > 200:
+            total_score += 10  # likely a focused single-command tool
+            issues.append(Issue(
+                severity="info",
+                message="No subcommands detected — if this is a multi-command tool, explicit subcommand listing helps agents discover capabilities",
+                location="--help output",
+            ))
+        else:
+            issues.append(Issue(
+                severity="info",
+                message="No subcommands detected in help output",
+                location="--help output",
+            ))
+
+    # 4. --version works — 15 pts
+    vr = probe.version_result
+    if vr and vr.exit_code == 0 and (vr.stdout or vr.stderr).strip():
+        total_score += 15
+    elif vr and not vr.error:
+        issues.append(Issue(
+            severity="warning",
+            message="--version flag missing or non-functional — agents cannot verify tool version for compatibility checks",
+            location="--version",
+        ))
+
+    # 5. Subcommand --help works — 15 pts
+    if probe.subcommand_help_results:
+        working = sum(1 for _, r in probe.subcommand_help_results if r.exit_code == 0)
+        ratio = working / len(probe.subcommand_help_results)
+        total_score += 15 * ratio
+        if ratio < 1.0:
+            issues.append(Issue(
+                severity="warning",
+                message=f"{len(probe.subcommand_help_results) - working}/{len(probe.subcommand_help_results)} subcommands don't respond to --help",
+                location="<subcommand> --help",
+            ))
+    else:
+        total_score += 15  # no subcommands to test — not penalized
+
+    return DimensionResult(name=NAME, score=round(min(total_score, 100), 1), issues=issues)

clitic/dimensions/error_handling.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import re
+
+from ..models import DimensionResult, Issue
+from ..analyzer import ToolProbe
+
+NAME = "Error Handling"
+
+
+def _strip_ansi(text: str) -> str:
+    return re.sub(r"\x1b\[[0-9;]*m", "", text)
+
+
+def _looks_like_traceback(text: str) -> bool:
+    patterns = [
+        r"Traceback \(most recent call last\)",
+        r'^\s+File ".*", line \d+',
+        r"^\s+at \w+[\.\w]+ \(",  # JS/Java
+        r"\w+Error:",
+        r"Exception in thread",
+    ]
+    return any(re.search(p, text, re.MULTILINE) for p in patterns)
+
+
+def _error_is_informative(text: str) -> bool:
+    clean = _strip_ansi(text).lower().strip()
+    if not clean or len(clean) < 20:
+        return False
+    helpful_keywords = ["usage", "try", "see", "run", "--help", "expected", "unknown",
+                        "unrecognized", "invalid", "error:", "did you mean"]
+    return any(kw in clean for kw in helpful_keywords)
+
+
+def score(probe: ToolProbe) -> DimensionResult:
+    issues: list[Issue] = []
+    total_score = 0.0
+
+    br = probe.bad_args_result
+    if br is None or br.error:
+        return DimensionResult(name=NAME, score=50, issues=[
+            Issue(severity="info", message="Could not probe error handling — tool unavailable", location="error probe")
+        ])
+
+    stdout_clean = _strip_ansi(br.stdout or "")
+    stderr_clean = _strip_ansi(br.stderr or "")
+    combined = stdout_clean + stderr_clean
+
+    # 1. Error output goes to stderr — 30 pts
+    if br.exit_code != 0:
+        if stderr_clean.strip():
+            total_score += 30
+        elif stdout_clean.strip():
+            total_score += 15
+            issues.append(Issue(
+                severity="warning",
+                message="Error message printed to stdout instead of stderr — agents parsing stdout may misinterpret it as valid output",
+                location="stderr",
+            ))
+        else:
+            issues.append(Issue(
+                severity="warning",
+                message="Bad args produced no error output at all",
+                location="stderr",
+            ))
+
+    # 2. Error message is informative — 40 pts
+    if _error_is_informative(combined):
+        total_score += 40
+    elif combined.strip():
+        total_score += 20
+        issues.append(Issue(
+            severity="warning",
+            message="Error message for unknown flag is minimal — agents need clear guidance on what went wrong",
+            location="error message",
+        ))
+    else:
+        issues.append(Issue(
+            severity="error",
+            message="No error message for invalid flag — agents cannot diagnose the failure",
+            location="error message",
+        ))
+
+    # 3. No stack trace exposed — 30 pts
+    if not _looks_like_traceback(combined):
+        total_score += 30
+    else:
+        issues.append(Issue(
+            severity="error",
+            message="Stack trace exposed on invalid input — leaks internals and confuses agents parsing errors",
+            location="stderr",
+        ))
+
+    return DimensionResult(name=NAME, score=round(min(total_score, 100), 1), issues=issues)

clitic/dimensions/exit_codes.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from ..models import DimensionResult, Issue
+from ..analyzer import ToolProbe
+
+NAME = "Exit Code Semantics"
+
+
+def score(probe: ToolProbe) -> DimensionResult:
+    issues: list[Issue] = []
+    total_score = 0.0
+
+    # 1. --help exits 0 — 25 pts
+    hr = probe.help_result
+    if hr and not hr.timed_out and not hr.error:
+        if hr.exit_code == 0:
+            total_score += 25
+        else:
+            issues.append(Issue(
+                severity="warning",
+                message=f"--help exits {hr.exit_code} instead of 0 — agents interpret non-zero as failure",
+                location="--help exit code",
+            ))
+
+    # 2. --version exits 0 — 20 pts
+    vr = probe.version_result
+    if vr and not vr.timed_out and not vr.error:
+        if vr.exit_code == 0:
+            total_score += 20
+        else:
+            issues.append(Issue(
+                severity="warning",
+                message=f"--version exits {vr.exit_code} instead of 0",
+                location="--version exit code",
+            ))
+    else:
+        total_score += 10  # --version absent — partial credit, not a fatal flaw
+
+    # 3. Unknown flag exits non-zero — 35 pts
+    br = probe.bad_args_result
+    if br and not br.timed_out and not br.error:
+        if br.exit_code != 0:
+            total_score += 35
+        else:
+            issues.append(Issue(
+                severity="error",
+                message="Unknown flag '--xxxxclitictest-unknown' returned exit 0 — agents cannot detect invalid invocations",
+                location="bad args exit code",
+            ))
+
+    # 4. No-args behavior is intentional — 20 pts
+    nar = probe.no_args_result
+    if nar and not nar.timed_out and not nar.error:
+        combined = (nar.stdout or "") + (nar.stderr or "")
+        if combined.strip():
+            total_score += 20  # produces output (help or usage error) — intentional
+        elif nar.exit_code == 0:
+            total_score += 10
+            issues.append(Issue(
+                severity="info",
+                message="Running with no args produces no output and exits 0 — consider showing usage or help",
+                location="no-args exit code",
+            ))
+        else:
+            issues.append(Issue(
+                severity="warning",
+                message="No-args invocation is silent with non-zero exit — agents cannot tell what went wrong",
+                location="no-args exit code",
+            ))
+
+    return DimensionResult(name=NAME, score=round(min(total_score, 100), 1), issues=issues)