artie-cli 0.7.0__py3-none-any.whl

artie/__init__.py

@@ -0,0 +1,11 @@
+"""artie-cli: Score your API documentation for AI-readiness."""
+
+__version__ = "0.7.0"
+
+# Version of the scoring rubric: the thresholds, buckets, and weightings that
+# turn documentation into scores. Bump this whenever any of those change, so
+# JSON consumers can tell whether two runs are comparable. This is the runtime
+# source of truth; pyproject.toml [tool.artie] mirrors it for packaging tools,
+# and tests/test_scoring_version.py asserts the two stay in sync.
+# See docs/scoring.md for the full rubric.
+SCORING_VERSION = "1.0"

artie/__main__.py

@@ -0,0 +1,6 @@
+"""Allow running with `python -m artie`."""
+
+from artie.cli import app
+
+if __name__ == "__main__":
+    app()

artie/baseline.py

@@ -0,0 +1,71 @@
+"""Load a previously saved JSON report for run-over-run comparison.
+
+`artie check --output json > previous.json` produces a report; passing it
+back via `--baseline previous.json` on a later run lets the reporters show
+per-check score deltas, which is what turns artie into a regression gate.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+class BaselineError(Exception):
+    """Raised when a baseline report cannot be loaded or understood."""
+
+
+@dataclass
+class Baseline:
+    """Scores from a prior run, keyed by check name."""
+
+    scores: dict[str, int | None] = field(default_factory=dict)
+    scoring_version: str | None = None
+    source: str | None = None
+    path: Path | None = None
+
+    def score_for(self, check_name: str) -> int | None:
+        """Prior score for a check, or None if absent or unscored."""
+        return self.scores.get(check_name)
+
+    def delta_for(self, check_name: str, current: int | None) -> int | None:
+        """Signed change from the prior run, or None if not comparable."""
+        prior = self.scores.get(check_name)
+        if prior is None or current is None:
+            return None
+        return current - prior
+
+
+def load_baseline(path: str | Path) -> Baseline:
+    """Load a saved artie JSON report as a Baseline."""
+    path = Path(path)
+    if not path.is_file():
+        raise BaselineError(f"Baseline file not found: {path}")
+
+    try:
+        data: Any = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise BaselineError(f"Baseline {path} is not valid JSON: {exc}") from exc
+    except OSError as exc:
+        raise BaselineError(f"Could not read baseline {path}: {exc}") from exc
+
+    if not isinstance(data, dict) or not isinstance(data.get("checks"), list):
+        raise BaselineError(
+            f"Baseline {path} is not an artie JSON report "
+            f"(expected a top-level `checks` array)."
+        )
+
+    scores: dict[str, int | None] = {}
+    for check in data["checks"]:
+        if isinstance(check, dict) and isinstance(check.get("name"), str):
+            score = check.get("score")
+            scores[check["name"]] = score if isinstance(score, int) else None
+
+    return Baseline(
+        scores=scores,
+        scoring_version=data.get("scoring_version"),
+        source=data.get("source"),
+        path=path,
+    )

artie/checks/__init__.py

@@ -0,0 +1,23 @@
+"""Documentation check registry."""
+
+from artie.checks.auth_clarity import AuthClarityCheck
+from artie.checks.base import BaseCheck, CheckResult, Severity
+from artie.checks.endpoint_completeness import EndpointCompletenessCheck
+from artie.checks.error_documentation import ErrorDocumentationCheck
+from artie.checks.example_coverage import ExampleCoverageCheck
+from artie.checks.format_efficiency import FormatEfficiencyCheck
+from artie.checks.parameter_naming import ParameterNamingCheck
+from artie.checks.schema_complexity import SchemaComplexityCheck
+
+# Registry of all available checks, ordered roughly format -> content -> quality.
+ALL_CHECKS: list[type[BaseCheck]] = [
+    FormatEfficiencyCheck,
+    EndpointCompletenessCheck,
+    ExampleCoverageCheck,
+    ErrorDocumentationCheck,
+    AuthClarityCheck,
+    ParameterNamingCheck,
+    SchemaComplexityCheck,
+]
+
+__all__ = ["ALL_CHECKS", "BaseCheck", "CheckResult", "Severity"]

artie/checks/auth_clarity.py

@@ -0,0 +1,173 @@
+"""Auth Clarity check.
+
+Scores documentation based on how clearly authentication is defined and
+applied. Agents need to know what credentials to send, where to send them,
+and on which operations.
+
+Three subscores:
+
+1. Schemes defined: at least one security scheme exists in
+   components.securitySchemes.
+2. Schemes described: every scheme has a non-trivial description (or, for
+   schemes where the type is self-documenting like `http bearer`, the type
+   itself counts).
+3. Application coverage: either a top-level `security` requirement is set,
+   or every operation declares one. Operations that explicitly opt out with
+   `security: []` count as documented (they're public on purpose).
+"""
+
+from typing import Any
+
+from artie.checks.base import BaseCheck, CheckResult
+from artie.parsers.types import Endpoint, ParsedDocs
+
+
+SELF_DOCUMENTING_TYPES = {"http", "apiKey", "openIdConnect"}
+
+
+class AuthClarityCheck(BaseCheck):
+    name = "Auth Clarity"
+    description = "Security schemes defined, described, and applied to operations."
+
+    def run(
+        self, content: str, format_type: str, parsed: Any = None
+    ) -> CheckResult:
+        if not isinstance(parsed, ParsedDocs) or not parsed.is_structured:
+            return self.not_evaluable(
+                "Auth Clarity requires a parsed OpenAPI document."
+            )
+
+        schemes = _security_schemes(parsed.components)
+        endpoints = parsed.endpoints
+
+        if not schemes and not _global_security(parsed.raw) and not _any_operation_has_security(endpoints):
+            # No security mentioned anywhere. This is either a fully public
+            # API (legitimate) or undocumented auth (not legitimate). We can't
+            # tell from the spec alone, so flag it but do not score zero.
+            return CheckResult(
+                name=self.name,
+                description=self.description,
+                score=0,
+                max_score=self.max_score,
+                severity=self.severity_for(0),
+                findings=[
+                    "No security schemes defined and no operations declare security",
+                    "API appears to be fully unauthenticated, or auth is undocumented",
+                ],
+                recommendations=[
+                    "If this API requires authentication, declare a scheme under "
+                    "components.securitySchemes and reference it from operations "
+                    "(or globally). If it is genuinely public, add a top-level "
+                    "comment to make that explicit."
+                ],
+                metadata={"schemes": 0, "global_security": False},
+            )
+
+        scheme_count = len(schemes)
+        described_schemes = sum(1 for s in schemes.values() if _is_described(s))
+        global_security = _global_security(parsed.raw)
+
+        operation_coverage = _operation_coverage(endpoints, global_security)
+
+        scheme_defined_ratio = 1.0 if scheme_count > 0 else 0.0
+        described_ratio = (
+            described_schemes / scheme_count if scheme_count > 0 else 0.0
+        )
+        application_ratio = operation_coverage
+
+        composite = (scheme_defined_ratio + described_ratio + application_ratio) / 3
+        score = round(composite * self.max_score)
+        severity = self.severity_for(score)
+
+        scheme_names = ", ".join(schemes.keys()) if schemes else "none"
+        findings = [
+            f"{scheme_count} security scheme{'s' if scheme_count != 1 else ''} "
+            f"defined: {scheme_names}",
+            f"{described_schemes} of {scheme_count} schemes have a description "
+            f"or self-documenting type"
+            if scheme_count
+            else "No security schemes defined",
+            "Top-level security requirement present" if global_security
+            else "No top-level security requirement",
+            f"{int(application_ratio * 100)}% of operations have an explicit "
+            "or inherited security requirement",
+        ]
+
+        recommendations: list[str] = []
+        undescribed = [name for name, s in schemes.items() if not _is_described(s)]
+        if undescribed:
+            recommendations.append(
+                "Add a description to each security scheme explaining how to "
+                "obtain credentials and what they authorize. Missing on: "
+                + ", ".join(undescribed)
+            )
+        if scheme_count and application_ratio < 1.0:
+            recommendations.append(
+                "Set a top-level security requirement so every operation "
+                "inherits it by default, or declare security on every operation. "
+                "Operations that are genuinely public should opt out explicitly "
+                "with security: []."
+            )
+
+        return CheckResult(
+            name=self.name,
+            description=self.description,
+            score=score,
+            max_score=self.max_score,
+            severity=severity,
+            findings=findings,
+            recommendations=recommendations,
+            metadata={
+                "schemes": scheme_count,
+                "schemes_described": described_schemes,
+                "global_security": global_security,
+                "operation_coverage": application_ratio,
+            },
+        )
+
+
+def _security_schemes(components: dict[str, Any]) -> dict[str, dict[str, Any]]:
+    if not isinstance(components, dict):
+        return {}
+    schemes = components.get("securitySchemes")
+    if not isinstance(schemes, dict):
+        return {}
+    return {k: v for k, v in schemes.items() if isinstance(v, dict)}
+
+
+def _global_security(raw: dict[str, Any] | None) -> bool:
+    if not isinstance(raw, dict):
+        return False
+    security = raw.get("security")
+    return isinstance(security, list) and len(security) > 0
+
+
+def _any_operation_has_security(endpoints: list[Endpoint]) -> bool:
+    return any("security" in e.operation for e in endpoints)
+
+
+def _operation_coverage(endpoints: list[Endpoint], global_security: bool) -> float:
+    """Fraction of operations whose security posture is documented.
+
+    An operation is covered if it explicitly sets `security` (including the
+    empty list to opt out), or if a top-level security requirement applies.
+    """
+    if not endpoints:
+        return 0.0
+    covered = 0
+    for endpoint in endpoints:
+        if "security" in endpoint.operation:
+            covered += 1
+        elif global_security:
+            covered += 1
+    return covered / len(endpoints)
+
+
+def _is_described(scheme: dict[str, Any]) -> bool:
+    """A scheme is described if it has a non-trivial description, or if its
+    type is self-documenting (http bearer, apiKey, openIdConnect)."""
+    description = (scheme.get("description") or "").strip()
+    if len(description) >= 10:
+        return True
+    scheme_type = scheme.get("type")
+    return scheme_type in SELF_DOCUMENTING_TYPES

artie/checks/base.py

@@ -0,0 +1,93 @@
+"""Base class for all artie checks."""
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+
+class Severity(str, Enum):
+    """Severity levels for check results."""
+
+    EXCELLENT = "excellent"
+    GOOD = "good"
+    NEEDS_WORK = "needs_work"
+    POOR = "poor"
+    NOT_EVALUABLE = "not_evaluable"
+
+
+@dataclass
+class CheckResult:
+    """Result of running a single check against documentation."""
+
+    name: str
+    description: str
+    score: int | None
+    max_score: int
+    severity: Severity
+    findings: list[str] = field(default_factory=list)
+    recommendations: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+    # An informational check ran and produced findings, but deliberately
+    # assigns no score. Sample Generation is the only one: it presents
+    # generated code as a deliverable, not a 0-10 grade.
+    informational: bool = False
+
+    @property
+    def is_evaluable(self) -> bool:
+        """True when this check produced a numeric score worth gating on.
+
+        Excludes both not-evaluable results (the check couldn't run) and
+        informational results (the check ran but assigns no score).
+        """
+        return self.severity != Severity.NOT_EVALUABLE and not self.informational
+
+    @property
+    def score_display(self) -> str:
+        if self.informational:
+            return "—"
+        if self.severity == Severity.NOT_EVALUABLE:
+            return "N/A"
+        return f"{self.score}/{self.max_score}"
+
+
+class BaseCheck:
+    """Base class for all documentation checks."""
+
+    name: str = "unnamed check"
+    description: str = ""
+    max_score: int = 10
+
+    def run(self, content: str, format_type: str, parsed: Any = None) -> CheckResult:
+        """Run this check against documentation content.
+
+        Args:
+            content: Raw documentation text.
+            format_type: Detected format (e.g. "openapi-yaml", "markdown").
+            parsed: Optional pre-parsed representation of the content.
+
+        Returns:
+            CheckResult with score, severity, findings, and recommendations.
+        """
+        raise NotImplementedError
+
+    def not_evaluable(self, reason: str) -> CheckResult:
+        """Return a not-evaluable result when this check can't be assessed."""
+        return CheckResult(
+            name=self.name,
+            description=self.description,
+            score=None,
+            max_score=self.max_score,
+            severity=Severity.NOT_EVALUABLE,
+            findings=[reason],
+        )
+
+    def severity_for(self, score: int) -> Severity:
+        """Map a numeric score to a severity level."""
+        ratio = score / self.max_score
+        if ratio >= 0.9:
+            return Severity.EXCELLENT
+        if ratio >= 0.7:
+            return Severity.GOOD
+        if ratio >= 0.4:
+            return Severity.NEEDS_WORK
+        return Severity.POOR

artie/checks/endpoint_completeness.py

@@ -0,0 +1,138 @@
+"""Endpoint Completeness check.
+
+Scores documentation based on how completely each endpoint is described.
+Three signals contribute equally:
+
+1. Endpoints with a description or summary (agents need to know what an
+   endpoint does, not just its shape).
+2. Endpoints with an operationId (the canonical name agents use for tool
+   calling and code generation).
+3. Path parameters with descriptions (agents must know what {bookId}
+   actually identifies before constructing requests).
+"""
+
+from typing import Any
+
+from artie.checks.base import BaseCheck, CheckResult
+from artie.parsers.openapi import path_parameters
+from artie.parsers.types import Endpoint, ParsedDocs
+
+
+class EndpointCompletenessCheck(BaseCheck):
+    name = "Endpoint Completeness"
+    description = (
+        "Endpoints with descriptions, operationIds, and documented "
+        "path parameters."
+    )
+
+    def run(
+        self, content: str, format_type: str, parsed: Any = None
+    ) -> CheckResult:
+        if not isinstance(parsed, ParsedDocs) or not parsed.is_structured:
+            return self.not_evaluable(
+                "Endpoint Completeness requires a parsed OpenAPI document."
+            )
+
+        endpoints = parsed.endpoints
+        if not endpoints:
+            return self.not_evaluable("No endpoints found in paths section.")
+
+        described = [e for e in endpoints if _has_description(e)]
+        with_op_id = [e for e in endpoints if e.operation_id]
+        path_param_total, path_param_described = _count_path_params(endpoints)
+
+        # Three subscores out of 1.0, averaged, scaled to max_score.
+        desc_ratio = len(described) / len(endpoints)
+        op_id_ratio = len(with_op_id) / len(endpoints)
+        if path_param_total == 0:
+            # No path params is not a failure mode; collapse to two-component avg.
+            subscores = [desc_ratio, op_id_ratio]
+        else:
+            path_ratio = path_param_described / path_param_total
+            subscores = [desc_ratio, op_id_ratio, path_ratio]
+
+        composite = sum(subscores) / len(subscores)
+        score = round(composite * self.max_score)
+        severity = self.severity_for(score)
+
+        findings = [
+            f"{len(described)} of {len(endpoints)} endpoints have a description or summary",
+            f"{len(with_op_id)} of {len(endpoints)} endpoints have an operationId",
+        ]
+        if path_param_total > 0:
+            findings.append(
+                f"{path_param_described} of {path_param_total} path parameters "
+                f"have descriptions"
+            )
+
+        recommendations: list[str] = []
+        if len(described) < len(endpoints):
+            missing = [e.display for e in endpoints if not _has_description(e)][:5]
+            recommendations.append(
+                "Add a summary or description to every operation. Examples "
+                f"missing one: {', '.join(missing)}"
+                + ("..." if len(missing) == 5 else "")
+            )
+        if len(with_op_id) < len(endpoints):
+            missing = [e.display for e in endpoints if not e.operation_id][:5]
+            recommendations.append(
+                "Add an operationId to every operation. operationIds become "
+                "function names in generated SDKs and tool definitions. Missing "
+                f"on: {', '.join(missing)}"
+                + ("..." if len(missing) == 5 else "")
+            )
+        if path_param_total > 0 and path_param_described < path_param_total:
+            recommendations.append(
+                "Document every path parameter with a description. Without "
+                "descriptions, agents guess at what an ID identifies."
+            )
+
+        return CheckResult(
+            name=self.name,
+            description=self.description,
+            score=score,
+            max_score=self.max_score,
+            severity=severity,
+            findings=findings,
+            recommendations=recommendations,
+            metadata={
+                "endpoints": len(endpoints),
+                "endpoints_with_descriptions": len(described),
+                "endpoints_with_operation_ids": len(with_op_id),
+                "path_parameters_total": path_param_total,
+                "path_parameters_described": path_param_described,
+            },
+        )
+
+
+def _has_description(endpoint: Endpoint) -> bool:
+    """Either description or summary counts; agents read both."""
+    return bool((endpoint.description or "").strip() or (endpoint.summary or "").strip())
+
+
+def _count_path_params(endpoints: list[Endpoint]) -> tuple[int, int]:
+    """Count path parameters across all endpoints, and how many are described.
+
+    A path parameter is described when an operation declares it under
+    `parameters` with `in: path` and a non-empty description. We dedupe per
+    endpoint so the same {id} appearing in two operations counts twice
+    (it can have different documentation on each).
+    """
+    total = 0
+    described = 0
+    for endpoint in endpoints:
+        names_in_path = path_parameters(endpoint.path)
+        if not names_in_path:
+            continue
+        # Build a lookup of declared path parameters on this operation.
+        declared = {
+            p.get("name"): p
+            for p in endpoint.parameters
+            if isinstance(p, dict) and p.get("in") == "path"
+        }
+        for name in names_in_path:
+            total += 1
+            param = declared.get(name)
+            if param and (param.get("description") or "").strip():
+                described += 1
+    return total, described