aspice-eval 0.1.0__py3-none-any.whl

aspice_eval/__init__.py

@@ -0,0 +1,71 @@
+"""ASPICE evaluation engine — knowledge base, evaluator, and reports."""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+# --- Core classes ---
+from aspice_eval.knowledge_base import KnowledgeBase
+from aspice_eval.evaluator import GapAnalysisEvaluator
+from aspice_eval.report_renderer import ReportRenderer
+
+# --- Factory & registry functions ---
+from aspice_eval.providers import create_evaluator, register_evaluator
+from aspice_eval.report_renderer import register_renderer
+from aspice_eval.knowledge_base import register_kb_loader
+
+# --- Convenience functions ---
+from aspice_eval.convenience import evaluate_sdp, validate_kb
+
+# --- Models ---
+from aspice_eval.models import (
+    ModelConfig,
+    EvaluationConfig,
+    EvaluationResult,
+    CriteriaEntry,
+    CriteriaRating,
+    SDPDocument,
+    CapabilityLevelResult,
+    ValidationResult,
+)
+
+# --- Exceptions ---
+from aspice_eval.exceptions import (
+    KBValidationError,
+    UnsupportedFormatError,
+    InvalidConfigError,
+    AIModelError,
+    AIResponseParseError,
+)
+
+__all__ = [
+    # Version
+    "__version__",
+    # Core classes
+    "KnowledgeBase",
+    "GapAnalysisEvaluator",
+    "ReportRenderer",
+    # Factory & registry
+    "create_evaluator",
+    "register_evaluator",
+    "register_renderer",
+    "register_kb_loader",
+    # Convenience functions
+    "evaluate_sdp",
+    "validate_kb",
+    # Models
+    "ModelConfig",
+    "EvaluationConfig",
+    "EvaluationResult",
+    "CriteriaEntry",
+    "CriteriaRating",
+    "SDPDocument",
+    "CapabilityLevelResult",
+    "ValidationResult",
+    # Exceptions
+    "KBValidationError",
+    "UnsupportedFormatError",
+    "InvalidConfigError",
+    "AIModelError",
+    "AIResponseParseError",
+]

aspice_eval/cli.py

@@ -0,0 +1,343 @@
+"""Click-based CLI entry point for the ASPICE evaluation tool.
+
+Provides three commands:
+
+- ``evaluate`` — Run a gap analysis of an SDP document against the KB.
+- ``validate-kb`` — Validate the knowledge base schema and completeness.
+- ``version`` — Print the package version.
+
+The entry point is ``aspice_eval.cli:main`` (configured in pyproject.toml).
+
+Requirements: 3.3, 7.1, 7.2, 7.3, 9.3
+"""
+
+from __future__ import annotations
+
+import os
+import pathlib
+import sys
+
+import click
+import yaml
+
+from aspice_eval import __version__
+from aspice_eval.exceptions import (
+    AIModelError,
+    AIResponseParseError,
+    InvalidConfigError,
+    KBValidationError,
+    UnsupportedFormatError,
+)
+
+
+def _validate_config(
+    target_level: int,
+    groups: list[str],
+    kb_path: str,
+) -> None:
+    """Validate configuration parameters early (fail fast).
+
+    Raises
+    ------
+    InvalidConfigError
+        If *target_level* is outside 1–5 or *groups* contains unknown codes.
+    FileNotFoundError
+        If *kb_path* does not exist.
+    """
+    # Validate target level
+    if target_level < 1 or target_level > 5:
+        raise InvalidConfigError(
+            f"Target level {target_level} is out of range. Must be 1–5.",
+            parameter="target_level",
+            actual_value=target_level,
+            expected_values=[1, 2, 3, 4, 5],
+        )
+
+    # Validate KB path exists
+    kb = pathlib.Path(kb_path)
+    if not kb.exists():
+        raise FileNotFoundError(
+            f"Knowledge base path does not exist: {kb_path}"
+        )
+
+    # Load metadata to get valid process group codes
+    metadata_path = kb / "aspice" / "_metadata.yaml"
+    if metadata_path.exists():
+        with open(metadata_path) as fh:
+            metadata = yaml.safe_load(fh)
+        valid_codes = {
+            pg.get("code", "")
+            for pg in metadata.get("process_groups", [])
+        }
+        unknown = [g for g in groups if g not in valid_codes]
+        if unknown:
+            raise InvalidConfigError(
+                f"Unknown process group(s): {', '.join(unknown)}. "
+                f"Valid groups: {', '.join(sorted(valid_codes))}.",
+                parameter="process_groups",
+                actual_value=unknown,
+                expected_values=sorted(valid_codes),
+            )
+
+
+@click.group()
+def main() -> None:
+    """ASPICE evaluation tool for SDP gap analysis."""
+
+
+@main.command()
+@click.option(
+    "--sdp",
+    required=True,
+    type=click.Path(),
+    help="Path to the SDP Markdown document.",
+)
+@click.option(
+    "--target-level",
+    default=3,
+    type=int,
+    show_default=True,
+    help="Target ASPICE capability level (1–5).",
+)
+@click.option(
+    "--groups",
+    default="SWE,SYS,MAN,SUP",
+    show_default=True,
+    help="Comma-separated process group codes to evaluate.",
+)
+@click.option(
+    "--output",
+    default=None,
+    type=click.Path(),
+    help="Output file path for the report. Defaults to stdout.",
+)
+@click.option(
+    "--kb-path",
+    default="knowledge_base",
+    show_default=True,
+    type=click.Path(),
+    help="Path to the knowledge base directory.",
+)
+@click.option(
+    "--model",
+    default=None,
+    type=str,
+    help="AI model name (provider-specific model identifier).",
+)
+@click.option(
+    "--provider",
+    default=None,
+    type=str,
+    help="AI provider name: bedrock, openai, anthropic, mock. "
+    "Defaults to ASPICE_EVAL_PROVIDER env var, then 'mock'.",
+)
+@click.option(
+    "--region",
+    default=None,
+    type=str,
+    help="AWS region for Bedrock provider (default us-east-1).",
+)
+@click.option(
+    "--output-format",
+    "output_format",
+    default="markdown",
+    type=click.Choice(["markdown", "html"], case_sensitive=False),
+    show_default=True,
+    help="Report output format.",
+)
+def evaluate(
+    sdp: str,
+    target_level: int,
+    groups: str,
+    output: str | None,
+    kb_path: str,
+    model: str | None,
+    provider: str | None,
+    region: str | None,
+    output_format: str,
+) -> None:
+    """Evaluate an SDP document against ASPICE criteria."""
+    process_groups = [g.strip() for g in groups.split(",") if g.strip()]
+
+    # Resolve provider: CLI flag > env var > default "mock"
+    resolved_provider = (
+        provider
+        or os.environ.get("ASPICE_EVAL_PROVIDER")
+        or "mock"
+    )
+
+    # Resolve model: CLI flag > env var > provider default
+    _model_defaults: dict[str, str] = {
+        "bedrock": "us.anthropic.claude-sonnet-4-20250514-v1:0",
+        "openai": "gpt-4o",
+        "anthropic": "claude-sonnet-4-20250514",
+        "mock": "",
+    }
+    resolved_model = (
+        model
+        or os.environ.get("ASPICE_EVAL_MODEL")
+        or _model_defaults.get(resolved_provider, "")
+    )
+
+    # Resolve temperature: env var (CLI doesn't expose this directly)
+    resolved_temperature = float(
+        os.environ.get("ASPICE_EVAL_TEMPERATURE", "0.0")
+    )
+
+    # Resolve max_tokens: env var
+    resolved_max_tokens = int(
+        os.environ.get("ASPICE_EVAL_MAX_TOKENS", "4096")
+    )
+
+    # Resolve region: CLI flag > env var > default ""
+    resolved_region = (
+        region
+        or os.environ.get("AWS_REGION")
+        or ""
+    )
+
+    try:
+        # --- Fail-fast validation ---
+        _validate_config(target_level, process_groups, kb_path)
+
+        # Check SDP path exists before doing anything else
+        sdp_path = pathlib.Path(sdp)
+        if not sdp_path.exists():
+            raise FileNotFoundError(f"SDP document not found: {sdp}")
+
+        # --- Wire components ---
+        from aspice_eval.knowledge_base import KnowledgeBase
+        from aspice_eval.level_calculator import CapabilityLevelCalculator
+        from aspice_eval.models import EvaluationConfig, ModelConfig
+        from aspice_eval.providers import create_evaluator
+        from aspice_eval.report_generator import ReportGenerator
+        from aspice_eval.sdp_ingester import SDPIngester
+
+        # 1. Load KB
+        kb = KnowledgeBase(kb_path)
+        kb.load("aspice")
+        kb_metadata = kb.get_metadata()
+
+        # 2. Ingest SDP
+        ingester = SDPIngester()
+        sdp_doc = ingester.ingest(sdp)
+
+        # 3. Get criteria for requested groups and level
+        criteria = kb.get_criteria(process_groups, target_level)
+
+        # 4. Build config
+        config = EvaluationConfig(
+            sdp_path=sdp,
+            target_capability_level=target_level,
+            process_groups=process_groups,
+            kb_path=kb_path,
+            output_path=output,
+        )
+
+        # 5. Evaluate using provider factory
+        model_config = ModelConfig(
+            provider=resolved_provider,
+            model_name=resolved_model,
+            temperature=resolved_temperature,
+            max_tokens=resolved_max_tokens,
+            region=resolved_region,
+        )
+        evaluator = create_evaluator(model_config)
+        evaluation = evaluator.evaluate(sdp_doc, criteria, config)
+
+        # 6. Calculate capability levels
+        calculator = CapabilityLevelCalculator(target_level)
+        levels = calculator.calculate(evaluation.ratings, process_groups)
+
+        # 7. Generate report
+        reporter = ReportGenerator()
+        report = reporter.generate(evaluation, levels, config, kb_metadata, output_format=output_format)
+
+        # 8. Output
+        if output:
+            pathlib.Path(output).write_text(report, encoding="utf-8")
+            click.echo(f"Report written to {output}")
+        else:
+            click.echo(report)
+
+    except FileNotFoundError as exc:
+        click.echo(f"Error: {exc}", err=True)
+        sys.exit(1)
+    except UnsupportedFormatError as exc:
+        click.echo(
+            f"Error: {exc}",
+            err=True,
+        )
+        sys.exit(1)
+    except KBValidationError as exc:
+        click.echo(f"KB validation error: {exc}", err=True)
+        sys.exit(1)
+    except InvalidConfigError as exc:
+        click.echo(f"Configuration error: {exc}", err=True)
+        sys.exit(1)
+    except AIModelError as exc:
+        click.echo(f"AI model error: {exc}", err=True)
+        sys.exit(1)
+    except AIResponseParseError as exc:
+        click.echo(f"AI response parse error: {exc}", err=True)
+        sys.exit(1)
+
+
+@main.command("validate-kb")
+@click.option(
+    "--kb-path",
+    default="knowledge_base",
+    show_default=True,
+    type=click.Path(),
+    help="Path to the knowledge base directory.",
+)
+def validate_kb(kb_path: str) -> None:
+    """Validate the knowledge base schema and completeness."""
+    try:
+        kb_dir = pathlib.Path(kb_path)
+        if not kb_dir.exists():
+            raise FileNotFoundError(
+                f"Knowledge base path does not exist: {kb_path}"
+            )
+
+        from aspice_eval.knowledge_base import KnowledgeBase
+
+        kb = KnowledgeBase(kb_path)
+        kb.load("aspice")
+        result = kb.validate()
+
+        if result.is_valid:
+            click.echo("Knowledge base validation passed.")
+        else:
+            click.echo("Knowledge base validation FAILED.", err=True)
+
+        if result.schema_errors:
+            click.echo("\nSchema errors:", err=True)
+            for err in result.schema_errors:
+                click.echo(f"  - {err}", err=True)
+
+        if result.completeness_gaps:
+            click.echo("\nCompleteness gaps:", err=True)
+            for gap in result.completeness_gaps:
+                click.echo(f"  - {gap}", err=True)
+
+        if result.warnings:
+            click.echo("\nWarnings:")
+            for warn in result.warnings:
+                click.echo(f"  - {warn}")
+
+        if not result.is_valid:
+            sys.exit(1)
+
+    except FileNotFoundError as exc:
+        click.echo(f"Error: {exc}", err=True)
+        sys.exit(1)
+    except KBValidationError as exc:
+        click.echo(f"KB validation error: {exc}", err=True)
+        sys.exit(1)
+
+
+@main.command()
+def version() -> None:
+    """Print the package version."""
+    click.echo(f"aspice-eval {__version__}")

aspice_eval/convenience.py

@@ -0,0 +1,218 @@
+"""High-level convenience functions for the ASPICE evaluation engine.
+
+Provides :func:`evaluate_sdp` and :func:`validate_kb` — single-call
+entry points that orchestrate the full evaluation and validation
+pipelines without requiring users to wire together multiple classes.
+
+Requirements: 11.1–11.6, 12.1–12.5, 22.6
+"""
+
+from __future__ import annotations
+
+import pathlib
+from typing import Any
+
+from aspice_eval.knowledge_base import KnowledgeBase, get_kb_loader
+from aspice_eval.level_calculator import CapabilityLevelCalculator
+from aspice_eval.models import (
+    EvaluationConfig,
+    EvaluationResult,
+    ModelConfig,
+    ValidationResult,
+)
+from aspice_eval.providers import create_evaluator
+from aspice_eval.sdp_ingester import SDPIngester
+
+
+def _resolve_default_kb_path() -> str:
+    """Resolve the bundled knowledge base path.
+
+    Searches package-relative and repository-relative locations for the
+    ``knowledge_base`` directory shipped with the package.
+
+    Returns
+    -------
+    str
+        Absolute path to the bundled knowledge base directory.
+
+    Raises
+    ------
+    FileNotFoundError
+        If the bundled knowledge base cannot be located.
+    """
+    pkg_root = pathlib.Path(__file__).resolve().parent
+    candidates = [
+        pkg_root / "knowledge_base",
+        pkg_root.parent.parent / "knowledge_base",
+    ]
+    for candidate in candidates:
+        if candidate.exists():
+            return str(candidate)
+    raise FileNotFoundError(
+        "Bundled knowledge base not found. Provide an explicit kb_path."
+    )
+
+
+def evaluate_sdp(
+    sdp_path: str,
+    model_config: ModelConfig,
+    *,
+    target_level: int = 3,
+    process_groups: list[str] | None = None,
+    kb_path: str | None = None,
+    standard: str = "aspice",
+) -> EvaluationResult:
+    """Evaluate an SDP document against knowledge base criteria.
+
+    Orchestrates the full evaluation pipeline: SDP ingestion → KB loading →
+    criteria filtering → AI evaluation → capability level calculation.
+
+    Parameters
+    ----------
+    sdp_path:
+        Path to the SDP Markdown file.
+    model_config:
+        AI model configuration (provider, model name, temperature, etc.).
+    target_level:
+        Target ASPICE capability level (1–5). Defaults to 3 (Established).
+    process_groups:
+        Process groups to evaluate. Defaults to ``["SWE", "SYS", "MAN", "SUP"]``.
+    kb_path:
+        Path to the knowledge base directory. Defaults to the bundled KB.
+    standard:
+        Standard identifier (subdirectory name under kb_path).
+        Defaults to ``"aspice"``.
+
+    Returns
+    -------
+    EvaluationResult
+        Contains per-criteria ratings, capability levels, and token usage.
+
+    Raises
+    ------
+    FileNotFoundError
+        If ``sdp_path`` or ``kb_path`` does not exist.
+    UnsupportedFormatError
+        If the SDP file is not Markdown format.
+    InvalidConfigError
+        If ``target_level`` is outside 1–5 or ``process_groups`` contains
+        unknown codes.
+    AIModelError
+        If the AI model call fails after retries.
+
+    Examples
+    --------
+    >>> from aspice_eval.convenience import evaluate_sdp
+    >>> from aspice_eval.models import ModelConfig
+    >>> result = evaluate_sdp(
+    ...     "docs/sdp.md",
+    ...     ModelConfig(provider="bedrock", model_name="us.anthropic.claude-sonnet-4-20250514-v1:0", region="us-east-1"),
+    ...     target_level=3,
+    ...     process_groups=["SWE", "SYS"],
+    ... )
+    >>> print(f"Gaps found: {len([r for r in result.ratings if r.gaps])}")
+    """
+    # Validate sdp_path exists
+    sdp = pathlib.Path(sdp_path)
+    if not sdp.exists():
+        raise FileNotFoundError(f"SDP file does not exist: {sdp_path}")
+
+    # Resolve kb_path
+    resolved_kb_path = kb_path if kb_path is not None else _resolve_default_kb_path()
+    if not pathlib.Path(resolved_kb_path).exists():
+        raise FileNotFoundError(
+            f"Knowledge base path does not exist: {resolved_kb_path}"
+        )
+
+    # Default process groups
+    groups = process_groups if process_groups is not None else ["SWE", "SYS", "MAN", "SUP"]
+
+    # Check for custom KB loader in the registry
+    custom_loader_cls = get_kb_loader(standard)
+    if custom_loader_cls is not None:
+        kb = custom_loader_cls(resolved_kb_path)
+    else:
+        kb = KnowledgeBase(resolved_kb_path)
+
+    kb.load(standard)
+    criteria = kb.get_criteria(groups, target_level)
+
+    # Ingest SDP
+    ingester = SDPIngester()
+    sdp_doc = ingester.ingest(sdp_path)
+
+    # Build evaluation config
+    config = EvaluationConfig(
+        sdp_path=sdp_path,
+        target_capability_level=target_level,
+        process_groups=groups,
+        kb_path=resolved_kb_path,
+        standard=standard,
+    )
+
+    # Create evaluator and run evaluation
+    evaluator = create_evaluator(model_config)
+    evaluation = evaluator.evaluate(sdp_doc, criteria, config)
+
+    # Calculate capability levels for metadata enrichment
+    calculator = CapabilityLevelCalculator(target_level)
+    levels = calculator.calculate(evaluation.ratings, groups)
+
+    # Attach capability levels to evaluation metadata
+    evaluation.sdp_metadata["capability_levels"] = {
+        group: {
+            "achieved_level": result.achieved_level,
+            "target_level": result.target_level,
+            "blocking_attributes": result.blocking_attributes,
+        }
+        for group, result in levels.items()
+    }
+
+    return evaluation
+
+
+def validate_kb(
+    kb_path: str,
+    *,
+    standard: str = "aspice",
+) -> ValidationResult:
+    """Validate a knowledge base directory for schema and completeness.
+
+    Loads the specified standard from the knowledge base directory and
+    runs schema validation and completeness checks against the bundled
+    criteria JSON Schema.
+
+    Parameters
+    ----------
+    kb_path:
+        Path to the knowledge base root directory.
+    standard:
+        Standard identifier to validate. Defaults to ``"aspice"``.
+
+    Returns
+    -------
+    ValidationResult
+        Contains ``is_valid`` flag, ``schema_errors``, ``completeness_gaps``,
+        and ``warnings``.
+
+    Raises
+    ------
+    FileNotFoundError
+        If ``kb_path`` does not exist.
+
+    Examples
+    --------
+    >>> from aspice_eval.convenience import validate_kb
+    >>> result = validate_kb("knowledge_base")
+    >>> if not result.is_valid:
+    ...     for error in result.schema_errors:
+    ...         print(f"Schema error: {error}")
+    """
+    if not pathlib.Path(kb_path).exists():
+        raise FileNotFoundError(
+            f"Knowledge base path does not exist: {kb_path}"
+        )
+
+    kb = KnowledgeBase(kb_path)
+    kb.load(standard)
+    return kb.validate()