aspice-eval 0.1.0__tar.gz

aspice_eval-0.1.0/MANIFEST.in

@@ -0,0 +1,2 @@
+recursive-include src/aspice_eval/knowledge_base *.yaml *.json
+recursive-include knowledge_base *.yaml *.json

aspice_eval-0.1.0/PKG-INFO

@@ -0,0 +1,234 @@
+Metadata-Version: 2.4
+Name: aspice-eval
+Version: 0.1.0
+Summary: ASPICE Knowledge Base & Agent Workflow evaluation tool for SDP gap analysis
+Author: ASPICE Eval Contributors
+License: MIT
+Keywords: aspice,automotive,compliance,gap-analysis,spice
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: jsonschema>=4.20.0
+Requires-Dist: click>=8.1.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == "dev"
+Requires-Dist: hypothesis>=6.90.0; extra == "dev"
+Provides-Extra: bedrock
+Requires-Dist: boto3>=1.34.0; extra == "bedrock"
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == "openai"
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.30.0; extra == "anthropic"
+Provides-Extra: all
+Requires-Dist: boto3>=1.34.0; extra == "all"
+Requires-Dist: openai>=1.0.0; extra == "all"
+Requires-Dist: anthropic>=0.30.0; extra == "all"
+
+# aspice-eval
+
+ASPICE evaluation engine — knowledge base, gap analysis evaluator, and reports.
+
+## Library Usage
+
+### Evaluate an SDP
+
+```python
+from aspice_eval import evaluate_sdp, ModelConfig
+
+result = evaluate_sdp(
+    "docs/sdp.md",
+    ModelConfig(
+        provider="bedrock",
+        model_name="us.anthropic.claude-sonnet-4-20250514-v1:0",
+        region="us-east-1",
+    ),
+    target_level=3,
+    process_groups=["SWE", "SYS"],
+)
+
+print(f"Criteria assessed: {len(result.ratings)}")
+print(f"Gaps found: {len([r for r in result.ratings if r.gaps])}")
+print(f"Tokens used: {result.token_usage['total_tokens']}")
+```
+
+### Validate a Knowledge Base
+
+```python
+from aspice_eval import validate_kb
+
+result = validate_kb("knowledge_base")
+
+if not result.is_valid:
+    for error in result.schema_errors:
+        print(f"Schema error: {error}")
+    for gap in result.completeness_gaps:
+        print(f"Completeness gap: {gap}")
+else:
+    print("Knowledge base is valid")
+```
+
+## Extension Points
+
+### Custom Evaluator
+
+Subclass `GapAnalysisEvaluator` to plug in a custom LLM provider or rule-based engine:
+
+```python
+from aspice_eval import GapAnalysisEvaluator, ModelConfig, register_evaluator
+
+class LocalLlamaEvaluator(GapAnalysisEvaluator):
+    """Evaluator using a local Llama model."""
+
+    def _call_model(self, prompt: str) -> str:
+        # Call your local model and return JSON response
+        ...
+
+# Register the custom provider
+register_evaluator("local-llama", LocalLlamaEvaluator)
+
+# Use it via the standard factory
+from aspice_eval import create_evaluator
+
+evaluator = create_evaluator(ModelConfig(provider="local-llama"))
+```
+
+### Custom Knowledge Base Standards
+
+Three levels of extensibility for non-ASPICE standards:
+
+**Level 1 — Custom YAML files (no code required):**
+
+Drop a new subdirectory under the KB root with YAML files conforming to the criteria schema:
+
+```
+knowledge_base/
+├── aspice/          # Built-in ASPICE v4.0
+└── iso26262/        # Your custom standard
+    ├── _metadata.yaml
+    └── functional_safety.yaml
+```
+
+```python
+from aspice_eval import evaluate_sdp, ModelConfig
+
+result = evaluate_sdp(
+    "docs/sdp.md",
+    ModelConfig(provider="bedrock", model_name="...", region="us-east-1"),
+    standard="iso26262",
+)
+```
+
+**Level 2 — In-memory construction via `from_dict`:**
+
+```python
+from aspice_eval import KnowledgeBase
+
+kb = KnowledgeBase.from_dict({
+    "processes": [
+        {
+            "process_id": "SWE.1",
+            "process_name": "Software Requirements Analysis",
+            "criteria": [...],
+        }
+    ]
+})
+criteria = kb.get_criteria(groups=["SWE"], max_level=3)
+```
+
+**Level 3 — Custom KB loader (pluggable schema):**
+
+For standards with fundamentally different structures, subclass `KnowledgeBase` and register a loader:
+
+```python
+from aspice_eval import KnowledgeBase, register_kb_loader, CriteriaEntry
+
+class NISTCSFKnowledgeBase(KnowledgeBase):
+    """Custom loader for NIST Cybersecurity Framework."""
+
+    def load(self, standard: str) -> None:
+        # Read NIST-shaped YAML/JSON, convert to CriteriaEntry list
+        ...
+
+    def get_criteria(self, groups, max_level) -> list[CriteriaEntry]:
+        # Return entries filtered by NIST "Functions" instead of ASPICE groups
+        ...
+
+register_kb_loader("nist-csf", NISTCSFKnowledgeBase)
+```
+
+### Custom Report Renderer
+
+Subclass `ReportRenderer` to output evaluation results in formats beyond Markdown and HTML:
+
+```python
+from aspice_eval import ReportRenderer, register_renderer
+from aspice_eval import EvaluationResult, CapabilityLevelResult, EvaluationConfig
+
+class JSONReportRenderer(ReportRenderer):
+    """Render evaluation results as JSON."""
+
+    def render(self, evaluation, levels, config, kb_metadata) -> str:
+        import json
+        return json.dumps({
+            "ratings": [
+                {"criteria_id": r.criteria_id, "rating": r.rating, "gaps": r.gaps}
+                for r in evaluation.ratings
+            ],
+        }, indent=2)
+
+# Register and use
+register_renderer("json", JSONReportRenderer)
+```
+
+## CLI Usage
+
+### aspice-eval evaluate
+
+```bash
+# Evaluate an SDP document
+aspice-eval evaluate --sdp path/to/sdp.md --target-level 2 --groups SWE,MAN
+
+# Write report to a file
+aspice-eval evaluate --sdp path/to/sdp.md --output report.md
+
+# Use a specific AI provider
+aspice-eval evaluate --sdp path/to/sdp.md --provider bedrock \
+  --model us.anthropic.claude-sonnet-4-20250514-v1:0 --region us-east-1
+```
+
+### aspice-eval validate-kb
+
+```bash
+# Validate the default knowledge base
+aspice-eval validate-kb
+
+# Validate a custom knowledge base
+aspice-eval validate-kb --kb-path /path/to/knowledge_base
+```
+
+## Installation
+
+```bash
+pip install aspice-eval
+
+# With AI provider support
+pip install "aspice-eval[bedrock]"     # Amazon Bedrock (Claude)
+pip install "aspice-eval[openai]"      # OpenAI GPT-4o
+pip install "aspice-eval[anthropic]"   # Anthropic Claude (direct API)
+pip install "aspice-eval[all]"         # All providers
+```
+
+Requires Python 3.10+.
+
+## License
+
+MIT

aspice_eval-0.1.0/README.md

@@ -0,0 +1,199 @@
+# aspice-eval
+
+ASPICE evaluation engine — knowledge base, gap analysis evaluator, and reports.
+
+## Library Usage
+
+### Evaluate an SDP
+
+```python
+from aspice_eval import evaluate_sdp, ModelConfig
+
+result = evaluate_sdp(
+    "docs/sdp.md",
+    ModelConfig(
+        provider="bedrock",
+        model_name="us.anthropic.claude-sonnet-4-20250514-v1:0",
+        region="us-east-1",
+    ),
+    target_level=3,
+    process_groups=["SWE", "SYS"],
+)
+
+print(f"Criteria assessed: {len(result.ratings)}")
+print(f"Gaps found: {len([r for r in result.ratings if r.gaps])}")
+print(f"Tokens used: {result.token_usage['total_tokens']}")
+```
+
+### Validate a Knowledge Base
+
+```python
+from aspice_eval import validate_kb
+
+result = validate_kb("knowledge_base")
+
+if not result.is_valid:
+    for error in result.schema_errors:
+        print(f"Schema error: {error}")
+    for gap in result.completeness_gaps:
+        print(f"Completeness gap: {gap}")
+else:
+    print("Knowledge base is valid")
+```
+
+## Extension Points
+
+### Custom Evaluator
+
+Subclass `GapAnalysisEvaluator` to plug in a custom LLM provider or rule-based engine:
+
+```python
+from aspice_eval import GapAnalysisEvaluator, ModelConfig, register_evaluator
+
+class LocalLlamaEvaluator(GapAnalysisEvaluator):
+    """Evaluator using a local Llama model."""
+
+    def _call_model(self, prompt: str) -> str:
+        # Call your local model and return JSON response
+        ...
+
+# Register the custom provider
+register_evaluator("local-llama", LocalLlamaEvaluator)
+
+# Use it via the standard factory
+from aspice_eval import create_evaluator
+
+evaluator = create_evaluator(ModelConfig(provider="local-llama"))
+```
+
+### Custom Knowledge Base Standards
+
+Three levels of extensibility for non-ASPICE standards:
+
+**Level 1 — Custom YAML files (no code required):**
+
+Drop a new subdirectory under the KB root with YAML files conforming to the criteria schema:
+
+```
+knowledge_base/
+├── aspice/          # Built-in ASPICE v4.0
+└── iso26262/        # Your custom standard
+    ├── _metadata.yaml
+    └── functional_safety.yaml
+```
+
+```python
+from aspice_eval import evaluate_sdp, ModelConfig
+
+result = evaluate_sdp(
+    "docs/sdp.md",
+    ModelConfig(provider="bedrock", model_name="...", region="us-east-1"),
+    standard="iso26262",
+)
+```
+
+**Level 2 — In-memory construction via `from_dict`:**
+
+```python
+from aspice_eval import KnowledgeBase
+
+kb = KnowledgeBase.from_dict({
+    "processes": [
+        {
+            "process_id": "SWE.1",
+            "process_name": "Software Requirements Analysis",
+            "criteria": [...],
+        }
+    ]
+})
+criteria = kb.get_criteria(groups=["SWE"], max_level=3)
+```
+
+**Level 3 — Custom KB loader (pluggable schema):**
+
+For standards with fundamentally different structures, subclass `KnowledgeBase` and register a loader:
+
+```python
+from aspice_eval import KnowledgeBase, register_kb_loader, CriteriaEntry
+
+class NISTCSFKnowledgeBase(KnowledgeBase):
+    """Custom loader for NIST Cybersecurity Framework."""
+
+    def load(self, standard: str) -> None:
+        # Read NIST-shaped YAML/JSON, convert to CriteriaEntry list
+        ...
+
+    def get_criteria(self, groups, max_level) -> list[CriteriaEntry]:
+        # Return entries filtered by NIST "Functions" instead of ASPICE groups
+        ...
+
+register_kb_loader("nist-csf", NISTCSFKnowledgeBase)
+```
+
+### Custom Report Renderer
+
+Subclass `ReportRenderer` to output evaluation results in formats beyond Markdown and HTML:
+
+```python
+from aspice_eval import ReportRenderer, register_renderer
+from aspice_eval import EvaluationResult, CapabilityLevelResult, EvaluationConfig
+
+class JSONReportRenderer(ReportRenderer):
+    """Render evaluation results as JSON."""
+
+    def render(self, evaluation, levels, config, kb_metadata) -> str:
+        import json
+        return json.dumps({
+            "ratings": [
+                {"criteria_id": r.criteria_id, "rating": r.rating, "gaps": r.gaps}
+                for r in evaluation.ratings
+            ],
+        }, indent=2)
+
+# Register and use
+register_renderer("json", JSONReportRenderer)
+```
+
+## CLI Usage
+
+### aspice-eval evaluate
+
+```bash
+# Evaluate an SDP document
+aspice-eval evaluate --sdp path/to/sdp.md --target-level 2 --groups SWE,MAN
+
+# Write report to a file
+aspice-eval evaluate --sdp path/to/sdp.md --output report.md
+
+# Use a specific AI provider
+aspice-eval evaluate --sdp path/to/sdp.md --provider bedrock \
+  --model us.anthropic.claude-sonnet-4-20250514-v1:0 --region us-east-1
+```
+
+### aspice-eval validate-kb
+
+```bash
+# Validate the default knowledge base
+aspice-eval validate-kb
+
+# Validate a custom knowledge base
+aspice-eval validate-kb --kb-path /path/to/knowledge_base
+```
+
+## Installation
+
+```bash
+pip install aspice-eval
+
+# With AI provider support
+pip install "aspice-eval[bedrock]"     # Amazon Bedrock (Claude)
+pip install "aspice-eval[openai]"      # OpenAI GPT-4o
+pip install "aspice-eval[anthropic]"   # Anthropic Claude (direct API)
+pip install "aspice-eval[all]"         # All providers
+```
+
+Requires Python 3.10+.
+
+## License
+
+MIT

aspice_eval-0.1.0/knowledge_base/aspice/_metadata.yaml

@@ -0,0 +1,62 @@
+standard:
+  name: "Automotive SPICE"
+  short_name: "ASPICE"
+  version: "4.0"
+  release_date: "2023-12"
+  source_references:
+    - title: "VDA Automotive SPICE Guidelines"
+      url: "https://www.automotivespice.com"
+    - title: "Wikipedia — Automotive SPICE"
+      url: "https://en.wikipedia.org/wiki/Automotive_SPICE"
+  license_note: >
+    Knowledge base content is derived from the Automotive SPICE PAM v4.0
+    (publicly available from vda-qmc.de). Base practice and generic practice
+    descriptions are reproduced under the PAM copyright release notice for
+    use in process assessment tooling.
+kb_version: "2.0.0"
+last_updated: "2026-05-01"
+process_groups:
+  - code: "SWE"
+    name: "Software Engineering"
+    processes: ["SWE.1", "SWE.2", "SWE.3", "SWE.4", "SWE.5", "SWE.6"]
+  - code: "SYS"
+    name: "System Engineering"
+    processes: ["SYS.1", "SYS.2", "SYS.3", "SYS.4", "SYS.5"]
+  - code: "MAN"
+    name: "Management"
+    processes: ["MAN.3", "MAN.5", "MAN.6"]
+  - code: "SUP"
+    name: "Support"
+    processes: ["SUP.1", "SUP.8", "SUP.9", "SUP.10", "SUP.11"]
+capability_levels:
+  - level: 0
+    name: "Incomplete"
+    process_attributes: []
+  - level: 1
+    name: "Performed"
+    process_attributes: ["PA 1.1"]
+  - level: 2
+    name: "Managed"
+    process_attributes: ["PA 2.1", "PA 2.2"]
+  - level: 3
+    name: "Established"
+    process_attributes: ["PA 3.1", "PA 3.2"]
+  - level: 4
+    name: "Predictable"
+    process_attributes: ["PA 4.1", "PA 4.2"]
+  - level: 5
+    name: "Innovating"
+    process_attributes: ["PA 5.1", "PA 5.2"]
+rating_scale:
+  - rating: "Fully achieved"
+    abbreviation: "F"
+    range: "86-100%"
+  - rating: "Largely achieved"
+    abbreviation: "L"
+    range: "51-85%"
+  - rating: "Partially achieved"
+    abbreviation: "P"
+    range: "16-50%"
+  - rating: "Not achieved"
+    abbreviation: "N"
+    range: "0-15%"