cisco-ai-skill-scanner 1.0.0__py3-none-any.whl

skillanalyzer/core/analyzers/base.py

@@ -0,0 +1,53 @@
+# Copyright 2026 Cisco Systems, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Base analyzer interface for skill security scanning.
+"""
+
+from abc import ABC, abstractmethod
+
+from ..models import Finding, Skill
+
+
+class BaseAnalyzer(ABC):
+    """Abstract base class for all security analyzers."""
+
+    def __init__(self, name: str):
+        """
+        Initialize analyzer.
+
+        Args:
+            name: Name of the analyzer
+        """
+        self.name = name
+
+    @abstractmethod
+    def analyze(self, skill: Skill) -> list[Finding]:
+        """
+        Analyze a skill for security issues.
+
+        Args:
+            skill: The skill to analyze
+
+        Returns:
+            List of security findings
+        """
+        pass
+
+    def get_name(self) -> str:
+        """Get the analyzer name."""
+        return self.name

skillanalyzer/core/analyzers/behavioral/__init__.py

@@ -0,0 +1,30 @@
+# Copyright 2026 Cisco Systems, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Behavioral Analysis Package for Claude Skills.
+
+This package provides enhanced behavioral analysis capabilities including:
+- Alignment verification between skill description and code behavior
+- LLM-powered semantic analysis
+- Cross-file analysis and call graph tracking
+
+Components:
+- alignment/: Alignment verification layer for skill description vs code
+"""
+
+from .alignment import AlignmentOrchestrator
+
+__all__ = ["AlignmentOrchestrator"]

skillanalyzer/core/analyzers/behavioral/alignment/__init__.py

@@ -0,0 +1,45 @@
+# Copyright 2026 Cisco Systems, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Alignment Verification Layer for Semantic Analysis.
+
+This package provides the alignment verification layer that checks if skill
+descriptions and manifest claims accurately match their actual implementation behavior.
+
+Components:
+- AlignmentOrchestrator: Main orchestrator coordinating alignment verification
+- AlignmentPromptBuilder: Constructs comprehensive prompts with evidence
+- AlignmentLLMClient: Handles LLM API interaction for verification
+- AlignmentResponseValidator: Validates and parses LLM responses
+- ThreatVulnerabilityClassifier: Classifies findings as threats or vulnerabilities
+
+All components use the 'alignment_' prefix to indicate they are part of
+the semantic alignment verification layer.
+"""
+
+from .alignment_llm_client import AlignmentLLMClient
+from .alignment_orchestrator import AlignmentOrchestrator
+from .alignment_prompt_builder import AlignmentPromptBuilder
+from .alignment_response_validator import AlignmentResponseValidator
+from .threat_vulnerability_classifier import ThreatVulnerabilityClassifier
+
+__all__ = [
+    "AlignmentOrchestrator",
+    "AlignmentPromptBuilder",
+    "AlignmentLLMClient",
+    "AlignmentResponseValidator",
+    "ThreatVulnerabilityClassifier",
+]

skillanalyzer/core/analyzers/behavioral/alignment/alignment_llm_client.py

@@ -0,0 +1,240 @@
+# Copyright 2026 Cisco Systems, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Alignment LLM Client for Semantic Verification.
+
+This module handles all LLM API interactions specifically for semantic alignment
+verification between skill descriptions and their implementation.
+
+The client manages:
+- LLM configuration (API keys, endpoints, models)
+- Request construction for alignment verification
+- API communication via litellm
+- Response retrieval with retry logic
+"""
+
+import asyncio
+import logging
+import os
+
+try:
+    from litellm import acompletion
+
+    LITELLM_AVAILABLE = True
+except (ImportError, ModuleNotFoundError):
+    LITELLM_AVAILABLE = False
+    acompletion = None
+
+
+class AlignmentLLMClient:
+    """LLM client for semantic alignment verification queries.
+
+    Handles communication with LLM providers (OpenAI, Azure, Gemini, Bedrock, etc.)
+    specifically for alignment verification tasks.
+
+    Uses litellm for unified interface across providers and per-request
+    parameter passing to avoid configuration conflicts.
+    """
+
+    # Default configuration
+    DEFAULT_MAX_RETRIES = 3
+    DEFAULT_RETRY_BASE_DELAY = 2
+    PROMPT_LENGTH_THRESHOLD = 50000  # Warn if prompt exceeds this
+
+    def __init__(
+        self,
+        model: str = "gemini/gemini-2.0-flash",
+        api_key: str | None = None,
+        base_url: str | None = None,
+        api_version: str | None = None,
+        temperature: float = 0.1,
+        max_tokens: int = 4096,
+        timeout: int = 120,
+    ):
+        """Initialize the alignment LLM client.
+
+        Args:
+            model: LLM model to use
+            api_key: API key (or resolved from environment)
+            base_url: Optional base URL for API
+            api_version: Optional API version
+            temperature: Temperature for responses
+            max_tokens: Max tokens for responses
+            timeout: Request timeout in seconds
+
+        Raises:
+            ImportError: If litellm is not available
+            ValueError: If API key is not provided
+        """
+        if not LITELLM_AVAILABLE:
+            raise ImportError("litellm is required for alignment verification. Install with: pip install litellm")
+
+        # Resolve API key from environment if not provided
+        self._api_key = api_key or self._resolve_api_key(model)
+        if not self._api_key and not self._is_bedrock_model(model):
+            raise ValueError("LLM provider API key is required for alignment verification")
+
+        # Store configuration for per-request usage
+        self._model = model
+        self._base_url = base_url
+        self._api_version = api_version
+        self._temperature = temperature
+        self._max_tokens = max_tokens
+        self._timeout = timeout
+
+        self.logger = logging.getLogger(__name__)
+        self.logger.debug(f"AlignmentLLMClient initialized with model: {self._model}")
+
+    def _resolve_api_key(self, model: str) -> str | None:
+        """Resolve API key from environment variables.
+
+        Args:
+            model: Model name to determine provider
+
+        Returns:
+            API key or None
+        """
+        model_lower = model.lower()
+
+        # Special cases with different auth mechanisms
+        if "vertex" in model_lower:
+            # Vertex AI uses Google Cloud service account credentials
+            return os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")
+        elif "ollama" in model_lower:
+            # Ollama is local and typically doesn't need API key
+            return None
+
+        # All providers (including Bedrock, Gemini, OpenAI, Anthropic, Azure):
+        # Use SKILL_SCANNER_LLM_API_KEY
+        return os.environ.get("SKILL_SCANNER_LLM_API_KEY")
+
+    def _is_bedrock_model(self, model: str) -> bool:
+        """Check if model is AWS Bedrock.
+
+        Args:
+            model: Model name
+
+        Returns:
+            True if Bedrock model
+        """
+        return "bedrock" in model.lower()
+
+    async def verify_alignment(self, prompt: str) -> str:
+        """Send alignment verification prompt to LLM with retry logic.
+
+        Args:
+            prompt: Comprehensive prompt with alignment verification evidence
+
+        Returns:
+            LLM response (JSON string)
+
+        Raises:
+            Exception: If LLM API call fails after retries
+        """
+        # Log prompt length for debugging
+        prompt_length = len(prompt)
+        self.logger.debug(f"Prompt length: {prompt_length} characters")
+
+        # Check against threshold
+        if prompt_length > self.PROMPT_LENGTH_THRESHOLD:
+            self.logger.warning(
+                f"Large prompt detected: {prompt_length} characters "
+                f"(threshold: {self.PROMPT_LENGTH_THRESHOLD}) - may be truncated by LLM"
+            )
+
+        # Retry logic with exponential backoff
+        max_retries = self.DEFAULT_MAX_RETRIES
+        base_delay = self.DEFAULT_RETRY_BASE_DELAY
+
+        for attempt in range(max_retries):
+            try:
+                return await self._make_llm_request(prompt)
+            except Exception as e:
+                if attempt < max_retries - 1:
+                    delay = base_delay * (2**attempt)
+                    self.logger.warning(
+                        f"LLM request failed (attempt {attempt + 1}/{max_retries}): {e}. Retrying in {delay}s..."
+                    )
+                    await asyncio.sleep(delay)
+                else:
+                    self.logger.error(f"LLM request failed after {max_retries} attempts: {e}")
+                    raise
+
+    async def _make_llm_request(self, prompt: str) -> str:
+        """Make a single LLM API request.
+
+        Args:
+            prompt: Prompt to send
+
+        Returns:
+            LLM response content
+
+        Raises:
+            Exception: If API call fails
+        """
+        try:
+            request_params = {
+                "model": self._model,
+                "messages": [
+                    {
+                        "role": "system",
+                        "content": (
+                            "You are a security expert analyzing Claude Skills. "
+                            "You receive complete dataflow analysis and code context. "
+                            "Analyze if the skill description accurately describes what the code actually does. "
+                            "Respond ONLY with valid JSON. Do not include any markdown formatting or code blocks."
+                        ),
+                    },
+                    {"role": "user", "content": prompt},
+                ],
+                "max_tokens": self._max_tokens,
+                "temperature": self._temperature,
+                "timeout": self._timeout,
+            }
+
+            # Add API key if available
+            if self._api_key:
+                request_params["api_key"] = self._api_key
+
+            # Only enable JSON mode for supported models/providers
+            # Azure OpenAI with older API versions may not support this
+            if not self._model.startswith("azure/"):
+                request_params["response_format"] = {"type": "json_object"}
+
+            # Add optional parameters if configured
+            if self._base_url:
+                request_params["api_base"] = self._base_url
+            if self._api_version:
+                request_params["api_version"] = self._api_version
+
+            self.logger.debug(f"Sending alignment verification request to {self._model}")
+            response = await acompletion(**request_params)
+
+            # Extract content from response
+            content = response.choices[0].message.content
+
+            # Log response for debugging
+            if not content or not content.strip():
+                self.logger.warning(f"Empty response from LLM model {self._model}")
+                self.logger.debug(f"Full response object: {response}")
+            else:
+                self.logger.debug(f"LLM response length: {len(content)} chars")
+
+            return content if content else ""
+
+        except Exception as e:
+            self.logger.error(f"LLM alignment verification failed: {e}", exc_info=True)
+            raise

skillanalyzer/core/analyzers/behavioral/alignment/alignment_orchestrator.py

@@ -0,0 +1,216 @@
+# Copyright 2026 Cisco Systems, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Alignment Orchestrator - Main Coordinator for Skill Analysis.
+
+This module provides the main orchestrator for semantic alignment verification
+between skill descriptions/manifest claims and actual code implementation.
+
+The orchestrator coordinates:
+1. Building comprehensive prompts with evidence
+2. Querying LLM for alignment verification
+3. Validating and parsing responses
+4. Creating security findings for mismatches
+"""
+
+import logging
+from typing import Any
+
+from .....core.static_analysis.context_extractor import SkillFunctionContext
+from .alignment_llm_client import AlignmentLLMClient
+from .alignment_prompt_builder import AlignmentPromptBuilder
+from .alignment_response_validator import AlignmentResponseValidator
+from .threat_vulnerability_classifier import ThreatVulnerabilityClassifier
+
+
+class AlignmentOrchestrator:
+    """Orchestrates semantic alignment verification between skill descriptions and code.
+
+    This is the main alignment verification layer that coordinates:
+    - Prompt building with comprehensive evidence
+    - LLM-based alignment verification
+    - Response validation and finding creation
+    - Threat vs vulnerability classification
+
+    This class provides a clean interface for alignment checking and hides
+    the complexity of prompt construction, LLM interaction, and parsing.
+    """
+
+    def __init__(
+        self,
+        llm_model: str = "gemini/gemini-2.0-flash",
+        llm_api_key: str | None = None,
+        llm_base_url: str | None = None,
+        llm_temperature: float = 0.1,
+        llm_max_tokens: int = 4096,
+        llm_timeout: int = 120,
+    ):
+        """Initialize alignment orchestrator.
+
+        Args:
+            llm_model: LLM model to use (e.g., "gemini/gemini-2.0-flash")
+            llm_api_key: API key for the LLM provider
+            llm_base_url: Optional base URL for LLM API
+            llm_temperature: Temperature for LLM responses
+            llm_max_tokens: Max tokens for LLM responses
+            llm_timeout: Timeout for LLM requests in seconds
+
+        Raises:
+            ValueError: If LLM API key is not provided
+        """
+        self.logger = logging.getLogger(__name__)
+
+        # Initialize alignment verification components
+        self.prompt_builder = AlignmentPromptBuilder()
+        self.llm_client = AlignmentLLMClient(
+            model=llm_model,
+            api_key=llm_api_key,
+            base_url=llm_base_url,
+            temperature=llm_temperature,
+            max_tokens=llm_max_tokens,
+            timeout=llm_timeout,
+        )
+        self.response_validator = AlignmentResponseValidator()
+        self.threat_vuln_classifier = ThreatVulnerabilityClassifier(
+            model=llm_model,
+            api_key=llm_api_key,
+            base_url=llm_base_url,
+        )
+
+        # Track analysis statistics
+        self.stats = {
+            "total_analyzed": 0,
+            "mismatches_detected": 0,
+            "no_mismatch": 0,
+            "skipped_invalid_response": 0,
+            "skipped_error": 0,
+        }
+
+        self.logger.debug("AlignmentOrchestrator initialized")
+
+    async def check_alignment(
+        self,
+        func_context: SkillFunctionContext,
+        skill_description: str | None = None,
+    ) -> tuple[dict[str, Any], SkillFunctionContext] | None:
+        """Check if function behavior aligns with skill description.
+
+        This is the main entry point for alignment verification. It coordinates
+        the full verification pipeline:
+        1. Build comprehensive prompt with evidence
+        2. Query LLM for alignment analysis
+        3. Validate response
+        4. Return analysis and context for Finding creation
+
+        Args:
+            func_context: Complete function context with dataflow analysis
+            skill_description: Overall skill description from SKILL.md
+
+        Returns:
+            Tuple of (analysis_dict, func_context) if mismatch detected, None if aligned
+        """
+        self.stats["total_analyzed"] += 1
+
+        try:
+            # Step 1: Build alignment verification prompt
+            self.logger.debug(f"Building alignment prompt for {func_context.name}")
+            try:
+                prompt = self.prompt_builder.build_prompt(func_context, skill_description=skill_description)
+            except Exception as e:
+                self.logger.error(f"Prompt building failed for {func_context.name}: {e}", exc_info=True)
+                self.stats["skipped_error"] += 1
+                raise
+
+            # Step 2: Query LLM for alignment verification
+            self.logger.debug(f"Querying LLM for alignment verification of {func_context.name}")
+            try:
+                response = await self.llm_client.verify_alignment(prompt)
+            except Exception as e:
+                self.logger.error(f"LLM verification failed for {func_context.name}: {e}", exc_info=True)
+                self.stats["skipped_error"] += 1
+                raise
+
+            # Step 3: Validate and parse response
+            self.logger.debug(f"Validating alignment response for {func_context.name}")
+            try:
+                result = self.response_validator.validate(response)
+            except Exception as e:
+                self.logger.error(f"Response validation failed for {func_context.name}: {e}", exc_info=True)
+                self.stats["skipped_error"] += 1
+                raise
+
+            if not result:
+                self.logger.warning(f"Invalid response for {func_context.name}, skipping")
+                self.stats["skipped_invalid_response"] += 1
+                return None
+
+            # Step 4: Return analysis if mismatch detected
+            if result.get("mismatch_detected"):
+                self.logger.debug(f"Alignment mismatch detected in {func_context.name}")
+                self.stats["mismatches_detected"] += 1
+
+                # Step 5: Classify as threat or vulnerability (second alignment layer)
+                # Skip classification for INFO severity (documentation issues)
+                threat_name = result.get("threat_name", "")
+                if threat_name != "GENERAL DESCRIPTION-CODE MISMATCH":
+                    self.logger.debug(f"Classifying finding as threat or vulnerability for {func_context.name}")
+                    try:
+                        classification = await self.threat_vuln_classifier.classify_finding(
+                            threat_name=result.get("threat_name", "UNKNOWN"),
+                            severity=result.get("severity", "UNKNOWN"),
+                            summary=result.get("summary", ""),
+                            description_claims=result.get("description_claims", ""),
+                            actual_behavior=result.get("actual_behavior", ""),
+                            security_implications=result.get("security_implications", ""),
+                            dataflow_evidence=result.get("dataflow_evidence", ""),
+                        )
+                        if classification:
+                            # Add just the classification value to the result
+                            result["threat_vulnerability_classification"] = classification["classification"]
+                            self.logger.debug(
+                                f"Classified as {classification['classification']} with {classification['confidence']} confidence"
+                            )
+                        else:
+                            self.logger.warning(f"Failed to classify finding for {func_context.name}")
+                            result["threat_vulnerability_classification"] = "UNCLEAR"
+                    except Exception as e:
+                        self.logger.error(f"Classification failed for {func_context.name}: {e}", exc_info=True)
+                        # Continue without classification - mark as UNCLEAR
+                        result["threat_vulnerability_classification"] = "UNCLEAR"
+
+                return (result, func_context)
+            else:
+                self.logger.debug(f"No alignment mismatch in {func_context.name}")
+                self.stats["no_mismatch"] += 1
+                return None
+
+        except Exception as e:
+            self.logger.error(f"Alignment check failed for {func_context.name}: {e}")
+            self.stats["skipped_error"] += 1
+            return None
+
+    def get_statistics(self) -> dict[str, int]:
+        """Get analysis statistics.
+
+        Returns:
+            Dictionary with analysis statistics including:
+            - total_analyzed: Total functions analyzed
+            - mismatches_detected: Functions with detected mismatches
+            - no_mismatch: Functions with no mismatch
+            - skipped_invalid_response: Functions skipped due to invalid LLM response
+            - skipped_error: Functions skipped due to errors
+        """
+        return self.stats.copy()