cisco-ai-skill-scanner 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. cisco_ai_skill_scanner-1.0.0.dist-info/METADATA +253 -0
  2. cisco_ai_skill_scanner-1.0.0.dist-info/RECORD +100 -0
  3. cisco_ai_skill_scanner-1.0.0.dist-info/WHEEL +4 -0
  4. cisco_ai_skill_scanner-1.0.0.dist-info/entry_points.txt +4 -0
  5. cisco_ai_skill_scanner-1.0.0.dist-info/licenses/LICENSE +17 -0
  6. skillanalyzer/__init__.py +45 -0
  7. skillanalyzer/_version.py +34 -0
  8. skillanalyzer/api/__init__.py +25 -0
  9. skillanalyzer/api/api.py +34 -0
  10. skillanalyzer/api/api_cli.py +78 -0
  11. skillanalyzer/api/api_server.py +634 -0
  12. skillanalyzer/api/router.py +527 -0
  13. skillanalyzer/cli/__init__.py +25 -0
  14. skillanalyzer/cli/cli.py +816 -0
  15. skillanalyzer/config/__init__.py +26 -0
  16. skillanalyzer/config/config.py +149 -0
  17. skillanalyzer/config/config_parser.py +122 -0
  18. skillanalyzer/config/constants.py +85 -0
  19. skillanalyzer/core/__init__.py +24 -0
  20. skillanalyzer/core/analyzers/__init__.py +75 -0
  21. skillanalyzer/core/analyzers/aidefense_analyzer.py +872 -0
  22. skillanalyzer/core/analyzers/base.py +53 -0
  23. skillanalyzer/core/analyzers/behavioral/__init__.py +30 -0
  24. skillanalyzer/core/analyzers/behavioral/alignment/__init__.py +45 -0
  25. skillanalyzer/core/analyzers/behavioral/alignment/alignment_llm_client.py +240 -0
  26. skillanalyzer/core/analyzers/behavioral/alignment/alignment_orchestrator.py +216 -0
  27. skillanalyzer/core/analyzers/behavioral/alignment/alignment_prompt_builder.py +422 -0
  28. skillanalyzer/core/analyzers/behavioral/alignment/alignment_response_validator.py +136 -0
  29. skillanalyzer/core/analyzers/behavioral/alignment/threat_vulnerability_classifier.py +198 -0
  30. skillanalyzer/core/analyzers/behavioral_analyzer.py +453 -0
  31. skillanalyzer/core/analyzers/cross_skill_analyzer.py +490 -0
  32. skillanalyzer/core/analyzers/llm_analyzer.py +440 -0
  33. skillanalyzer/core/analyzers/llm_prompt_builder.py +270 -0
  34. skillanalyzer/core/analyzers/llm_provider_config.py +215 -0
  35. skillanalyzer/core/analyzers/llm_request_handler.py +284 -0
  36. skillanalyzer/core/analyzers/llm_response_parser.py +81 -0
  37. skillanalyzer/core/analyzers/meta_analyzer.py +845 -0
  38. skillanalyzer/core/analyzers/static.py +1105 -0
  39. skillanalyzer/core/analyzers/trigger_analyzer.py +341 -0
  40. skillanalyzer/core/analyzers/virustotal_analyzer.py +463 -0
  41. skillanalyzer/core/exceptions.py +77 -0
  42. skillanalyzer/core/loader.py +377 -0
  43. skillanalyzer/core/models.py +300 -0
  44. skillanalyzer/core/reporters/__init__.py +26 -0
  45. skillanalyzer/core/reporters/json_reporter.py +65 -0
  46. skillanalyzer/core/reporters/markdown_reporter.py +209 -0
  47. skillanalyzer/core/reporters/sarif_reporter.py +246 -0
  48. skillanalyzer/core/reporters/table_reporter.py +195 -0
  49. skillanalyzer/core/rules/__init__.py +19 -0
  50. skillanalyzer/core/rules/patterns.py +165 -0
  51. skillanalyzer/core/rules/yara_scanner.py +157 -0
  52. skillanalyzer/core/scanner.py +437 -0
  53. skillanalyzer/core/static_analysis/__init__.py +27 -0
  54. skillanalyzer/core/static_analysis/cfg/__init__.py +21 -0
  55. skillanalyzer/core/static_analysis/cfg/builder.py +439 -0
  56. skillanalyzer/core/static_analysis/context_extractor.py +742 -0
  57. skillanalyzer/core/static_analysis/dataflow/__init__.py +25 -0
  58. skillanalyzer/core/static_analysis/dataflow/forward_analysis.py +715 -0
  59. skillanalyzer/core/static_analysis/interprocedural/__init__.py +21 -0
  60. skillanalyzer/core/static_analysis/interprocedural/call_graph_analyzer.py +406 -0
  61. skillanalyzer/core/static_analysis/interprocedural/cross_file_analyzer.py +190 -0
  62. skillanalyzer/core/static_analysis/parser/__init__.py +21 -0
  63. skillanalyzer/core/static_analysis/parser/python_parser.py +380 -0
  64. skillanalyzer/core/static_analysis/semantic/__init__.py +28 -0
  65. skillanalyzer/core/static_analysis/semantic/name_resolver.py +206 -0
  66. skillanalyzer/core/static_analysis/semantic/type_analyzer.py +200 -0
  67. skillanalyzer/core/static_analysis/taint/__init__.py +21 -0
  68. skillanalyzer/core/static_analysis/taint/tracker.py +252 -0
  69. skillanalyzer/core/static_analysis/types/__init__.py +36 -0
  70. skillanalyzer/data/__init__.py +30 -0
  71. skillanalyzer/data/prompts/boilerplate_protection_rule_prompt.md +26 -0
  72. skillanalyzer/data/prompts/code_alignment_threat_analysis_prompt.md +901 -0
  73. skillanalyzer/data/prompts/llm_response_schema.json +71 -0
  74. skillanalyzer/data/prompts/skill_meta_analysis_prompt.md +303 -0
  75. skillanalyzer/data/prompts/skill_threat_analysis_prompt.md +263 -0
  76. skillanalyzer/data/prompts/unified_response_schema.md +97 -0
  77. skillanalyzer/data/rules/signatures.yaml +440 -0
  78. skillanalyzer/data/yara_rules/autonomy_abuse.yara +66 -0
  79. skillanalyzer/data/yara_rules/code_execution.yara +61 -0
  80. skillanalyzer/data/yara_rules/coercive_injection.yara +115 -0
  81. skillanalyzer/data/yara_rules/command_injection.yara +54 -0
  82. skillanalyzer/data/yara_rules/credential_harvesting.yara +115 -0
  83. skillanalyzer/data/yara_rules/prompt_injection.yara +71 -0
  84. skillanalyzer/data/yara_rules/script_injection.yara +83 -0
  85. skillanalyzer/data/yara_rules/skill_discovery_abuse.yara +57 -0
  86. skillanalyzer/data/yara_rules/sql_injection.yara +73 -0
  87. skillanalyzer/data/yara_rules/system_manipulation.yara +65 -0
  88. skillanalyzer/data/yara_rules/tool_chaining_abuse.yara +60 -0
  89. skillanalyzer/data/yara_rules/transitive_trust_abuse.yara +73 -0
  90. skillanalyzer/data/yara_rules/unicode_steganography.yara +65 -0
  91. skillanalyzer/hooks/__init__.py +21 -0
  92. skillanalyzer/hooks/pre_commit.py +450 -0
  93. skillanalyzer/threats/__init__.py +25 -0
  94. skillanalyzer/threats/threats.py +480 -0
  95. skillanalyzer/utils/__init__.py +28 -0
  96. skillanalyzer/utils/command_utils.py +129 -0
  97. skillanalyzer/utils/di_container.py +154 -0
  98. skillanalyzer/utils/file_utils.py +86 -0
  99. skillanalyzer/utils/logging_config.py +96 -0
  100. skillanalyzer/utils/logging_utils.py +71 -0
@@ -0,0 +1,845 @@
1
+ # Copyright 2026 Cisco Systems, Inc.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+ # SPDX-License-Identifier: Apache-2.0
16
+
17
+ """
18
+ LLM Meta-Analyzer for Claude Skills Security Scanner.
19
+
20
+ Performs second-pass LLM analysis on findings from multiple analyzers to:
21
+ - Filter false positives based on contextual understanding
22
+ - Prioritize findings by actual exploitability and impact
23
+ - Correlate related findings across analyzers
24
+ - Detect threats that other analyzers may have missed
25
+ - Provide actionable remediation guidance
26
+
27
+ The meta-analyzer runs AFTER all other analyzers complete, reviewing their
28
+ collective findings to provide expert-level security assessment.
29
+
30
+ Requirements:
31
+ - Enable via CLI --enable-meta flag
32
+ - Requires LLM API key (uses same config as LLM analyzer)
33
+ - Works best with 2+ analyzers for cross-correlation
34
+ """
35
+
36
+ import asyncio
37
+ import json
38
+ import os
39
+ import secrets
40
+ from dataclasses import dataclass, field
41
+ from pathlib import Path
42
+ from typing import Any
43
+
44
+ from ...threats.threats import ThreatMapping
45
+ from ..models import Finding, Severity, Skill, ThreatCategory
46
+ from .base import BaseAnalyzer
47
+ from .llm_provider_config import ProviderConfig
48
+ from .llm_request_handler import LLMRequestHandler
49
+
50
+ # Check for LiteLLM availability
51
+ try:
52
+ from litellm import acompletion
53
+
54
+ LITELLM_AVAILABLE = True
55
+ except (ImportError, ModuleNotFoundError):
56
+ LITELLM_AVAILABLE = False
57
+ acompletion = None
58
+
59
+
60
+ @dataclass
61
+ class MetaAnalysisResult:
62
+ """Result of meta-analysis on security findings.
63
+
64
+ Attributes:
65
+ validated_findings: Findings confirmed as true positives with enriched data.
66
+ false_positives: Findings identified as likely false positives.
67
+ missed_threats: NEW threats found by meta-analyzer that other analyzers missed.
68
+ priority_order: Ordered list of finding indices by priority (highest first).
69
+ correlations: Groups of related findings.
70
+ recommendations: Actionable recommendations for remediation.
71
+ overall_risk_assessment: Summary risk assessment for the skill.
72
+ """
73
+
74
+ validated_findings: list[dict[str, Any]] = field(default_factory=list)
75
+ false_positives: list[dict[str, Any]] = field(default_factory=list)
76
+ missed_threats: list[dict[str, Any]] = field(default_factory=list)
77
+ priority_order: list[int] = field(default_factory=list)
78
+ correlations: list[dict[str, Any]] = field(default_factory=list)
79
+ recommendations: list[dict[str, Any]] = field(default_factory=list)
80
+ overall_risk_assessment: dict[str, Any] = field(default_factory=dict)
81
+
82
+ def to_dict(self) -> dict[str, Any]:
83
+ """Convert to dictionary format."""
84
+ return {
85
+ "validated_findings": self.validated_findings,
86
+ "false_positives": self.false_positives,
87
+ "missed_threats": self.missed_threats,
88
+ "priority_order": self.priority_order,
89
+ "correlations": self.correlations,
90
+ "recommendations": self.recommendations,
91
+ "overall_risk_assessment": self.overall_risk_assessment,
92
+ "summary": {
93
+ "total_original": len(self.validated_findings) + len(self.false_positives),
94
+ "validated_count": len(self.validated_findings),
95
+ "false_positive_count": len(self.false_positives),
96
+ "missed_threats_count": len(self.missed_threats),
97
+ "recommendations_count": len(self.recommendations),
98
+ },
99
+ }
100
+
101
+ def get_validated_findings(self, skill: Skill) -> list[Finding]:
102
+ """Convert validated findings back to Finding objects.
103
+
104
+ Args:
105
+ skill: The skill being analyzed (for context).
106
+
107
+ Returns:
108
+ List of validated Finding objects with meta-analysis enrichments.
109
+ """
110
+ findings = []
111
+ for finding_data in self.validated_findings:
112
+ try:
113
+ # Parse severity
114
+ severity_str = finding_data.get("severity", "MEDIUM").upper()
115
+ severity = Severity(severity_str)
116
+
117
+ # Parse category
118
+ category_str = finding_data.get("category", "policy_violation")
119
+ try:
120
+ category = ThreatCategory(category_str)
121
+ except ValueError:
122
+ category = ThreatCategory.POLICY_VIOLATION
123
+
124
+ # Build metadata with meta-analysis enrichments
125
+ metadata = dict(finding_data.get("metadata", {}))
126
+ if "confidence" in finding_data:
127
+ metadata["meta_confidence"] = finding_data["confidence"]
128
+ if "confidence_reason" in finding_data:
129
+ metadata["meta_confidence_reason"] = finding_data["confidence_reason"]
130
+ if "exploitability" in finding_data:
131
+ metadata["meta_exploitability"] = finding_data["exploitability"]
132
+ if "impact" in finding_data:
133
+ metadata["meta_impact"] = finding_data["impact"]
134
+ if "priority_rank" in finding_data:
135
+ metadata["meta_priority_rank"] = finding_data["priority_rank"]
136
+ metadata["meta_validated"] = True
137
+
138
+ finding = Finding(
139
+ id=finding_data.get("id", f"meta_{skill.name}_{len(findings)}"),
140
+ rule_id=finding_data.get("rule_id", "META_VALIDATED"),
141
+ category=category,
142
+ severity=severity,
143
+ title=finding_data.get("title", ""),
144
+ description=finding_data.get("description", ""),
145
+ file_path=finding_data.get("file_path"),
146
+ line_number=finding_data.get("line_number"),
147
+ snippet=finding_data.get("snippet"),
148
+ remediation=finding_data.get("remediation"),
149
+ analyzer="meta",
150
+ metadata=metadata,
151
+ )
152
+ findings.append(finding)
153
+ except Exception:
154
+ # Skip malformed findings
155
+ continue
156
+ return findings
157
+
158
+ def get_missed_threats(self, skill: Skill) -> list[Finding]:
159
+ """Convert missed threats to Finding objects.
160
+
161
+ These are NEW threats detected by meta-analyzer that other analyzers missed.
162
+
163
+ Args:
164
+ skill: The skill being analyzed.
165
+
166
+ Returns:
167
+ List of new Finding objects from meta-analysis.
168
+ """
169
+ findings = []
170
+ for idx, threat_data in enumerate(self.missed_threats):
171
+ try:
172
+ severity_str = threat_data.get("severity", "HIGH").upper()
173
+ severity = Severity(severity_str)
174
+
175
+ # Map threat category from AITech code if available
176
+ aitech_code = threat_data.get("aitech")
177
+ if aitech_code:
178
+ category_str = ThreatMapping.get_threat_category_from_aitech(aitech_code)
179
+ else:
180
+ category_str = threat_data.get("category", "policy_violation")
181
+
182
+ try:
183
+ category = ThreatCategory(category_str)
184
+ except ValueError:
185
+ category = ThreatCategory.POLICY_VIOLATION
186
+
187
+ finding = Finding(
188
+ id=f"meta_missed_{skill.name}_{idx}",
189
+ rule_id="META_DETECTED",
190
+ category=category,
191
+ severity=severity,
192
+ title=threat_data.get("title", "Threat detected by meta-analysis"),
193
+ description=threat_data.get("description", ""),
194
+ file_path=threat_data.get("file_path"),
195
+ line_number=threat_data.get("line_number"),
196
+ snippet=threat_data.get("evidence"),
197
+ remediation=threat_data.get("remediation"),
198
+ analyzer="meta",
199
+ metadata={
200
+ "meta_detected": True,
201
+ "detection_reason": threat_data.get("detection_reason", ""),
202
+ "meta_confidence": threat_data.get("confidence", "MEDIUM"),
203
+ "aitech": aitech_code,
204
+ },
205
+ )
206
+ findings.append(finding)
207
+ except Exception:
208
+ continue
209
+ return findings
210
+
211
+
212
+ class MetaAnalyzer(BaseAnalyzer):
213
+ """LLM-based meta-analyzer for reviewing and refining security findings.
214
+
215
+ This analyzer performs a second-pass analysis on findings from all other
216
+ analyzers to provide expert-level security assessment. It:
217
+ - Filters false positives using contextual understanding
218
+ - Prioritizes findings by actual risk
219
+ - Correlates related findings across analyzers
220
+ - Detects threats that other analyzers may have missed
221
+ - Provides specific remediation recommendations
222
+
223
+ The meta-analyzer runs AFTER all other analyzers complete.
224
+
225
+ Example:
226
+ >>> meta = MetaAnalyzer(model="claude-3-5-sonnet-20241022", api_key=api_key)
227
+ >>> result = await meta.analyze_with_findings(skill, all_findings, analyzers_used)
228
+ >>> validated = result.get_validated_findings(skill)
229
+ """
230
+
231
+ def __init__(
232
+ self,
233
+ model: str | None = None,
234
+ api_key: str | None = None,
235
+ max_tokens: int = 8000,
236
+ temperature: float = 0.1,
237
+ max_retries: int = 3,
238
+ timeout: int = 180,
239
+ # Azure-specific
240
+ base_url: str | None = None,
241
+ api_version: str | None = None,
242
+ # AWS Bedrock-specific
243
+ aws_region: str | None = None,
244
+ aws_profile: str | None = None,
245
+ aws_session_token: str | None = None,
246
+ ):
247
+ """Initialize the Meta Analyzer.
248
+
249
+ Args:
250
+ model: Model identifier (defaults to claude-3-5-sonnet-20241022)
251
+ api_key: API key (if None, reads from environment)
252
+ max_tokens: Maximum tokens for response
253
+ temperature: Sampling temperature (low for consistency)
254
+ max_retries: Max retry attempts on rate limits
255
+ timeout: Request timeout in seconds
256
+ base_url: Custom base URL (for Azure)
257
+ api_version: API version (for Azure)
258
+ aws_region: AWS region (for Bedrock)
259
+ aws_profile: AWS profile name (for Bedrock)
260
+ aws_session_token: AWS session token (for Bedrock)
261
+ """
262
+ super().__init__("meta_analyzer")
263
+
264
+ if not LITELLM_AVAILABLE:
265
+ raise ImportError("LiteLLM is required for MetaAnalyzer. Install with: pip install litellm")
266
+
267
+ # Use SKILL_SCANNER_* env vars only (no provider-specific fallbacks)
268
+ # Priority: meta-specific > scanner-wide
269
+ self.api_key = (
270
+ api_key
271
+ or os.getenv("SKILL_SCANNER_META_LLM_API_KEY") # Meta-specific
272
+ or os.getenv("SKILL_SCANNER_LLM_API_KEY") # Scanner-wide
273
+ )
274
+ self.model = (
275
+ model
276
+ or os.getenv("SKILL_SCANNER_META_LLM_MODEL") # Meta-specific
277
+ or os.getenv("SKILL_SCANNER_LLM_MODEL") # Scanner-wide
278
+ or "claude-3-5-sonnet-20241022"
279
+ )
280
+ self.base_url = (
281
+ base_url
282
+ or os.getenv("SKILL_SCANNER_META_LLM_BASE_URL") # Meta-specific
283
+ or os.getenv("SKILL_SCANNER_LLM_BASE_URL") # Scanner-wide
284
+ )
285
+ self.api_version = (
286
+ api_version
287
+ or os.getenv("SKILL_SCANNER_META_LLM_API_VERSION") # Meta-specific
288
+ or os.getenv("SKILL_SCANNER_LLM_API_VERSION") # Scanner-wide
289
+ )
290
+
291
+ # AWS Bedrock settings
292
+ self.aws_region = aws_region
293
+ self.aws_profile = aws_profile
294
+ self.aws_session_token = aws_session_token
295
+ self.is_bedrock = self.model and "bedrock/" in self.model
296
+
297
+ # Validate configuration
298
+ if not self.api_key and not self.is_bedrock:
299
+ raise ValueError(
300
+ "Meta-Analyzer LLM API key not configured. "
301
+ "Set SKILL_SCANNER_META_LLM_API_KEY or SKILL_SCANNER_LLM_API_KEY environment variable."
302
+ )
303
+
304
+ # Azure validation
305
+ if self.model and self.model.startswith("azure/"):
306
+ if not self.base_url:
307
+ raise ValueError(
308
+ "Azure OpenAI base URL not configured for meta-analyzer. "
309
+ "Set SKILL_SCANNER_META_LLM_BASE_URL environment variable."
310
+ )
311
+ if not self.api_version:
312
+ raise ValueError(
313
+ "Azure OpenAI API version not configured for meta-analyzer. "
314
+ "Set SKILL_SCANNER_META_LLM_API_VERSION environment variable."
315
+ )
316
+
317
+ self.max_tokens = max_tokens
318
+ self.temperature = temperature
319
+ self.max_retries = max_retries
320
+ self.timeout = timeout
321
+
322
+ # Load prompts
323
+ self._load_prompts()
324
+
325
+ def _load_prompts(self):
326
+ """Load meta-analysis prompt templates from files."""
327
+ prompts_dir = Path(__file__).parent.parent.parent / "data" / "prompts"
328
+ meta_prompt_file = prompts_dir / "skill_meta_analysis_prompt.md"
329
+
330
+ try:
331
+ if meta_prompt_file.exists():
332
+ self.system_prompt = meta_prompt_file.read_text(encoding="utf-8")
333
+ else:
334
+ print(f"Warning: Meta-analysis prompt not found at {meta_prompt_file}")
335
+ self.system_prompt = self._get_default_system_prompt()
336
+ except Exception as e:
337
+ print(f"Warning: Failed to load meta-analysis prompt: {e}")
338
+ self.system_prompt = self._get_default_system_prompt()
339
+
340
+ def _get_default_system_prompt(self) -> str:
341
+ """Get default system prompt if file not found."""
342
+ return """You are a senior security analyst performing meta-analysis on Claude Skill security findings.
343
+ Your role is to review findings from multiple analyzers, identify false positives,
344
+ prioritize by actual risk, correlate related issues, and provide actionable recommendations.
345
+
346
+ Respond with JSON containing your analysis following the required schema."""
347
+
348
+ def analyze(self, skill: Skill) -> list[Finding]:
349
+ """Analyze a skill (no-op for meta-analyzer).
350
+
351
+ The meta-analyzer requires findings from other analyzers.
352
+ Use analyze_with_findings() instead.
353
+
354
+ Args:
355
+ skill: The skill to analyze
356
+
357
+ Returns:
358
+ Empty list (meta-analyzer needs existing findings)
359
+ """
360
+ return []
361
+
362
+ async def analyze_with_findings(
363
+ self,
364
+ skill: Skill,
365
+ findings: list[Finding],
366
+ analyzers_used: list[str],
367
+ ) -> MetaAnalysisResult:
368
+ """Perform meta-analysis on findings from other analyzers.
369
+
370
+ Args:
371
+ skill: The skill being analyzed
372
+ findings: List of findings from all other analyzers
373
+ analyzers_used: Names of analyzers that produced the findings
374
+
375
+ Returns:
376
+ MetaAnalysisResult with validated findings, false positives, and recommendations
377
+ """
378
+ if not findings:
379
+ return MetaAnalysisResult(
380
+ overall_risk_assessment={
381
+ "risk_level": "SAFE",
382
+ "summary": "No security findings to analyze - skill appears safe.",
383
+ }
384
+ )
385
+
386
+ # Generate random delimiters for prompt injection protection
387
+ random_id = secrets.token_hex(16)
388
+ start_tag = f"<!---SKILL_CONTENT_START_{random_id}--->"
389
+ end_tag = f"<!---SKILL_CONTENT_END_{random_id}--->"
390
+
391
+ # Build skill context
392
+ skill_context = self._build_skill_context(skill)
393
+
394
+ # Build findings data
395
+ findings_data = self._serialize_findings(findings)
396
+
397
+ # Build user prompt
398
+ user_prompt = self._build_user_prompt(
399
+ skill=skill,
400
+ skill_context=skill_context,
401
+ findings_data=findings_data,
402
+ analyzers_used=analyzers_used,
403
+ start_tag=start_tag,
404
+ end_tag=end_tag,
405
+ )
406
+
407
+ try:
408
+ # Make LLM request
409
+ response = await self._make_llm_request(self.system_prompt, user_prompt)
410
+
411
+ # Parse response
412
+ result = self._parse_response(response, findings)
413
+
414
+ print(
415
+ f"Meta-analysis complete: {len(result.validated_findings)} validated, "
416
+ f"{len(result.false_positives)} false positives filtered, "
417
+ f"{len(result.missed_threats)} new threats detected"
418
+ )
419
+
420
+ return result
421
+
422
+ except Exception as e:
423
+ print(f"Meta-analysis failed: {e}")
424
+ # Return original findings as validated if analysis fails
425
+ return MetaAnalysisResult(
426
+ validated_findings=[self._finding_to_dict(f) for f in findings],
427
+ overall_risk_assessment={
428
+ "risk_level": "UNKNOWN",
429
+ "summary": f"Meta-analysis failed: {str(e)}. Original findings preserved.",
430
+ },
431
+ )
432
+
433
+ def _build_skill_context(self, skill: Skill) -> str:
434
+ """Build comprehensive skill context for meta-analysis.
435
+
436
+ Includes full skill content to enable accurate validation of findings.
437
+ """
438
+ lines = []
439
+ lines.append(f"## Skill: {skill.name}")
440
+ lines.append(f"**Description:** {skill.description}")
441
+ lines.append(f"**Directory:** {skill.directory}")
442
+ lines.append("")
443
+
444
+ # Manifest info
445
+ lines.append("### Manifest")
446
+ lines.append(f"- License: {skill.manifest.license or 'Not specified'}")
447
+ lines.append(f"- Compatibility: {skill.manifest.compatibility or 'Not specified'}")
448
+ lines.append(
449
+ f"- Allowed Tools: {', '.join(skill.manifest.allowed_tools) if skill.manifest.allowed_tools else 'Not specified'}"
450
+ )
451
+ lines.append("")
452
+
453
+ # Full instruction body (SKILL.md content)
454
+ lines.append("### SKILL.md Instructions (Full)")
455
+ # Limit to 50KB to avoid excessive token usage
456
+ max_instruction_size = 50000
457
+ if len(skill.instruction_body) > max_instruction_size:
458
+ lines.append(
459
+ f"```markdown\n{skill.instruction_body[:max_instruction_size]}\n... [TRUNCATED - {len(skill.instruction_body)} chars total]\n```"
460
+ )
461
+ else:
462
+ lines.append(f"```markdown\n{skill.instruction_body}\n```")
463
+ lines.append("")
464
+
465
+ # Files summary
466
+ lines.append("### Files in Skill Package")
467
+ for f in skill.files:
468
+ lines.append(f"- {f.relative_path} ({f.file_type}, {f.size_bytes} bytes)")
469
+ lines.append("")
470
+
471
+ # Full file contents for code files
472
+ lines.append("### File Contents")
473
+ code_extensions = {".py", ".sh", ".bash", ".js", ".ts", ".rb", ".pl", ".yaml", ".yml", ".json", ".toml"}
474
+ max_file_size = 30000 # 30KB per file
475
+ total_code_size = 0
476
+ max_total_code_size = 150000 # 150KB total for all code
477
+
478
+ for f in skill.files:
479
+ # Skip if we've already included too much code
480
+ if total_code_size >= max_total_code_size:
481
+ lines.append("\n... [REMAINING FILES OMITTED - total code size limit reached]")
482
+ break
483
+
484
+ # Check if it's a code file worth including
485
+ file_ext = Path(f.relative_path).suffix.lower()
486
+ if file_ext in code_extensions or f.file_type in ["python", "bash", "script"]:
487
+ try:
488
+ file_path = Path(skill.directory) / f.relative_path
489
+ if file_path.exists() and file_path.is_file():
490
+ content = file_path.read_text(encoding="utf-8", errors="replace")
491
+
492
+ # Truncate large files
493
+ if len(content) > max_file_size:
494
+ content = content[:max_file_size] + f"\n... [TRUNCATED - {len(content)} chars total]"
495
+
496
+ lines.append(f"\n#### {f.relative_path}")
497
+ lines.append(f"```{file_ext.lstrip('.') or 'text'}\n{content}\n```")
498
+ total_code_size += len(content)
499
+ except Exception:
500
+ # Skip files that can't be read
501
+ pass
502
+
503
+ lines.append("")
504
+
505
+ # Referenced files
506
+ if skill.referenced_files:
507
+ lines.append("### Referenced Files")
508
+ for ref in skill.referenced_files:
509
+ lines.append(f"- {ref}")
510
+ lines.append("")
511
+
512
+ return "\n".join(lines)
513
+
514
+ def _serialize_findings(self, findings: list[Finding]) -> str:
515
+ """Serialize findings to JSON for the prompt."""
516
+ findings_list = []
517
+ for i, f in enumerate(findings):
518
+ findings_list.append(
519
+ {
520
+ "_index": i,
521
+ "id": f.id,
522
+ "rule_id": f.rule_id,
523
+ "category": f.category.value,
524
+ "severity": f.severity.value,
525
+ "title": f.title,
526
+ "description": f.description,
527
+ "file_path": f.file_path,
528
+ "line_number": f.line_number,
529
+ "snippet": f.snippet[:500] if f.snippet else None,
530
+ "analyzer": f.analyzer,
531
+ "metadata": f.metadata,
532
+ }
533
+ )
534
+ return json.dumps(findings_list, indent=2)
535
+
536
+ def _finding_to_dict(self, finding: Finding) -> dict[str, Any]:
537
+ """Convert Finding to dictionary."""
538
+ return {
539
+ "id": finding.id,
540
+ "rule_id": finding.rule_id,
541
+ "category": finding.category.value,
542
+ "severity": finding.severity.value,
543
+ "title": finding.title,
544
+ "description": finding.description,
545
+ "file_path": finding.file_path,
546
+ "line_number": finding.line_number,
547
+ "snippet": finding.snippet,
548
+ "remediation": finding.remediation,
549
+ "analyzer": finding.analyzer,
550
+ "metadata": finding.metadata,
551
+ }
552
+
553
+ def _build_user_prompt(
554
+ self,
555
+ skill: Skill,
556
+ skill_context: str,
557
+ findings_data: str,
558
+ analyzers_used: list[str],
559
+ start_tag: str,
560
+ end_tag: str,
561
+ ) -> str:
562
+ """Build the user prompt for meta-analysis."""
563
+ num_findings = findings_data.count('"_index"')
564
+ return f"""## Meta-Analysis Request
565
+
566
+ You have {num_findings} findings from {len(analyzers_used)} analyzers. Your job is to **filter the noise and prioritize what matters**.
567
+
568
+ **IMPORTANT**: You have FULL ACCESS to the skill content below - including complete SKILL.md and all code files. Use this to VERIFY findings are accurate.
569
+
570
+ ### Analyzers Used
571
+ {", ".join(analyzers_used)}
572
+
573
+ ### Skill Context (FULL CONTENT)
574
+ {start_tag}
575
+ {skill_context}
576
+ {end_tag}
577
+
578
+ ### Findings from Analyzers ({num_findings} total)
579
+ ```json
580
+ {findings_data}
581
+ ```
582
+
583
+ ### Your Task (IN ORDER OF IMPORTANCE)
584
+
585
+ 1. **FILTER FALSE POSITIVES** (Most Important)
586
+ - VERIFY each finding against the actual code above. If the code doesn't match the claim → FALSE POSITIVE
587
+ - Pattern matches without actual malicious behavior → FALSE POSITIVE
588
+ - Static-only findings not confirmed by LLM/behavioral → likely FALSE POSITIVE
589
+ - Reading internal files, using standard libraries normally → FALSE POSITIVE
590
+ - Aim to filter 30-70% of static analyzer findings as noise
591
+
592
+ 2. **PRIORITIZE BY ACTUAL RISK**
593
+ - What should the developer fix FIRST? Put it at index 0 in priority_order
594
+ - CRITICAL: Active data exfiltration, credential theft
595
+ - HIGH: Command injection, prompt injection with clear exploitation path
596
+ - MEDIUM: Potential issues that need more context
597
+ - LOW/Filter: Informational, style, missing optional metadata
598
+
599
+ 3. **CONSOLIDATE RELATED FINDINGS**
600
+ - Multiple findings about the same issue = ONE entry in correlations
601
+ - Example: "Reads AWS creds" + "Makes HTTP POST" + "Sends data" = ONE "Credential Exfiltration" issue
602
+
603
+ 4. **MAKE ACTIONABLE**
604
+ - Every recommendation needs a specific fix (code example if possible)
605
+ - "Don't do X" is not actionable. "Replace X with Y" is actionable.
606
+
607
+ 5. **DETECT MISSED THREATS** (ONLY if obvious)
608
+ - This should be RARE. Leave missed_threats EMPTY unless there's something critical and obvious.
609
+ - Don't invent problems to fill this field.
610
+
611
+ Respond with a JSON object following the schema in the system prompt."""
612
+
613
+ async def _make_llm_request(self, system_prompt: str, user_prompt: str) -> str:
614
+ """Make a request to the LLM API."""
615
+ messages = [
616
+ {"role": "system", "content": system_prompt},
617
+ {"role": "user", "content": user_prompt},
618
+ ]
619
+
620
+ api_params = {
621
+ "model": self.model,
622
+ "messages": messages,
623
+ "temperature": self.temperature,
624
+ "max_tokens": self.max_tokens,
625
+ "timeout": float(self.timeout),
626
+ }
627
+
628
+ if self.api_key:
629
+ api_params["api_key"] = self.api_key
630
+
631
+ if self.base_url:
632
+ api_params["api_base"] = self.base_url
633
+
634
+ if self.api_version:
635
+ api_params["api_version"] = self.api_version
636
+
637
+ # AWS Bedrock configuration
638
+ if self.aws_region:
639
+ api_params["aws_region_name"] = self.aws_region
640
+ if self.aws_session_token:
641
+ api_params["aws_session_token"] = self.aws_session_token
642
+ if self.aws_profile:
643
+ api_params["aws_profile_name"] = self.aws_profile
644
+
645
+ # Retry logic with exponential backoff
646
+ last_exception = None
647
+ for attempt in range(self.max_retries):
648
+ try:
649
+ response = await acompletion(**api_params)
650
+ return response.choices[0].message.content
651
+
652
+ except Exception as e:
653
+ last_exception = e
654
+ error_msg = str(e).lower()
655
+
656
+ is_retryable = any(
657
+ keyword in error_msg
658
+ for keyword in [
659
+ "timeout",
660
+ "tls",
661
+ "connection",
662
+ "network",
663
+ "rate limit",
664
+ "throttle",
665
+ "429",
666
+ "503",
667
+ "504",
668
+ ]
669
+ )
670
+
671
+ if attempt < self.max_retries - 1 and is_retryable:
672
+ delay = (2**attempt) * 1.0
673
+ print(f"Meta-analysis LLM request failed (attempt {attempt + 1}): {e}")
674
+ await asyncio.sleep(delay)
675
+ else:
676
+ raise last_exception
677
+
678
+ raise last_exception
679
+
680
+ def _parse_response(self, response: str, original_findings: list[Finding]) -> MetaAnalysisResult:
681
+ """Parse the LLM meta-analysis response."""
682
+ try:
683
+ json_data = self._extract_json_from_response(response)
684
+
685
+ result = MetaAnalysisResult(
686
+ validated_findings=json_data.get("validated_findings", []),
687
+ false_positives=json_data.get("false_positives", []),
688
+ missed_threats=json_data.get("missed_threats", []),
689
+ priority_order=json_data.get("priority_order", []),
690
+ correlations=json_data.get("correlations", []),
691
+ recommendations=json_data.get("recommendations", []),
692
+ overall_risk_assessment=json_data.get("overall_risk_assessment", {}),
693
+ )
694
+
695
+ # Enrich validated findings with original data
696
+ self._enrich_findings(result, original_findings)
697
+
698
+ return result
699
+
700
+ except (json.JSONDecodeError, ValueError) as e:
701
+ print(f"Failed to parse meta-analysis response: {e}")
702
+ # Return original findings as validated
703
+ return MetaAnalysisResult(
704
+ validated_findings=[self._finding_to_dict(f) for f in original_findings],
705
+ overall_risk_assessment={
706
+ "risk_level": "UNKNOWN",
707
+ "summary": "Failed to parse meta-analysis response",
708
+ },
709
+ )
710
+
711
+ def _extract_json_from_response(self, response: str) -> dict[str, Any]:
712
+ """Extract JSON from LLM response using multiple strategies."""
713
+ if not response or not response.strip():
714
+ raise ValueError("Empty response from LLM")
715
+
716
+ # Strategy 1: Parse entire response as JSON
717
+ try:
718
+ return json.loads(response.strip())
719
+ except json.JSONDecodeError:
720
+ pass
721
+
722
+ # Strategy 2: Extract from markdown code blocks
723
+ try:
724
+ json_start = "```json"
725
+ json_end = "```"
726
+
727
+ start_idx = response.find(json_start)
728
+ if start_idx != -1:
729
+ content_start = start_idx + len(json_start)
730
+ end_idx = response.find(json_end, content_start)
731
+
732
+ if end_idx != -1:
733
+ json_str = response[content_start:end_idx].strip()
734
+ return json.loads(json_str)
735
+ except json.JSONDecodeError:
736
+ pass
737
+
738
+ # Strategy 3: Find JSON object by balanced braces
739
+ try:
740
+ start_idx = response.find("{")
741
+ if start_idx != -1:
742
+ brace_count = 0
743
+ end_idx = -1
744
+
745
+ for i in range(start_idx, len(response)):
746
+ if response[i] == "{":
747
+ brace_count += 1
748
+ elif response[i] == "}":
749
+ brace_count -= 1
750
+ if brace_count == 0:
751
+ end_idx = i + 1
752
+ break
753
+
754
+ if end_idx != -1:
755
+ json_content = response[start_idx:end_idx]
756
+ return json.loads(json_content)
757
+ except json.JSONDecodeError:
758
+ pass
759
+
760
+ raise ValueError("No valid JSON found in response")
761
+
762
+ def _enrich_findings(self, result: MetaAnalysisResult, original_findings: list[Finding]) -> None:
763
+ """Enrich validated findings with original finding data."""
764
+ original_lookup = {i: self._finding_to_dict(f) for i, f in enumerate(original_findings)}
765
+
766
+ # Enrich validated findings
767
+ for finding in result.validated_findings:
768
+ idx = finding.get("_index")
769
+ if idx is not None and idx in original_lookup:
770
+ original = original_lookup[idx]
771
+ for key, value in original.items():
772
+ if key not in finding:
773
+ finding[key] = value
774
+
775
+ # Enrich false positives
776
+ for finding in result.false_positives:
777
+ idx = finding.get("_index")
778
+ if idx is not None and idx in original_lookup:
779
+ original = original_lookup[idx]
780
+ for key, value in original.items():
781
+ if key not in finding:
782
+ finding[key] = value
783
+
784
+
785
+ def apply_meta_analysis_to_results(
786
+ original_findings: list[Finding],
787
+ meta_result: MetaAnalysisResult,
788
+ skill: Skill,
789
+ ) -> list[Finding]:
790
+ """Apply meta-analysis results to filter and enrich findings.
791
+
792
+ This function:
793
+ 1. Filters out false positives identified by meta-analysis
794
+ 2. Adds meta-analysis enrichments to validated findings
795
+ 3. Adds any new threats detected by meta-analyzer
796
+
797
+ Args:
798
+ original_findings: Original findings from all analyzers
799
+ meta_result: Results from meta-analysis
800
+ skill: The skill being analyzed
801
+
802
+ Returns:
803
+ Filtered and enriched list of findings
804
+ """
805
+ # Build set of false positive indices
806
+ fp_indices = set()
807
+ for fp in meta_result.false_positives:
808
+ if "_index" in fp:
809
+ fp_indices.add(fp["_index"])
810
+
811
+ # Build enrichment lookup from validated findings
812
+ enrichments = {}
813
+ for vf in meta_result.validated_findings:
814
+ idx = vf.get("_index")
815
+ if idx is not None:
816
+ enrichments[idx] = {
817
+ "meta_validated": True,
818
+ "meta_confidence": vf.get("confidence"),
819
+ "meta_confidence_reason": vf.get("confidence_reason"),
820
+ "meta_exploitability": vf.get("exploitability"),
821
+ "meta_impact": vf.get("impact"),
822
+ }
823
+
824
+ # Filter and enrich original findings
825
+ result_findings = []
826
+ for i, finding in enumerate(original_findings):
827
+ # Skip false positives
828
+ if i in fp_indices:
829
+ continue
830
+
831
+ # Add enrichments if available
832
+ if i in enrichments:
833
+ for key, value in enrichments[i].items():
834
+ if value is not None:
835
+ finding.metadata[key] = value
836
+ else:
837
+ finding.metadata["meta_reviewed"] = True
838
+
839
+ result_findings.append(finding)
840
+
841
+ # Add missed threats as new findings
842
+ missed_findings = meta_result.get_missed_threats(skill)
843
+ result_findings.extend(missed_findings)
844
+
845
+ return result_findings