trace-sec 0.0.0-stage → 2.1.0

package/src/trace_engine/config/__init__.py

@@ -0,0 +1,35 @@
+"""TRACE configuration module."""
+
+from trace_engine.config.settings import (
+    TraceConfig,
+    ProjectSettings,
+    AISettings,
+    TargetSettings,
+    RuntimeSettings,
+    AnalysisSettings,
+    TestsSettings,
+    ToolsSettings,
+)
+from trace_engine.config.loader import (
+    get_trace_dir,
+    get_config_path,
+    init_trace_dir,
+    load_config,
+    save_config,
+)
+
+__all__ = [
+    "TraceConfig",
+    "ProjectSettings",
+    "AISettings",
+    "TargetSettings",
+    "RuntimeSettings",
+    "AnalysisSettings",
+    "TestsSettings",
+    "ToolsSettings",
+    "get_trace_dir",
+    "get_config_path",
+    "init_trace_dir",
+    "load_config",
+    "save_config",
+]

package/src/trace_engine/config/defaults.py

@@ -0,0 +1,48 @@
+"""Default configuration values for TRACE."""
+
+DEFAULT_CONFIG_TOML = """# TRACE Configuration File
+# Threat Reconnaissance & Attack-path Correlation Engine
+
+[project]
+name = "default-project"
+version = "1.0.0"
+mode = "local"
+
+[ai]
+provider = "ollama"
+model = "qwen2.5-coder:latest"
+temperature = 0.1
+endpoint = "http://127.0.0.1:11434"
+
+[target]
+url = "http://127.0.0.1:8000"
+scope_mode = "local"
+allowed_hosts = ["localhost", "127.0.0.1", "0.0.0.0", "::1"]
+allowed_ports = [80, 443, 3000, 5000, 8000, 8080, 18080]
+
+[runtime]
+max_requests = 250
+concurrency = 2
+rate_limit_per_second = 5
+timeout_seconds = 8
+max_redirects = 3
+destructive_tests = false
+
+[analysis]
+max_graph_hops = 3
+max_source_context_lines = 120
+enable_tree_sitter = true
+
+[tests]
+enabled = [
+    "bola",
+    "bfla",
+    "authentication",
+    "ssrf",
+    "injection",
+    "mass_assignment"
+]
+
+[tools]
+enabled = ["schemathesis", "nuclei", "zap", "trufflehog"]
+"""

package/src/trace_engine/config/loader.py

@@ -0,0 +1,64 @@
+"""Configuration loader for TRACE."""
+
+import os
+from pathlib import Path
+from typing import Optional
+import tomllib
+import tomli_w
+
+from trace_engine.config.settings import TraceConfig
+from trace_engine.config.defaults import DEFAULT_CONFIG_TOML
+
+
+def get_trace_dir(repo_path: Path) -> Path:
+    """Get the .trace directory path within the given repository."""
+    return repo_path.resolve() / ".trace"
+
+
+def get_config_path(repo_path: Path) -> Path:
+    """Get the path to config.toml in the .trace directory."""
+    return get_trace_dir(repo_path) / "config.toml"
+
+
+def init_trace_dir(repo_path: Path, project_name: Optional[str] = None) -> Path:
+    """Initialize .trace directory structure in repository."""
+    trace_dir = get_trace_dir(repo_path)
+    trace_dir.mkdir(parents=True, exist_ok=True)
+    
+    (trace_dir / "cache").mkdir(exist_ok=True)
+    (trace_dir / "runs").mkdir(exist_ok=True)
+    (trace_dir / "reports").mkdir(exist_ok=True)
+    
+    config_file = trace_dir / "config.toml"
+    if not config_file.exists():
+        content = DEFAULT_CONFIG_TOML
+        if project_name:
+            content = content.replace('name = "default-project"', f'name = "{project_name}"')
+        config_file.write_text(content, encoding="utf-8")
+        
+    return trace_dir
+
+
+def load_config(repo_path: Path) -> TraceConfig:
+    """Load configuration from .trace/config.toml or return default configuration."""
+    config_file = get_config_path(repo_path)
+    if not config_file.exists():
+        return TraceConfig()
+        
+    try:
+        with open(config_file, "rb") as f:
+            data = tomllib.load(f)
+        return TraceConfig.model_validate(data)
+    except Exception:
+        return TraceConfig()
+
+
+def save_config(repo_path: Path, config: TraceConfig) -> None:
+    """Save configuration to .trace/config.toml."""
+    trace_dir = get_trace_dir(repo_path)
+    trace_dir.mkdir(parents=True, exist_ok=True)
+    config_file = trace_dir / "config.toml"
+    
+    data = config.model_dump()
+    with open(config_file, "wb") as f:
+        tomli_w.dump(data, f)

package/src/trace_engine/config/settings.py

@@ -0,0 +1,72 @@
+"""Configuration settings for TRACE using Pydantic."""
+
+from typing import List, Optional
+from pydantic import BaseModel, Field
+
+
+class ProjectSettings(BaseModel):
+    name: str = "default-project"
+    version: str = "1.0.0"
+    mode: str = "local"
+
+
+class AISettings(BaseModel):
+    provider: str = "ollama"
+    model: str = "qwen2.5-coder:latest"
+    temperature: float = 0.1
+    endpoint: str = "http://127.0.0.1:11434"
+
+
+class TargetSettings(BaseModel):
+    url: str = "http://127.0.0.1:8000"
+    scope_mode: str = "local"
+    allowed_hosts: List[str] = Field(
+        default_factory=lambda: ["localhost", "127.0.0.1", "0.0.0.0", "::1"]
+    )
+    allowed_ports: List[int] = Field(
+        default_factory=lambda: [80, 443, 3000, 5000, 8000, 8080, 18080]
+    )
+
+
+class RuntimeSettings(BaseModel):
+    max_requests: int = 250
+    concurrency: int = 2
+    rate_limit_per_second: int = 5
+    timeout_seconds: float = 8.0
+    max_redirects: int = 3
+    destructive_tests: bool = False
+
+
+class AnalysisSettings(BaseModel):
+    max_graph_hops: int = 3
+    max_source_context_lines: int = 120
+    enable_tree_sitter: bool = True
+
+
+class TestsSettings(BaseModel):
+    enabled: List[str] = Field(
+        default_factory=lambda: [
+            "bola",
+            "bfla",
+            "authentication",
+            "ssrf",
+            "injection",
+            "mass_assignment",
+        ]
+    )
+
+
+class ToolsSettings(BaseModel):
+    enabled: List[str] = Field(
+        default_factory=lambda: ["schemathesis", "nuclei", "zap", "trufflehog"]
+    )
+
+
+class TraceConfig(BaseModel):
+    project: ProjectSettings = Field(default_factory=ProjectSettings)
+    ai: AISettings = Field(default_factory=AISettings)
+    target: TargetSettings = Field(default_factory=TargetSettings)
+    runtime: RuntimeSettings = Field(default_factory=RuntimeSettings)
+    analysis: AnalysisSettings = Field(default_factory=AnalysisSettings)
+    tests: TestsSettings = Field(default_factory=TestsSettings)
+    tools: ToolsSettings = Field(default_factory=ToolsSettings)

package/src/trace_engine/doctor.py

@@ -0,0 +1,250 @@
+"""Environment diagnostics and system readiness check (trace doctor)."""
+
+import sys
+import sqlite3
+import shutil
+from typing import List, Dict, Any
+from pydantic import BaseModel
+
+from trace_engine.tools.registry import default_tool_registry
+from trace_engine.ai.ollama import OllamaProvider
+
+
+class CheckItem(BaseModel):
+    name: str
+    status: bool  # True for pass, False for fail/missing
+    category: str
+    details: str
+    critical: bool = False
+
+
+class DoctorReport(BaseModel):
+    items: List[CheckItem]
+    all_critical_passed: bool
+
+
+def run_doctor() -> DoctorReport:
+    """Runs system diagnostics across runtime, AI, tools, and dependencies."""
+    items: List[CheckItem] = []
+
+    # 1. Python version
+    py_ver = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
+    py_ok = sys.version_info >= (3, 11)
+    items.append(
+        CheckItem(
+            name=f"Python {py_ver}",
+            status=py_ok,
+            category="Core Runtime",
+            details="Requires Python >= 3.11",
+            critical=True,
+        )
+    )
+
+    # 2. SQLite
+    try:
+        conn = sqlite3.connect(":memory:")
+        conn.close()
+        items.append(
+            CheckItem(
+                name="SQLite 3",
+                status=True,
+                category="Core Runtime",
+                details=f"SQLite version {sqlite3.sqlite_version}",
+                critical=True,
+            )
+        )
+    except Exception as e:
+        items.append(
+            CheckItem(
+                name="SQLite 3",
+                status=False,
+                category="Core Runtime",
+                details=str(e),
+                critical=True,
+            )
+        )
+
+    # 3. Tree-sitter
+    try:
+        import tree_sitter_language_pack
+        items.append(
+            CheckItem(
+                name="Tree-sitter AST",
+                status=True,
+                category="Code Analysis",
+                details="Multi-language AST parsing enabled",
+                critical=True,
+            )
+        )
+    except Exception as e:
+        items.append(
+            CheckItem(
+                name="Tree-sitter AST",
+                status=False,
+                category="Code Analysis",
+                details="Fallback AST mode active",
+                critical=False,
+            )
+        )
+
+    # 4. NetworkX
+    try:
+        import networkx as nx
+        items.append(
+            CheckItem(
+                name="NetworkX APM",
+                status=True,
+                category="Attack-Path Model",
+                details=f"Graph engine v{nx.__version__}",
+                critical=True,
+            )
+        )
+    except Exception as e:
+        items.append(
+            CheckItem(
+                name="NetworkX APM",
+                status=False,
+                category="Attack-Path Model",
+                details=str(e),
+                critical=True,
+            )
+        )
+
+    # 5. HTTPX
+    try:
+        import httpx
+        items.append(
+            CheckItem(
+                name="HTTPX Client",
+                status=True,
+                category="Runtime Testing",
+                details=f"HTTP engine v{httpx.__version__}",
+                critical=True,
+            )
+        )
+    except Exception as e:
+        items.append(
+            CheckItem(
+                name="HTTPX Client",
+                status=False,
+                category="Runtime Testing",
+                details=str(e),
+                critical=True,
+            )
+        )
+
+    # 6. Local AI (Ollama)
+    ollama = OllamaProvider()
+    ollama_ok = ollama.is_available()
+    items.append(
+        CheckItem(
+            name="Ollama LLM",
+            status=ollama_ok,
+            category="AI Inference",
+            details="Connected at http://127.0.0.1:11434" if ollama_ok else "Not reachable (using deterministic planner)",
+            critical=False,
+        )
+    )
+
+    # 7. PyTorch & Transformers
+    try:
+        import torch
+        import transformers
+        items.append(
+            CheckItem(
+                name="PyTorch",
+                status=True,
+                category="AI Inference",
+                details=f"PyTorch v{torch.__version__}, Transformers v{transformers.__version__}",
+                critical=False,
+            )
+        )
+    except Exception as e:
+        items.append(
+            CheckItem(
+                name="PyTorch",
+                status=False,
+                category="AI Inference",
+                details=str(e),
+                critical=False,
+            )
+        )
+
+    # 8. Laya System 1 Engine
+    try:
+        import warnings
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            from trace_engine.intelligence.laya.router import LayaDecisionEngine
+            laya = LayaDecisionEngine()
+        laya_avail = laya.is_available()
+        items.append(
+            CheckItem(
+                name="Laya System 1",
+                status=laya_avail,
+                category="AI Inference",
+                details="Online (convaiinnovations/laya on CPU)" if laya_avail else "Standby (calibrated fallback active)",
+                critical=False,
+            )
+        )
+    except Exception as e:
+        items.append(
+            CheckItem(
+                name="Laya System 1",
+                status=False,
+                category="AI Inference",
+                details=str(e),
+                critical=False,
+            )
+        )
+
+    # 9. SecureBERT 2.0 Classifier
+    try:
+        from trace_engine.intelligence.securebert.classifier import SecureBERTClassifier
+        bert = SecureBERTClassifier()
+        items.append(
+            CheckItem(
+                name="SecureBERT 2.0",
+                status=True,
+                category="AI Inference",
+                details=f"Model: {bert.model_name}",
+                critical=False,
+            )
+        )
+    except Exception as e:
+        items.append(
+            CheckItem(
+                name="SecureBERT 2.0 Semantic Classifier",
+                status=False,
+                category="AI Inference",
+                details=str(e),
+                critical=False,
+            )
+        )
+
+    # 7. External Tools
+    for tool_info in default_tool_registry.get_all_info():
+        items.append(
+            CheckItem(
+                name=tool_info.name,
+                status=tool_info.installed,
+                category="Security Tool Adapters",
+                details=tool_info.version or f"Executable '{tool_info.command}' not found in PATH",
+                critical=False,
+            )
+        )
+
+    # 8. Docker
+    docker_installed = shutil.which("docker") is not None
+    items.append(
+        CheckItem(
+            name="Docker Desktop / CLI",
+            status=docker_installed,
+            category="Lab Isolation",
+            details="Available for isolated container labs" if docker_installed else "Not found (local process labs enabled)",
+            critical=False,
+        )
+    )
+
+    all_crit = all(item.status for item in items if item.critical)
+    return DoctorReport(items=items, all_critical_passed=all_crit)

package/src/trace_engine/findings/__init__.py

@@ -0,0 +1,15 @@
+"""Findings and evidence correlation module."""
+
+from trace_engine.findings.model import Finding, Severity, FindingConfidence
+from trace_engine.findings.correlate import EvidenceCorrelator
+from trace_engine.findings.store import FindingStore
+from trace_engine.findings.recommendations import get_remediation_for_category
+
+__all__ = [
+    "Finding",
+    "Severity",
+    "FindingConfidence",
+    "EvidenceCorrelator",
+    "FindingStore",
+    "get_remediation_for_category",
+]

package/src/trace_engine/findings/correlate.py

@@ -0,0 +1,103 @@
+"""Evidence correlator connecting static attack paths to runtime test results."""
+
+from typing import List, Optional
+from trace_engine.security.hypotheses import SecurityHypothesis
+from trace_engine.testpacks.base import TestExecutionResult
+from trace_engine.findings.model import Finding, Severity, FindingConfidence
+from trace_engine.findings.recommendations import get_remediation_for_category
+from trace_engine.apm.model import AttackPathModel
+
+
+SEVERITY_BY_CATEGORY = {
+    "BOLA": Severity.HIGH,
+    "BFLA": Severity.HIGH,
+    "AUTHENTICATION": Severity.CRITICAL,
+    "SSRF": Severity.HIGH,
+    "INJECTION": Severity.CRITICAL,
+    "MASS_ASSIGNMENT": Severity.MEDIUM,
+    "PATH_TRAVERSAL": Severity.HIGH,
+    "SSTI": Severity.CRITICAL,
+    "CORS": Severity.MEDIUM,
+    "DESERIALIZATION": Severity.CRITICAL,
+}
+
+
+class EvidenceCorrelator:
+    """Correlates static code evidence with runtime test executions using Neuro-Symbolic Bayesian Calibration."""
+
+    def correlate(
+        self,
+        hypothesis: SecurityHypothesis,
+        test_result: Optional[TestExecutionResult],
+        apm: AttackPathModel,
+    ) -> Optional[Finding]:
+        from trace_engine.security.fusion import NeuroSymbolicConfidenceEngine, ConfidenceEvidence
+
+        category_str = hypothesis.category.value
+
+        # Calculate combined confidence using Neuro-Symbolic Bayesian Fusion
+        evidence = ConfidenceEvidence(
+            symbolic_ast_score=hypothesis.confidence_prior,
+            dynamic_test_score=test_result.confidence if test_result else None,
+            is_dynamically_confirmed=bool(test_result and test_result.confirmed),
+        )
+        fused = NeuroSymbolicConfidenceEngine.fuse_evidence(evidence)
+        confidence = fused.confidence_level
+
+        # If not confirmed and posterior probability is below 0.40, do not generate noisy finding
+        if not (test_result and test_result.confirmed) and fused.posterior_probability < 0.40:
+            return None
+
+        # Build attack path node trail from APM
+        attack_path_nodes = []
+        if hypothesis.endpoint_id in apm.graph:
+            sinks = apm.find_paths_to_sinks(hypothesis.endpoint_id)
+            if sinks:
+                attack_path_nodes = sinks[0]
+            else:
+                paths = apm.find_paths_from_endpoint(hypothesis.endpoint_id, max_depth=3)
+                if paths:
+                    attack_path_nodes = paths[0]
+
+        # Runtime evidence bullet points
+        runtime_evidence = []
+        if test_result:
+            runtime_evidence.append(test_result.summary)
+            for obs in test_result.observations:
+                runtime_evidence.append(
+                    f"HTTP {obs.request_method} {obs.request_url} -> Status {obs.response_status} ({obs.latency_ms}ms)"
+                )
+
+        # Correlation explanation
+        if test_result and test_result.confirmed:
+            correlation_notes = (
+                f"Static Attack-Path Model identified vulnerable structure in source code, "
+                f"and active runtime test pack '{test_result.testpack_name}' successfully proved exploitability."
+            )
+        else:
+            correlation_notes = (
+                f"Static Attack-Path Model detected missing control boundary; runtime validation pending or inconclusive."
+            )
+
+        finding_id = hypothesis.id.replace("HYP-", "TR-")
+        severity = SEVERITY_BY_CATEGORY.get(category_str, Severity.MEDIUM)
+
+        ep_node = apm.get_node(hypothesis.endpoint_id)
+        source_loc = ep_node.location if ep_node else None
+
+        return Finding(
+            id=finding_id,
+            title=hypothesis.title,
+            category=category_str,
+            severity=severity,
+            confidence=confidence,
+            endpoint=hypothesis.endpoint_display,
+            source_location=source_loc,
+            attack_path=attack_path_nodes,
+            static_evidence=hypothesis.static_evidence,
+            runtime_evidence=runtime_evidence,
+            correlation_notes=correlation_notes,
+            remediation=get_remediation_for_category(category_str),
+            reproduction_steps=test_result.reproduction_steps if test_result else [],
+            observations=test_result.observations if test_result else [],
+        )

package/src/trace_engine/findings/model.py

@@ -0,0 +1,40 @@
+"""Correlated security finding schema."""
+
+from enum import Enum
+from typing import List, Optional, Dict, Any
+from pydantic import BaseModel, Field
+from trace_engine.parsing.locations import SourceLocation
+from trace_engine.runtime.observations import RuntimeObservation
+
+
+class Severity(str, Enum):
+    CRITICAL = "CRITICAL"
+    HIGH = "HIGH"
+    MEDIUM = "MEDIUM"
+    LOW = "LOW"
+    INFO = "INFO"
+
+
+class FindingConfidence(str, Enum):
+    CONFIRMED = "CONFIRMED"
+    HIGH = "HIGH"
+    MEDIUM = "MEDIUM"
+    POTENTIAL = "POTENTIAL"
+
+
+class Finding(BaseModel):
+    """Correlated vulnerability finding supported by both static AST and runtime proof."""
+    id: str
+    title: str
+    category: str
+    severity: Severity
+    confidence: FindingConfidence
+    endpoint: str
+    source_location: Optional[SourceLocation] = None
+    attack_path: List[str] = Field(default_factory=list)
+    static_evidence: List[str] = Field(default_factory=list)
+    runtime_evidence: List[str] = Field(default_factory=list)
+    correlation_notes: str
+    remediation: str
+    reproduction_steps: List[str] = Field(default_factory=list)
+    observations: List[RuntimeObservation] = Field(default_factory=list)

package/src/trace_engine/findings/recommendations.py

@@ -0,0 +1,35 @@
+"""Remediation recommendations for security findings."""
+
+REMEDIATION_MAP = {
+    "BOLA": (
+        "Enforce strict object ownership validation before returning records. "
+        "Query using both the object ID and the authenticated user/tenant ID (e.g. `WHERE id = :id AND user_id = :current_user.id`)."
+    ),
+    "BFLA": (
+        "Implement declarative role-based access control (RBAC) middleware. "
+        "Validate user privileges on the backend for every privileged/administrative route."
+    ),
+    "AUTHENTICATION": (
+        "Protect this route with authentication dependencies (e.g. `Depends(get_current_user)` or security middleware) "
+        "to ensure unauthorized users cannot trigger state-changing or sensitive operations."
+    ),
+    "SSRF": (
+        "Avoid making arbitrary outbound HTTP requests with user-controlled URLs. "
+        "Implement a strict destination domain whitelist and block loopback/internal IP addresses (RFC 1918)."
+    ),
+    "INJECTION": (
+        "Use parameterized queries, prepared statements, or ORM abstraction for all database operations. "
+        "Never format or concatenate raw client inputs into SQL/command strings."
+    ),
+    "MASS_ASSIGNMENT": (
+        "Define explicit request DTO schemas (e.g. Pydantic models with only allowed mutable fields). "
+        "Do not allow raw dictionary binding directly into database models."
+    ),
+}
+
+
+def get_remediation_for_category(category: str) -> str:
+    return REMEDIATION_MAP.get(
+        category.upper(),
+        "Review access control and input validation on this attack path.",
+    )