skill-sentry-cli 2.0.0__py3-none-any.whl

skill_sentry_cli/core/decisions/mock.py

@@ -0,0 +1,118 @@
+"""Deterministic decision client for unit tests and local wiring checks.
+
+``MockDecisionClient`` is never selected by the production decision selector.
+It exists so tests can exercise the semantic stage without downloading a Laya
+checkpoint.  A mock-only scan must never be presented as live-model evidence.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from skill_sentry_cli.core.decisions.base import (
+    DecisionAnswer,
+    DecisionQuestion,
+    DecisionResult,
+)
+
+_MALICIOUS_MARKERS = (
+    "aws/credentials",
+    "base64",
+    "curl ",
+    "dockerenv",
+    "evil",
+    "exfil",
+    "nslookup",
+    "private key",
+    "token",
+)
+
+
+class MockDecisionClient:
+    """Deterministic typed decisions with ``auto``/``benign``/``malicious`` profiles."""
+
+    backend = "mock"
+    model_version = "mock-laya-v1"
+
+    def __init__(
+        self,
+        profile: str = "auto",
+        *,
+        answers: dict[str, DecisionAnswer] | None = None,
+    ) -> None:
+        if profile not in {"auto", "benign", "malicious"}:
+            raise ValueError(f"unknown MockDecisionClient profile: {profile}")
+        self.profile = profile
+        self._answers = dict(answers or {})
+
+    def decide(
+        self,
+        state: str | dict[str, Any],
+        questions: dict[str, DecisionQuestion],
+        *,
+        timeout_s: float | None = None,
+    ) -> DecisionResult:
+        del timeout_s
+        malicious = self._is_malicious(state)
+        answers: dict[str, DecisionAnswer] = {}
+        for qid, question in questions.items():
+            if qid in self._answers:
+                answers[qid] = self._answers[qid]
+            else:
+                answers[qid] = _mock_answer(qid, question, malicious)
+        return DecisionResult(
+            answers=answers,
+            backend=self.backend,
+            checkpoint="mock",
+            model_version=self.model_version,
+            routing={"model": "mock", "reason": "deterministic test double"},
+            usage={"input_tokens": 0, "output_tokens": 0},
+        )
+
+    def health(self) -> bool:
+        return True
+
+    def close(self) -> None:
+        return None
+
+    def _is_malicious(self, state: str | dict[str, Any]) -> bool:
+        if self.profile == "malicious":
+            return True
+        if self.profile == "benign":
+            return False
+        text = state if isinstance(state, str) else str(state)
+        lowered = text.lower()
+        return any(marker in lowered for marker in _MALICIOUS_MARKERS)
+
+
+def _mock_answer(qid: str, question: DecisionQuestion, malicious: bool) -> DecisionAnswer:
+    if question.type == "choice":
+        criteria = question.criteria if isinstance(question.criteria, dict) else {}
+        keys = list(criteria)
+        if qid == "deception" and "deceptive" in criteria:
+            choice = "deceptive" if malicious else "consistent"
+        elif malicious and "data_export" in criteria:
+            choice = "data_export"
+        elif not malicious and "other" in criteria:
+            choice = "other"
+        else:
+            choice = keys[0] if keys else "other"
+        if len(keys) == 1:
+            probabilities: dict[str, float] = {keys[0]: 1.0}
+        else:
+            rest = (1.0 - 0.9) / max(1, len(keys) - 1)
+            probabilities = {key: (0.9 if key == choice else rest) for key in keys}
+        return DecisionAnswer(choice=choice, probabilities=probabilities, confidence=0.9)
+
+    if question.type == "score":
+        criteria = question.criteria if isinstance(question.criteria, list) else []
+        selected = 1.0 if malicious else 0.0
+        index = 1 if malicious and len(criteria) > 1 else 0
+        rest = (1.0 - 0.9) / max(1, len(criteria) - 1)
+        probabilities = [0.9 if position == index else rest for position in range(len(criteria))]
+        return DecisionAnswer(score=selected, probabilities=probabilities, confidence=0.9)
+
+    return DecisionAnswer(noul=0.9 if malicious else 0.05, confidence=0.9)
+
+
+__all__ = ["MockDecisionClient"]

skill_sentry_cli/core/decisions/questions.py

@@ -0,0 +1,118 @@
+"""Fixed semantic question taxonomy for local decision backends.
+
+The taxonomy is deliberately small.  Large choice sets degrade Laya accuracy
+and make calibration harder.  Add a capability only when there is a
+deterministic evidence source or a clear evaluation case for it.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+from skill_sentry_cli.core.decisions.base import DecisionQuestion, EvidenceCandidate
+
+INTENT_CHOICES: dict[str, str] = {
+    "text_processing": "Summarising, translating, transforming, or generating text.",
+    "file_management": "Reading, writing, moving, or organising files and directories.",
+    "network_access": "Calling APIs, downloading resources, or otherwise using the network.",
+    "credential_access": "Reading, requesting, or transmitting credentials or secrets.",
+    "code_execution": "Running commands, scripts, or arbitrary code.",
+    "ui_automation": "Interacting with a browser, desktop UI, or other graphical surface.",
+    "data_export": "Uploading, exporting, or transferring user or system data.",
+    "other": "None of the categories above is a good fit.",
+}
+
+DECEPTION_CHOICES: dict[str, str] = {
+    "consistent": "The implementation is consistent with the declared purpose.",
+    "unclear": "The available evidence is insufficient to decide.",
+    "deceptive": "The implementation hides or contradicts its declared purpose.",
+}
+
+# Question IDs are stable: they appear in telemetry, calibration files, and
+# report metadata.  Do not rename one without a migration.
+CAPABILITY_QUESTIONS: dict[str, str] = {
+    "cap_credentials": (
+        "Does the skill read, request, or transmit credential material such as "
+        "passwords, API keys, tokens, private keys, or cloud credentials?"
+    ),
+    "cap_env_secrets": (
+        "Does the skill inspect environment variables or shell profiles to discover "
+        "secrets or credentials?"
+    ),
+    "cap_process": "Does the skill spawn processes or execute shell commands?",
+    "cap_network": "Does the skill make outbound network requests?",
+    "cap_upload": "Does the skill send data to a remote endpoint?",
+    "cap_dns": "Does the skill perform DNS lookups or encode data in DNS queries?",
+    "cap_filesystem": (
+        "Does the skill write, modify, or delete files outside its own package directory?"
+    ),
+    "cap_persistence": (
+        "Does the skill modify startup, shell profile, scheduled task, or other "
+        "persistence mechanisms?"
+    ),
+    "cap_obfuscation": (
+        "Does the skill decode, encrypt, or otherwise obfuscate content to hide its behavior?"
+    ),
+    "cap_hidden_behavior": "Does the skill hide behavior from the user or operator?",
+    "cap_prompt_injection": (
+        "Does the skill contain instructions aimed at an auditing agent or language "
+        "model rather than at its own runtime?"
+    ),
+    "cap_deception": "Does the implementation contradict its declared intent?",
+}
+
+
+def adjudication_questions(candidates: Iterable[EvidenceCandidate]) -> dict[str, DecisionQuestion]:
+    """One yes/no question per candidate, keyed ``adj_<candidate_id>``.
+
+    The question names only the candidate's code-owned anchor type.  The code
+    itself travels in the state, never in the instructions.
+    """
+
+    return {
+        f"adj_{candidate.candidate_id}": DecisionQuestion(
+            type="noul",
+            instructions=(
+                f"Is the code candidate {candidate.candidate_id} (anchor type "
+                f"{candidate.anchor_type}) used for a harmful purpose rather than the "
+                "skill's declared function?"
+            ),
+        )
+        for candidate in candidates
+    }
+
+
+def semantic_questions() -> dict[str, DecisionQuestion]:
+    """Return a fresh copy of the V1 semantic question set.
+
+    A fresh mapping is returned so callers cannot mutate the module-level
+    taxonomy while building a request.
+    """
+
+    questions: dict[str, DecisionQuestion] = {
+        "intent": DecisionQuestion(
+            type="choice",
+            instructions="Which category best describes the skill's declared purpose?",
+            criteria=dict(INTENT_CHOICES),
+        ),
+        "deception": DecisionQuestion(
+            type="choice",
+            instructions=(
+                "Does the implementation behave consistently with the declared purpose, "
+                "or does it hide or contradict that purpose?"
+            ),
+            criteria=dict(DECEPTION_CHOICES),
+        ),
+    }
+    for question_id, instructions in CAPABILITY_QUESTIONS.items():
+        questions[question_id] = DecisionQuestion(type="noul", instructions=instructions)
+    return questions
+
+
+__all__ = [
+    "CAPABILITY_QUESTIONS",
+    "DECEPTION_CHOICES",
+    "INTENT_CHOICES",
+    "adjudication_questions",
+    "semantic_questions",
+]

skill_sentry_cli/core/decisions/report.py

@@ -0,0 +1,65 @@
+"""Report metadata for the typed decision layer.
+
+The summary is built from a whitelist of stage-telemetry keys.  It never copies
+prompts, routing payloads, or skill text, so a report cannot leak them even if
+a backend adds fields to its response.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import Any
+
+from skill_sentry_cli.core.contracts import StageResult
+
+_SEMANTIC_KEYS = (
+    "decision_backend",
+    "model_backend",
+    "checkpoint",
+    "model_version",
+    "calibration_version",
+    "instruction_floor",
+)
+_VISION_STATUSES = frozenset({"not_applicable", "evaluated", "partial", "not_evaluated"})
+
+
+def decision_summary(stage_results: Iterable[StageResult]) -> dict[str, Any]:
+    """Return the report-safe decision summary for one scan's stage results."""
+
+    summary: dict[str, Any] = {
+        "decision_backend": "legacy",
+        "checkpoint": None,
+        "model_version": None,
+        "calibration_version": "uncalibrated",
+        "semantic_degraded": False,
+        "semantic_error": None,
+        "threshold": None,
+        "capabilities": {},
+        "vision_status": "not_evaluated",
+    }
+    for result in stage_results:
+        if result.stage.value == "semantic":
+            telemetry = result.telemetry or {}
+            for key in _SEMANTIC_KEYS:
+                if telemetry.get(key) is not None:
+                    summary[key] = telemetry[key]
+            decisions = telemetry.get("decisions") or {}
+            if isinstance(decisions, dict):
+                summary["threshold"] = decisions.get("threshold")
+                capabilities = decisions.get("capabilities")
+                if isinstance(capabilities, dict):
+                    summary["capabilities"] = {
+                        str(key): round(float(value), 4)
+                        for key, value in capabilities.items()
+                        if isinstance(value, (int, float))
+                    }
+            summary["semantic_degraded"] = bool(result.degraded or result.error)
+            summary["semantic_error"] = str(result.error)[:300] if result.error else None
+        elif result.stage.value == "vision":
+            status = (result.telemetry or {}).get("vision_status")
+            if status in _VISION_STATUSES:
+                summary["vision_status"] = status
+    return summary
+
+
+__all__ = ["decision_summary"]

skill_sentry_cli/core/decisions/selector.py

@@ -0,0 +1,151 @@
+"""Decision-backend selection.
+
+This is separate from the legacy generation/vision selector.  ``auto`` prefers
+Laya when the package and router configuration are available, falls back to the
+legacy semantic path only when the caller says it exists, and otherwise returns
+an explicit degraded selection.  A missing required decision backend never
+silently becomes a clean scan.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from skill_sentry_cli.core.config import DecisionSettings, SkillsEntryConfig
+from skill_sentry_cli.core.decisions.base import DecisionClient, DecisionUnavailable
+from skill_sentry_cli.core.decisions.laya import LayaDecisionClient
+
+
+@dataclass(frozen=True)
+class DecisionSelection:
+    """Result of decision-backend selection.
+
+    ``client`` is set only for a typed decision backend.  ``backend`` may be
+    ``laya``, ``legacy``, or ``none`` so callers can route the legacy semantic
+    analyzer separately and record the degraded reason.
+    """
+
+    client: DecisionClient | None
+    backend: str
+    degraded: bool = False
+    reason: str | None = None
+    required: bool = False
+
+    @property
+    def available(self) -> bool:
+        return self.client is not None
+
+
+def select_decision_model(
+    config: SkillsEntryConfig | DecisionSettings | None = None,
+    *,
+    backend: str | None = None,
+    probe: bool = True,
+    laya_client: DecisionClient | None = None,
+    legacy_available: bool = False,
+) -> DecisionSelection:
+    """Return the typed decision backend or an explicit fallback/degradation.
+
+    ``auto`` prefers Laya if it imports and its router can be constructed.  It
+    does not load checkpoint weights; the first :meth:`decide` call does that
+    and any load failure becomes a degraded stage.  Explicit ``laya`` fails
+    closed with :class:`DecisionUnavailable` when the health probe fails.
+    """
+
+    settings = _decision_settings(config)
+    choice = _backend_value(backend if backend is not None else settings.backend)
+    required = settings.required
+
+    if choice == "none":
+        return DecisionSelection(
+            client=None,
+            backend="none",
+            degraded=True,
+            reason="decision backend disabled; semantic analysis is deterministic-only",
+            required=required,
+        )
+    if choice == "legacy":
+        return DecisionSelection(
+            client=None,
+            backend="legacy",
+            degraded=not legacy_available,
+            reason=None if legacy_available else "legacy semantic model is not available",
+            required=required,
+        )
+    if choice not in {"auto", "laya"}:
+        raise ValueError(f"unsupported decision backend: {choice!r}")
+
+    client = laya_client or LayaDecisionClient(settings)
+    healthy = True if not probe else _healthy(client)
+    if healthy:
+        return DecisionSelection(
+            client=client,
+            backend="laya",
+            degraded=False,
+            required=required,
+        )
+
+    error = _failure_detail(client)
+    if choice == "laya":
+        raise DecisionUnavailable(f"laya decision backend is not available: {error}")
+
+    if required:
+        return DecisionSelection(
+            client=None,
+            backend="none",
+            degraded=True,
+            reason=f"laya is required but unavailable: {error}",
+            required=True,
+        )
+    if legacy_available:
+        return DecisionSelection(
+            client=None,
+            backend="legacy",
+            degraded=False,
+            reason=f"laya unavailable; using legacy semantic path: {error}",
+            required=required,
+        )
+    return DecisionSelection(
+        client=None,
+        backend="none",
+        degraded=True,
+        reason=f"no decision backend available: {error}",
+        required=required,
+    )
+
+
+def _decision_settings(
+    config: SkillsEntryConfig | DecisionSettings | None,
+) -> DecisionSettings:
+    if config is None:
+        return DecisionSettings()
+    if isinstance(config, DecisionSettings):
+        return config
+    return config.decision
+
+
+def _backend_value(value: str) -> str:
+    normalised = str(value).strip().lower()
+    return {
+        "laya-local": "laya",
+        "local": "laya",
+        "off": "none",
+        "disabled": "none",
+    }.get(normalised, normalised)
+
+
+def _healthy(client: DecisionClient) -> bool:
+    try:
+        return bool(client.health())
+    except Exception:
+        return False
+
+
+def _failure_detail(client: DecisionClient) -> str:
+    detail = getattr(client, "last_error", None)
+    if isinstance(detail, str) and detail.strip():
+        return detail
+    return "health probe failed"
+
+
+__all__ = ["DecisionSelection", "select_decision_model"]