skillpool 4.3.0__py3-none-any.whl

skillpool/review/async_queue.py

@@ -0,0 +1,96 @@
+"""AsyncReviewQueue — in-memory review queue with cooldown tracking."""
+
+from __future__ import annotations
+
+import time
+import uuid
+from dataclasses import dataclass, field
+
+from skillpool.review.models import ReviewStatus, ReviewTriggerRequest
+
+
+@dataclass
+class _QueueEntry:
+    """Internal tracking entry for a submitted review."""
+
+    review_id: str
+    request: ReviewTriggerRequest
+    status: ReviewStatus = ReviewStatus.QUEUED
+    submitted_at: float = field(default_factory=time.time)
+
+
+class AsyncReviewQueue:
+    """In-memory review queue with cooldown enforcement.
+
+    - max_concurrent: maximum number of reviews that can be PROCESSING at once
+    - cooldown_seconds: minimum time between reviews for the same skill_id
+
+    Usage:
+        queue = AsyncReviewQueue()
+        review_id = queue.submit(request)
+        status = queue.get_status(review_id)
+    """
+
+    def __init__(self, max_concurrent: int = 10, cooldown_seconds: float = 86400.0) -> None:
+        self.max_concurrent = max_concurrent
+        self.cooldown_seconds = cooldown_seconds
+        self._entries: dict[str, _QueueEntry] = {}
+        self._skill_last_review: dict[str, float] = {}
+
+    def submit(self, request: ReviewTriggerRequest) -> str:
+        """Submit a review request. Returns the review_id.
+
+        Raises ValueError if any affected skill is still in cooldown.
+        Raises RuntimeError if max_concurrent reviews are already processing.
+        """
+        # Check cooldown for all affected skills
+        now = time.time()
+        for skill_id in request.affected_skills:
+            last = self._skill_last_review.get(skill_id, 0.0)
+            if now - last < self.cooldown_seconds:
+                remaining = round(self.cooldown_seconds - (now - last), 1)
+                raise ValueError(f"Skill '{skill_id}' is in cooldown ({remaining}s remaining)")
+
+        # Check max concurrent
+        processing_count = sum(1 for e in self._entries.values() if e.status == ReviewStatus.PROCESSING)
+        if processing_count >= self.max_concurrent:
+            raise RuntimeError(f"Max concurrent reviews ({self.max_concurrent}) reached")
+
+        review_id = uuid.uuid4().hex[:16]
+        entry = _QueueEntry(
+            review_id=review_id,
+            request=request,
+            status=ReviewStatus.QUEUED,
+            submitted_at=now,
+        )
+        self._entries[review_id] = entry
+
+        # Mark skill cooldown timestamps
+        for skill_id in request.affected_skills:
+            self._skill_last_review[skill_id] = now
+
+        return review_id
+
+    def get_status(self, review_id: str) -> ReviewStatus:
+        """Get the current status of a review."""
+        entry = self._entries.get(review_id)
+        if entry is None:
+            raise KeyError(f"Review '{review_id}' not found")
+        return entry.status
+
+    def set_status(self, review_id: str, status: ReviewStatus) -> None:
+        """Update the status of a review entry."""
+        entry = self._entries.get(review_id)
+        if entry is None:
+            raise KeyError(f"Review '{review_id}' not found")
+        entry.status = status
+
+    def is_in_cooldown(self, skill_id: str) -> bool:
+        """Check whether a skill is currently in cooldown."""
+        last = self._skill_last_review.get(skill_id, 0.0)
+        return (time.time() - last) < self.cooldown_seconds
+
+    def clear(self) -> None:
+        """Clear all entries and cooldown tracking (for testing)."""
+        self._entries.clear()
+        self._skill_last_review.clear()

skillpool/review/checkpoint_runner.py

@@ -0,0 +1,345 @@
+"""CheckpointRunner — runs dimension scoring for each checkpoint level."""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+from pathlib import Path
+from typing import Callable, Optional
+
+import yaml
+
+from skillpool.config import get_data_dir
+from skillpool.review.models import CheckpointLevel
+
+logger = logging.getLogger(__name__)
+
+# Dimensions per checkpoint level
+SHADOW_DIMENSIONS = ("D1", "D2", "D4", "D6", "D8", "D9", "D12")
+BASELINE_DIMENSIONS = ("D3", "D5", "D7", "D10", "D11")
+ALL_DIMENSIONS = BASELINE_DIMENSIONS + SHADOW_DIMENSIONS
+
+# Dimension → required skills mapping
+DIMENSION_SKILLS: dict[str, list[str]] = {
+    "D1": ["S01"],
+    "D2": ["S02"],
+    "D3": ["S05a", "S05b", "S06"],
+    "D4": ["S04"],
+    "D5": ["S09", "S10"],
+    "D6": ["S07", "S08"],
+    "D7": ["S13a", "S13b"],
+    "D8": ["S11"],
+    "D9": ["S12"],
+    "D10": ["S19"],
+    "D11": ["S20"],
+    "D12": ["S21"],
+}
+
+# Required fields in CSDF YAML for scoring
+REQUIRED_CSDF_FIELDS = {"id", "name", "version", "dimension", "description"}
+
+# Default skills directory
+_DEFAULT_SKILLS_DIR = get_data_dir() / "skills"
+
+
+class CheckpointRunner:
+    """Runs dimension scoring for a given checkpoint level.
+
+    Evaluates each dimension by inspecting the corresponding skill YAML files:
+    - File existence and YAML parseability
+    - Required field completeness
+    - Checklist item count and severity distribution
+
+    Falls back to deterministic hash-based scores when skill files are unavailable.
+    """
+
+    def __init__(self, seed: Optional[int] = None, skills_dir: Optional[Path] = None):
+        self._base_seed = seed
+        self._skills_dir = skills_dir or _DEFAULT_SKILLS_DIR
+
+    def run_checkpoint(
+        self,
+        level: CheckpointLevel,
+        skills: list[str],
+    ) -> dict[str, float]:
+        """Evaluate dimensions for the given checkpoint level.
+
+        Returns a dict of dimension → score (0.0-10.0).
+        """
+        dimensions = self._dimensions_for_level(level)
+        scores: dict[str, float] = {}
+        for d in dimensions:
+            scores[d] = self._score_dimension(d)
+        return scores
+
+    def _dimensions_for_level(self, level: CheckpointLevel) -> tuple[str, ...]:
+        """Return the dimensions to evaluate for each checkpoint level."""
+        if level == CheckpointLevel.L1:
+            return SHADOW_DIMENSIONS
+        elif level == CheckpointLevel.L2:
+            return ALL_DIMENSIONS
+        elif level == CheckpointLevel.L3:
+            return BASELINE_DIMENSIONS
+        elif level == CheckpointLevel.L4:
+            return BASELINE_DIMENSIONS
+        return ALL_DIMENSIONS
+
+    def _score_dimension(self, dimension: str) -> float:
+        """Score a single dimension by evaluating its associated skill files."""
+        skill_ids = DIMENSION_SKILLS.get(dimension, [])
+        if not skill_ids:
+            return self._fallback_score(dimension)
+
+        scores: list[float] = []
+        for sid in skill_ids:
+            s = self._score_skill(sid)
+            scores.append(s)
+
+        return round(sum(scores) / len(scores), 2) if scores else self._fallback_score(dimension)
+
+    def _score_skill(self, skill_id: str) -> float:
+        """Score a single skill based on CSDF file quality."""
+        yaml_path = self._find_skill_yaml(skill_id)
+        if yaml_path is None:
+            return self._fallback_score(skill_id)
+
+        # Parse YAML
+        try:
+            content = yaml_path.read_text(encoding="utf-8")
+            data = yaml.safe_load(content)
+        except (yaml.YAMLError, OSError):
+            return 3.0  # Unparseable YAML → low score
+
+        if not isinstance(data, dict):
+            return 2.0
+
+        # Determine which dimension this skill primarily serves
+        primary_dim = self._skill_dimension(skill_id)
+
+        # Use dimension-specific scoring if available
+        if primary_dim:
+            dim_scorer = self._DIMENSION_SCORERS.get(primary_dim)
+            if dim_scorer:
+                return dim_scorer(self, data, content)
+
+        # Default: generic quality scoring
+        return self._score_generic(data)
+
+    def _skill_dimension(self, skill_id: str) -> Optional[str]:
+        """Return the primary dimension for a skill ID."""
+        for dim, skills in DIMENSION_SKILLS.items():
+            if skill_id in skills:
+                return dim
+        return None
+
+    # ── Dimension-specific scorers ──
+
+    def _score_d3_security(self, data: dict, raw_content: str) -> float:
+        """D3 安全合规性: SecurityScanner scan + CSDF field quality."""
+        score = self._score_generic(data)
+        # Bonus: SecurityScanner scan passes
+        try:
+            from skillpool.hooks.security_scanner import SecurityScanner
+
+            scanner = SecurityScanner()
+            result = scanner.full_check(raw_content)
+            if result.is_safe:
+                score += 2.0
+            elif result.threat_level.value == "warning":
+                score += 0.5
+            # CRITICAL means is_safe=False → no bonus
+        except Exception as e:
+            logger.warning("Security scanner unavailable, skipping scan bonus: %s", e)
+
+        # Bonus: explicit security fields in CSDF
+        if data.get("security_scan_required"):
+            score += 0.5
+        if data.get("veto_rule"):
+            score += 0.5
+
+        return min(round(score, 2), 10.0)
+
+    def _score_d5_resilience(self, data: dict, raw_content: str) -> float:
+        """D5 弹性容错: Degradation config + recovery strategy."""
+        score = self._score_generic(data)
+
+        # Bonus: degradation/fallback declared
+        if data.get("fallback") or data.get("degradation"):
+            score += 1.5
+        # Bonus: recovery/retry declared
+        if data.get("recovery") or data.get("retry_strategy"):
+            score += 1.0
+        # Bonus: circuit breaker or timeout declared
+        if data.get("circuit_breaker") or data.get("timeout"):
+            score += 0.5
+
+        # Penalty: no resilience fields at all
+        resilience_fields = {
+            "fallback",
+            "degradation",
+            "recovery",
+            "retry_strategy",
+            "circuit_breaker",
+            "timeout",
+            "grace_period",
+        }
+        if not any(k in data for k in resilience_fields):
+            # Check checklist for resilience-related items
+            checklist = data.get("checklist", [])
+            has_resilience_item = any(
+                isinstance(item, dict)
+                and any(
+                    kw in str(item.get("description", "")).lower()
+                    for kw in ("fallback", "degrad", "recover", "retry", "timeout")
+                )
+                for item in checklist
+            )
+            if not has_resilience_item:
+                score -= 1.0
+
+        return min(max(round(score, 2), 0.0), 10.0)
+
+    def _score_d7_testability(self, data: dict, raw_content: str) -> float:
+        """D7 可测试性: Test coverage + BDD specification."""
+        score = self._score_generic(data)
+
+        # Bonus: test_coverage field declared
+        coverage = data.get("test_coverage")
+        if isinstance(coverage, (int, float)):
+            if coverage >= 90:
+                score += 2.0
+            elif coverage >= 80:
+                score += 1.5
+            elif coverage >= 70:
+                score += 1.0
+            else:
+                score += 0.5
+        elif coverage:
+            score += 0.5
+
+        # Bonus: BDD/acceptance criteria in checklist
+        checklist = data.get("checklist", [])
+        bdd_keywords = {"given", "when", "then", "should", "must", "verify"}
+        bdd_count = sum(
+            1
+            for item in checklist
+            if isinstance(item, dict) and any(kw in str(item.get("description", "")).lower() for kw in bdd_keywords)
+        )
+        if bdd_count >= 3:
+            score += 1.5
+        elif bdd_count >= 1:
+            score += 0.5
+
+        return min(round(score, 2), 10.0)
+
+    def _score_d10_protocol(self, data: dict, raw_content: str) -> float:
+        """D10 协议时效性: Version freshness + deprecation status."""
+        score = self._score_generic(data)
+
+        # Bonus: version follows semver
+        version = str(data.get("version", ""))
+        import re
+
+        if re.match(r"\d+\.\d+\.\d+", version):
+            score += 0.5
+
+        # Bonus: last_updated or effective_date declared
+        if data.get("last_updated") or data.get("effective_date"):
+            score += 0.5
+
+        # Penalty: deprecated without replacement
+        if data.get("deprecated"):
+            replacement = data.get("replacement")
+            if not replacement:
+                score -= 1.0
+
+        return min(max(round(score, 2), 0.0), 10.0)
+
+    def _score_d11_feasibility(self, data: dict, raw_content: str) -> float:
+        """D11 工程可行性: Dependency completeness + implementation hints."""
+        score = self._score_generic(data)
+
+        # Bonus: dependencies declared (even empty = explicitly checked)
+        if "dependencies" in data:
+            score += 0.5
+            deps = data.get("dependencies", [])
+            if isinstance(deps, list) and deps:
+                score += 0.5  # Has actual dependencies documented
+
+        # Bonus: implementation hints
+        if data.get("implementation") or data.get("implementation_notes"):
+            score += 1.0
+
+        # Penalty: no dependencies field and no checklist
+        if "dependencies" not in data and not data.get("checklist"):
+            score -= 1.0
+
+        return min(max(round(score, 2), 0.0), 10.0)
+
+    # Dimension → scorer mapping
+    _DIMENSION_SCORERS: dict[str, Callable] = {
+        "D3": _score_d3_security,
+        "D5": _score_d5_resilience,
+        "D7": _score_d7_testability,
+        "D10": _score_d10_protocol,
+        "D11": _score_d11_feasibility,
+    }
+
+    def _score_generic(self, data: dict) -> float:
+        """Generic skill quality scoring (field completeness + checklist)."""
+        score = 0.0
+
+        # 1. Required field completeness (0-4 points)
+        present = REQUIRED_CSDF_FIELDS & set(data.keys())
+        field_ratio = len(present) / len(REQUIRED_CSDF_FIELDS) if REQUIRED_CSDF_FIELDS else 1.0
+        score += 4.0 * field_ratio
+
+        # 2. Checklist quality (0-3 points)
+        checklist = data.get("checklist", [])
+        if isinstance(checklist, list) and checklist:
+            item_count = len(checklist)
+            if item_count >= 8:
+                score += 3.0
+            elif item_count >= 5:
+                score += 2.0
+            elif item_count >= 2:
+                score += 1.0
+            else:
+                score += 0.5
+
+            severities = [item.get("severity", "") for item in checklist if isinstance(item, dict)]
+            if "critical" in severities:
+                score += 0.5
+        else:
+            score += 0.3
+
+        # 3. Description quality (0-1.5 points)
+        desc = data.get("description", "")
+        if desc and len(str(desc).strip()) > 20:
+            score += 1.5
+        elif desc:
+            score += 0.7
+
+        # 4. Weight/veto declaration (0-1 point)
+        if "weight" in data:
+            score += 0.5
+        if "veto_rule" in data:
+            score += 0.5
+
+        return min(round(score, 2), 10.0)
+
+    def _find_skill_yaml(self, skill_id: str) -> Optional[Path]:
+        """Find the CSDF YAML file for a skill ID."""
+        if not self._skills_dir.exists():
+            return None
+        for p in self._skills_dir.iterdir():
+            if p.name.startswith(f"{skill_id}-") and p.suffix == ".yaml":
+                return p
+        return None
+
+    def _fallback_score(self, key: str) -> float:
+        """Generate a deterministic fallback score when skill files are unavailable."""
+        seed = self._base_seed or int(hashlib.sha256(key.encode()).hexdigest()[:8], 16)
+        combined = f"{key}:{seed}".encode()
+        h = int(hashlib.sha256(combined).hexdigest()[:8], 16)
+        return round(5.0 + (h % 5000) / 1000.0, 2)

skillpool/review/models.py

@@ -0,0 +1,164 @@
+"""Review models — Pydantic schemas for multi-dimension review pipeline.
+
+Aligned with contracts/sdd/review-trigger-spec.yaml
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field
+
+
+class ReviewTrigger(StrEnum):
+    """What triggered this review."""
+
+    L3_REGRESSION_FAIL = "l3_regression_fail"
+    L4_E2E_FAIL = "l4_e2e_fail"
+    L5_ERROR_BUDGET_BURN = "l5_error_budget_burn"
+    MANUAL = "manual"
+    SCHEDULED_DIMENSION_SCAN = "scheduled_dimension_scan"
+
+
+class CheckpointLevel(StrEnum):
+    """Review checkpoint level — determines which dimensions are evaluated."""
+
+    L1 = "L1"
+    L2 = "L2"
+    L3 = "L3"
+    L4 = "L4"
+
+
+# Checkpoint SLA timeouts (per spec)
+CHECKPOINT_SLA_TIMEOUTS: dict[CheckpointLevel, float] = {
+    CheckpointLevel.L1: 10.0,  # seconds
+    CheckpointLevel.L2: 60.0,
+    CheckpointLevel.L3: 120.0,
+    CheckpointLevel.L4: 300.0,
+}
+
+
+class VetoRule(StrEnum):
+    """Veto rules V1-V6 with human-readable conditions."""
+
+    V1 = "V1"  # D3 < 7.0 → block
+    V2 = "V2"  # D5 < 7.0 → block
+    V3 = "V3"  # D7 < 7.5 → block
+    V4 = "V4"  # D11 < 6.0 → block
+    V5 = "V5"  # D10 < 5.5 → risk_notice (not block)
+    V6 = "V6"  # baseline_avg < 7.5 → veto_explanation
+
+
+class ReviewStatus(StrEnum):
+    """Status of a review execution."""
+
+    COMPLETED = "completed"
+    PARTIAL = "partial"
+    FAILED = "failed"
+    QUEUED = "queued"
+    PROCESSING = "processing"
+
+
+class UpgradeRecommendation(StrEnum):
+    """Recommended upgrade action based on review results."""
+
+    PATCH = "PATCH"
+    MINOR = "MINOR"
+    MAJOR = "MAJOR"
+    NONE = "NONE"
+
+
+class FailedTestDetail(BaseModel):
+    """Structured failed test information (per schema)."""
+
+    test_name: str
+    expected: str = ""
+    actual: str = ""
+    skill_id: str = ""
+    duration_ms: float = 0.0
+
+
+class VetoDetail(BaseModel):
+    """Detail of a single veto rule evaluation."""
+
+    rule: VetoRule
+    dimension: str
+    score: float
+    threshold: float
+    blocks: bool = Field(description="Whether this veto blocks admission")
+    recommendation: str = Field(default="", description="Action recommendation")
+
+
+class SuspectSkill(BaseModel):
+    """A skill marked as suspect during review."""
+
+    skill_id: str
+    reason: str
+    suspected_dimension: str = ""
+
+
+class BlindSpotFound(BaseModel):
+    """New blind spot discovered during review."""
+
+    id: str
+    description: str
+    dimension: str
+    severity: str = "P2"
+
+
+class ReviewTriggerRequest(BaseModel):
+    """Request to trigger a multi-dimension review.
+
+    Aligned with contracts/sdd/review-trigger-spec.yaml:
+    - trace_id: W3C TraceContext
+    - failed_tests: structured objects (not flat strings)
+    - baseline_metrics: before/after metrics for comparison
+    - pipeline_url: CI pipeline reference
+    """
+
+    trigger: ReviewTrigger
+    checkpoint: CheckpointLevel
+    affected_skills: list[str] = Field(min_length=1, description="Skill IDs under review")
+    failed_tests: Optional[list[str]] = Field(default=None, description="Test IDs that failed (backward compat)")
+    failed_test_details: list[FailedTestDetail] = Field(default_factory=list, description="Structured failed test info")
+    trace_id: str = Field(default="", description="W3C TraceContext trace_id")
+    baseline_metrics: dict[str, float] = Field(
+        default_factory=dict, description="Baseline metrics (previous_recall, current_recall, etc.)"
+    )
+    pipeline_url: str = Field(default="", description="CI pipeline URL for audit")
+
+    def get_all_failed_tests(self) -> list[str]:
+        """Get all failed test names (from both flat and structured sources)."""
+        flat = self.failed_tests or []
+        structured = [d.test_name for d in self.failed_test_details]
+        # Merge, dedup
+        return list(dict.fromkeys(flat + structured))
+
+
+class ReviewTriggerResponse(BaseModel):
+    """Response from a review trigger execution.
+
+    Aligned with contracts/sdd/review-trigger-spec.yaml:
+    - new_blind_spots: blind spots discovered
+    - estimated_cost: token/cost estimate
+    - merkle_commit: ClawMem savepoint hash
+    - retry_after_seconds: for async polling
+    """
+
+    review_id: str
+    status: ReviewStatus
+    checkpoint: CheckpointLevel
+    scores: dict[str, float] = Field(default_factory=dict, description="Dimension → score (D1-D12)")
+    veto_triggered: bool = False
+    veto_details: list[VetoDetail] = Field(default_factory=list)
+    suspect_skills: list[SuspectSkill] = Field(default_factory=list)
+    recommendation: UpgradeRecommendation = UpgradeRecommendation.NONE
+    duration_ms: float = 0.0
+    # Schema-aligned fields
+    new_blind_spots: list[BlindSpotFound] = Field(default_factory=list, description="New blind spots discovered")
+    estimated_cost: dict[str, Any] = Field(
+        default_factory=dict, description="Cost estimate {review_tokens, review_cost_usd}"
+    )
+    merkle_commit: str = Field(default="", description="ClawMem SAVEPOINT hash")
+    retry_after_seconds: float = Field(default=0.0, description="Seconds until async result available (0 = sync)")

skillpool/review/suspect_marker.py

@@ -0,0 +1,39 @@
+"""SuspectMarker — tracks skills marked as suspect during review."""
+
+from __future__ import annotations
+
+from skillpool.review.models import SuspectSkill
+
+
+class SuspectMarker:
+    """Tracks skills marked as suspect during review.
+
+    Usage:
+        marker = SuspectMarker()
+        marker.mark("S05a", reason="D3 below threshold", suspected_dimension="D3")
+        assert marker.is_suspect("S05a")
+        marker.clear()
+    """
+
+    def __init__(self) -> None:
+        self._suspects: dict[str, SuspectSkill] = {}
+
+    def mark(self, skill_id: str, reason: str, suspected_dimension: str = "") -> None:
+        """Add a skill to the suspect set."""
+        self._suspects[skill_id] = SuspectSkill(
+            skill_id=skill_id,
+            reason=reason,
+            suspected_dimension=suspected_dimension,
+        )
+
+    def is_suspect(self, skill_id: str) -> bool:
+        """Check whether a skill is currently marked as suspect."""
+        return skill_id in self._suspects
+
+    def clear(self) -> None:
+        """Remove all skills from the suspect set."""
+        self._suspects.clear()
+
+    def list_suspects(self) -> list[SuspectSkill]:
+        """Return all currently marked suspect skills."""
+        return list(self._suspects.values())