skillpool 4.3.0__py3-none-any.whl

skillpool/graph/ppr.py

@@ -0,0 +1,279 @@
+"""Personalized PageRank (PPR) — Three-layer implementation.
+
+V1.1 Section 8.5 compliance:
+  Layer 1: Pure Python — correctness verification only, not for production
+  Layer 2: SciPy CSR sparse matrix — CPU production path for medium graphs
+  Layer 3: scikit-network PageRank — fallback for large graphs when available
+
+Unified interface:
+    personalized_pagerank(adj, seeds, alpha=0.85, epsilon=1e-6, top_k=None)
+
+Performance acceptance criteria (must record hardware + params):
+  - 10K nodes × 50K edges: CSR path < 10ms
+  - 100K nodes: CSR path < 50ms (P95), requires documented benchmarks
+"""
+
+from __future__ import annotations
+
+__all__ = ["personalized_pagerank", "reverse_ppr"]
+
+import numpy as np
+from scipy import sparse as sp
+
+
+def _validate_inputs(
+    adj: sp.spmatrix,
+    seeds: list[int] | dict[int, float],
+) -> tuple[int, np.ndarray]:
+    """
+    Validate adjacency matrix and normalize seeds into a probability vector.
+
+    Args:
+        adj: Sparse adjacency matrix (n x n)
+        seeds: Either list of node indices (uniform weight) or {node: weight} dict
+
+    Returns:
+        Tuple of (n_nodes, seed_vector)
+
+    Raises:
+        ValueError: If adj is not square or seeds contain invalid nodes
+    """
+    n = adj.shape[0]
+    if adj.shape[1] != n:
+        raise ValueError(f"Adjacency must be square, got {adj.shape}")
+
+    seed_vec = np.zeros(n, dtype=np.float64)
+    if isinstance(seeds, dict):
+        total_w = sum(seeds.values())
+        for node, weight in seeds.items():
+            if node < 0 or node >= n:
+                raise ValueError(f"Seed node {node} out of range [0, {n})")
+            seed_vec[node] = weight / total_w
+    else:
+        for node in seeds:
+            if node < 0 or node >= n:
+                raise ValueError(f"Seed node {node} out of range [0, {n})")
+        seed_vec[list(seeds)] = 1.0 / len(seeds)
+
+    return n, seed_vec
+
+
+# ── Layer 1: Pure Python (correctness baseline) ──
+
+
+def _ppr_push_python(
+    adj: sp.spmatrix,
+    seeds_vec: np.ndarray,
+    alpha: float,
+    epsilon: float,
+    max_iter: int = 200,
+) -> np.ndarray:
+    """
+    Pure Python Push algorithm for PPR — small graph verification only.
+
+    NOT for production use. Use CSR or sknetwork path.
+    """
+    n = len(seeds_vec)
+    r = seeds_vec.copy()
+    x = np.zeros(n, dtype=np.float64)
+
+    if not sp.isspmatrix_csr(adj):
+        adj = adj.tocsr()
+
+    out_degree = np.array(adj.sum(axis=1)).flatten()
+    dangling_mask = out_degree == 0
+
+    for _ in range(max_iter):
+        max_r = r.max()
+        if max_r < epsilon:
+            break
+        for u in range(n):
+            threshold = max(out_degree[u], 1) * epsilon
+            if r[u] > threshold:
+                push_amount = alpha * r[u]
+                x[u] += push_amount
+                remain = r[u] - push_amount
+                r[u] = 0
+
+                if not dangling_mask[u]:
+                    degree_u = out_degree[u]
+                    share = remain / degree_u
+                    row_start = adj.indptr[u]
+                    row_end = adj.indptr[u + 1]
+                    for idx in range(row_start, row_end):
+                        v = adj.indices[idx]
+                        r[v] += share
+                else:
+                    share = remain / n
+                    r += share
+
+    x += r
+    return x
+
+
+# ── Layer 2: SciPy CSR Sparse Matrix (production CPU path) ──
+
+
+def _ppr_csr_power_iteration(
+    adj: sp.spmatrix,
+    seeds_vec: np.ndarray,
+    alpha: float,
+    epsilon: float,
+    max_iter: int = 100,
+) -> np.ndarray:
+    """
+    Power iteration using SciPy CSR sparse matrix for medium graphs.
+
+    This is the recommended production path for graphs up to ~500K nodes.
+    """
+    n = len(seeds_vec)
+    if not sp.isspmatrix_csr(adj):
+        adj = adj.tocsr()
+
+    # Build column-stochastic transition matrix M = D^-1 * A^T
+    out_degree_raw = np.array(adj.sum(axis=1)).flatten().astype(np.float64)
+    dangling_mask = out_degree_raw == 0
+    out_degree = out_degree_raw.copy()
+    out_degree[out_degree == 0] = 1.0
+    d_inv = sp.diags(1.0 / out_degree, format="csr")
+    mt = adj.T.dot(d_inv)  # M^T for left-multiplication
+
+    p_prev = seeds_vec.copy()
+
+    for _iteration in range(max_iter):
+        mt_p = mt.dot(p_prev)
+        p = alpha * seeds_vec + (1.0 - alpha) * mt_p
+
+        # Dangling-node correction
+        if dangling_mask.any():
+            dangling_mass = p_prev[dangling_mask].sum()
+            if dangling_mass > 0:
+                p += (1.0 - alpha) * dangling_mass / n
+
+        delta = np.abs(p - p_prev).sum()
+        if delta < epsilon * n:
+            break
+        p_prev = p
+
+    return p
+
+
+# ── Layer 3: scikit-network (optional, large graphs) ──
+
+
+def _ppr_sknetwork(
+    adj: sp.spmatrix,
+    seeds_vec: np.ndarray,
+    alpha: float,
+) -> np.ndarray:
+    """
+    Use scikit-network's PageRank for large graphs.
+
+    Falls back to CSR power iteration if sknetwork is not installed.
+    """
+    try:
+        from sknetwork.ranking import PageRank
+
+        pr = PageRank(damping_factor=alpha, solver="piteration")
+        scores = pr.fit_transform(adj)
+        if seeds_vec.sum() > 0:
+            scores = alpha * seeds_vec + (1 - alpha) * scores
+        return scores
+    except ImportError:
+        return _ppr_csr_power_iteration(adj, seeds_vec, alpha, 1e-6)
+
+
+# ── Unified Public API ──
+
+
+def personalized_pagerank(
+    adj: sp.spmatrix,
+    seeds: list[int] | dict[int, float],
+    alpha: float = 0.85,
+    epsilon: float = 1e-6,
+    top_k: int | None = None,
+    method: str = "auto",
+) -> np.ndarray:
+    """
+    Compute Personalized PageRank scores for seed nodes.
+
+    Args:
+        adj: Sparse adjacency matrix (n x n), scipy.sparse format
+        seeds: Seed nodes — list (uniform weight) or {node: weight} dict
+        alpha: Teleport probability (damping factor), default 0.85
+        epsilon: Convergence tolerance, default 1e-6
+        top_k: If set, return only top-k scores (approximate for large graphs)
+        method: "python" | "csr" | "sknetwork" | "auto" (default: auto-select)
+
+    Returns:
+        PPR score vector (n,)
+
+    Raises:
+        ValueError: For invalid inputs
+    """
+    n, seeds_vec = _validate_inputs(adj, seeds)
+
+    # Auto-select method based on graph size
+    if method == "auto":
+        if n < 1000:
+            method = "python"
+        elif n < 500000:
+            method = "csr"
+        else:
+            method = "sknetwork"
+
+    if method == "python":
+        scores = _ppr_push_python(adj, seeds_vec, alpha, epsilon)
+    elif method == "csr":
+        scores = _ppr_csr_power_iteration(adj, seeds_vec, alpha, epsilon)
+    elif method == "sknetwork":
+        scores = _ppr_sknetwork(adj, seeds_vec, alpha)
+    else:
+        raise ValueError(f"Unknown method: {method}")
+
+    # Normalize
+    s = scores.sum()
+    if s > 0:
+        scores /= s
+
+    if top_k is not None:
+        indices = np.argpartition(scores, -top_k)[-top_k:]
+        mask = np.zeros(n, dtype=np.float64)
+        mask[indices] = scores[indices]
+        mask /= mask.sum()
+        return mask
+
+    return scores
+
+
+# ── Reverse PPR (V1.1 Section 8.5 target-centric query path) ──
+
+
+def reverse_ppr(
+    adj: sp.spmatrix,
+    target: int,
+    alpha: float = 0.85,
+    epsilon: float = 1e-6,
+) -> np.ndarray:
+    """
+    Compute Reverse PPR for a target node.
+
+    Equivalent to running PPR on the transpose graph with the target as seed.
+    Used for "which skills contribute most to this skill?" queries.
+
+    Args:
+        adj: Sparse adjacency matrix (n x n)
+        target: Target node index
+        alpha: Teleport probability
+        epsilon: Convergence tolerance
+
+    Returns:
+        Reverse PPR score vector (n,)
+    """
+    return personalized_pagerank(
+        adj.T.tocsr() if hasattr(adj, "T") else adj.transpose().tocsr(),
+        [target],
+        alpha=alpha,
+        epsilon=epsilon,
+        method="csr",
+    )

skillpool/health/__init__.py

@@ -0,0 +1,73 @@
+"""Health module — component health checking and degradation management."""
+
+from __future__ import annotations
+
+from skillpool.health.check import HealthChecker
+from skillpool.health.degradation import DegradationManager
+from skillpool.health.models import (
+    ComponentHealth as ComponentHealth,
+    DegradationLevel,
+    HealthCheckResponse,
+    ServingStatus,
+)
+
+
+class HealthManager:
+    """Unified health management: checking + degradation + monitoring.
+
+    When a MonitorLayer is provided, health check results feed into the
+    monitoring system's metrics and five-dimension evaluation.
+
+    Usage:
+        hm = HealthManager()
+        hm.register_component("resolver")
+        response = hm.check_health()
+        level = hm.get_degradation_level()
+    """
+
+    def __init__(self, critical_threshold: int = 2, monitor=None) -> None:
+        self.checker = HealthChecker()
+        self.degradation = DegradationManager(critical_threshold=critical_threshold)
+        self._monitor = monitor
+
+    def register_component(
+        self,
+        name: str,
+        check_fn: callable = None,
+        critical: bool = True,
+    ) -> None:
+        """Register a component for health monitoring."""
+        self.checker.register(name, check_fn=check_fn, critical=critical)
+
+    def check_health(self) -> HealthCheckResponse:
+        """Run health checks and update degradation state + monitor."""
+        response = self.checker.check()
+        # Update degradation based on results
+        for comp in response.components:
+            if comp.status == ServingStatus.NOT_SERVING:
+                # Check if component is critical
+                comp_config = self.checker._components.get(comp.component, {})
+                is_critical = comp_config.get("critical", True)
+                self.degradation.report_failure(comp.component, critical=is_critical)
+            else:
+                self.degradation.report_recovery(comp.component)
+
+        # Update response with degradation level
+        response.degradation_level = self.degradation.get_degradation_level()
+
+        # Feed results to monitor layer if available
+        if self._monitor is not None:
+            from skillpool.monitor import MetricType
+
+            for comp in response.components:
+                status_val = 1.0 if comp.status == ServingStatus.SERVING else 0.0
+                self._monitor.record_metric(
+                    name=f"health.{comp.component}",
+                    value=status_val,
+                    metric_type=MetricType.GAUGE,
+                    labels={"component": comp.component},
+                )
+        return response
+
+    def get_degradation_level(self) -> DegradationLevel:
+        return self.degradation.get_degradation_level()

skillpool/health/check.py

@@ -0,0 +1,85 @@
+"""Health check — component health assessment."""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+from typing import Optional
+
+from skillpool.health.models import (
+    ComponentHealth,
+    HealthCheckResponse,
+    ServingStatus,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class HealthChecker:
+    """Assess health of skillpool components.
+
+    Usage:
+        checker = HealthChecker()
+        checker.register("resolver", check_fn=lambda: True)
+        response = checker.check()
+    """
+
+    def __init__(self) -> None:
+        self._components: dict[str, dict] = {}
+
+    def register(
+        self,
+        name: str,
+        check_fn: Optional[callable] = None,
+        critical: bool = True,
+    ) -> None:
+        """Register a component for health checking."""
+        self._components[name] = {
+            "check_fn": check_fn,
+            "critical": critical,
+            "last_status": ServingStatus.SERVING,
+        }
+
+    def check(self) -> HealthCheckResponse:
+        """Run health checks on all registered components."""
+        results = []
+        overall = ServingStatus.SERVING
+
+        for name, config in self._components.items():
+            check_fn = config.get("check_fn")
+            if check_fn is None:
+                comp_status = ServingStatus.SERVING
+            else:
+                try:
+                    healthy = check_fn()
+                    comp_status = ServingStatus.SERVING if healthy else ServingStatus.NOT_SERVING
+                except Exception as e:
+                    logger.warning("Health check failed for component %s: %s", name, e)
+                    comp_status = ServingStatus.NOT_SERVING
+
+            if comp_status == ServingStatus.NOT_SERVING and config.get("critical", True):
+                overall = ServingStatus.NOT_SERVING
+            elif comp_status == ServingStatus.NOT_SERVING and not config.get("critical", True):
+                if overall == ServingStatus.SERVING:
+                    overall = ServingStatus.DEGRADED
+
+            config["last_status"] = comp_status
+            results.append(
+                ComponentHealth(
+                    component=name,
+                    status=comp_status,
+                )
+            )
+
+        return HealthCheckResponse(
+            status=overall,
+            components=results,
+            timestamp=datetime.now(timezone.utc).isoformat(),
+        )
+
+    def get_component_status(self, name: str) -> ServingStatus:
+        """Get the last known status of a component."""
+        config = self._components.get(name)
+        if config:
+            return config["last_status"]
+        return ServingStatus.NOT_SERVING

skillpool/health/degradation.py

@@ -0,0 +1,90 @@
+"""Degradation management — 4-level fallback mode handling.
+
+Levels (aligned with cross-system-interfaces.yaml §5.2):
+  L0_full:      All components healthy
+  L1_partial:   Non-critical component(s) down, core still functional
+  L2_bm25_only: Vector search (VPLS) unavailable, BM25-only fallback
+  L3_disabled:  Multiple critical failures, minimal/disabled operation
+"""
+
+from __future__ import annotations
+
+from skillpool.health.models import DegradationLevel
+
+
+class DegradationManager:
+    """Manage service degradation levels.
+
+    When components fail, the system degrades gracefully:
+    - L0_full: All components healthy
+    - L1_partial: Non-critical component(s) down
+    - L2_bm25_only: Vector search unavailable, fall back to BM25
+    - L3_disabled: Multiple critical failures, minimal operation
+
+    Usage:
+        dm = DegradationManager()
+        dm.report_failure("vpls")
+        level = dm.get_degradation_level()
+    """
+
+    def __init__(self, critical_threshold: int = 2) -> None:
+        self.critical_threshold = critical_threshold
+        self._failures: dict[str, int] = {}  # component → consecutive failures
+        self._critical_failures: set[str] = set()
+        self._degradation_level = DegradationLevel.L0_FULL
+
+    def report_failure(self, component: str, critical: bool = True) -> DegradationLevel:
+        """Report a component failure. Returns new degradation level."""
+        self._failures[component] = self._failures.get(component, 0) + 1
+        if critical:
+            self._critical_failures.add(component)
+        self._update_level()
+        return self._degradation_level
+
+    def report_recovery(self, component: str) -> DegradationLevel:
+        """Report a component recovery. Returns new degradation level."""
+        self._failures.pop(component, None)
+        self._critical_failures.discard(component)
+        self._update_level()
+        return self._degradation_level
+
+    def get_degradation_level(self) -> DegradationLevel:
+        """Get current degradation level."""
+        return self._degradation_level
+
+    def get_fallback_mode(self) -> str:
+        """Get current fallback mode string."""
+        level = self._degradation_level
+        if level == DegradationLevel.L0_FULL:
+            return "vpls_vector"
+        if level == DegradationLevel.L1_PARTIAL:
+            return "vpls_vector"
+        if level == DegradationLevel.L2_BM25_ONLY:
+            return "bm25_keyword"
+        return "sqlite_fts5"
+
+    def _update_level(self) -> None:
+        """Recalculate degradation level based on failures."""
+        failed_count = len(self._failures)
+        critical_count = len(self._critical_failures)
+
+        if failed_count == 0:
+            self._degradation_level = DegradationLevel.L0_FULL
+        elif "vpls" in self._failures and critical_count < self.critical_threshold:
+            # VPLS down but not too many critical failures → BM25 fallback
+            self._degradation_level = DegradationLevel.L2_BM25_ONLY
+        elif critical_count >= self.critical_threshold:
+            # Too many critical failures → disabled
+            self._degradation_level = DegradationLevel.L3_DISABLED
+        elif failed_count > 0 and critical_count == 0:
+            # Only non-critical failures → partial degradation
+            self._degradation_level = DegradationLevel.L1_PARTIAL
+        else:
+            # Some critical but below threshold → BM25 fallback
+            self._degradation_level = DegradationLevel.L2_BM25_ONLY
+
+    def reset(self) -> None:
+        """Reset all failures and return to L0_full."""
+        self._failures.clear()
+        self._critical_failures.clear()
+        self._degradation_level = DegradationLevel.L0_FULL

skillpool/health/models.py

@@ -0,0 +1,43 @@
+"""Health models — Pydantic schemas for health check."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+from pydantic import BaseModel, Field
+
+
+class ServingStatus(StrEnum):
+    SERVING = "SERVING"
+    NOT_SERVING = "NOT_SERVING"
+    DEGRADED = "DEGRADED"
+
+
+class DegradationLevel(StrEnum):
+    L0_FULL = "L0_full"  # Full functionality
+    L1_PARTIAL = "L1_partial"  # Partial degradation (non-critical components down)
+    L2_BM25_ONLY = "L2_bm25_only"  # BM25-only fallback (vector search down)
+    L3_DISABLED = "L3_disabled"  # Minimal/disabled functionality
+
+
+class ComponentHealth(BaseModel):
+    """Health status of a single component."""
+
+    component: str
+    status: ServingStatus = ServingStatus.SERVING
+    latency_p99_ms: float = 0.0
+    message: str = ""
+    metadata: dict = Field(default_factory=dict)
+    fallback_mode: str = Field(default="", description="Fallback mode: vpls_vector/bm25_keyword/sqlite_fts5")
+
+
+class HealthCheckResponse(BaseModel):
+    """Aggregated health check response."""
+
+    status: ServingStatus = ServingStatus.SERVING
+    components: list[ComponentHealth] = Field(default_factory=list)
+    timestamp: str = ""
+    degradation_level: DegradationLevel = Field(
+        default=DegradationLevel.L0_FULL, description="Current degradation level (L0-L3)"
+    )
+    vpls_latency_p99_ms: float = Field(default=0.0, description="VPLS P99 latency in ms")

skillpool/hooks/__init__.py

@@ -0,0 +1,4 @@
+"""SkillPool hooks package — pre-materialization security checks.
+
+Part of SkillPool — independent infrastructure, shared by all agents.
+"""