devsper 2.1.6__py3-none-any.whl

devsper/cluster/registry.py

@@ -0,0 +1,71 @@
+"""Cluster registry backed by Redis hash."""
+
+from devsper.cluster.node_info import NodeInfo, NodeRole
+
+REGISTRY_KEY_PREFIX = "devsper:cluster:"
+REGISTRY_NODES_SUFFIX = ":nodes"
+REGISTRY_TTL = 60
+
+
+class ClusterRegistry:
+    """Backed by Redis hash: devsper:cluster:{run_id}:nodes. TTL 60s refreshed by heartbeat."""
+
+    def __init__(self, redis_client: object, run_id: str) -> None:
+        self._redis = redis_client
+        self._run_id = run_id
+        self._key = f"{REGISTRY_KEY_PREFIX}{run_id}{REGISTRY_NODES_SUFFIX}"
+
+    async def register(self, node: NodeInfo) -> None:
+        await self._redis.hset(self._key, node.node_id, node.to_json())
+        await self._redis.expire(self._key, REGISTRY_TTL)
+
+    async def heartbeat(self, node_id: str, updates: dict) -> None:
+        node = await self.get_node(node_id)
+        if node is None:
+            return
+        d = node.to_dict()
+        d.update(updates)
+        updated = NodeInfo.from_dict(d)
+        await self._redis.hset(self._key, node_id, updated.to_json())
+        await self._redis.expire(self._key, REGISTRY_TTL)
+
+    async def deregister(self, node_id: str) -> None:
+        await self._redis.hdel(self._key, node_id)
+
+    async def get_all(self) -> list[NodeInfo]:
+        raw = await self._redis.hgetall(self._key)
+        out: list[NodeInfo] = []
+        for _k, v in raw.items():
+            val = v.decode("utf-8") if isinstance(v, bytes) else v
+            try:
+                out.append(NodeInfo.from_json(val))
+            except Exception:
+                pass
+        return out
+
+    async def get_node(self, node_id: str) -> NodeInfo | None:
+        raw = await self._redis.hget(self._key, node_id)
+        if raw is None:
+            return None
+        val = raw.decode("utf-8") if isinstance(raw, bytes) else raw
+        try:
+            return NodeInfo.from_json(val)
+        except Exception:
+            return None
+
+    async def get_controllers(self) -> list[NodeInfo]:
+        all_nodes = await self.get_all()
+        return [
+            n for n in all_nodes
+            if n.role in (NodeRole.CONTROLLER, NodeRole.HYBRID)
+        ]
+
+    async def get_workers(self) -> list[NodeInfo]:
+        all_nodes = await self.get_all()
+        return [
+            n for n in all_nodes
+            if n.role in (NodeRole.WORKER, NodeRole.HYBRID)
+        ]
+
+    async def is_healthy(self) -> bool:
+        return len(await self.get_controllers()) >= 1

devsper/cluster/router.py

@@ -0,0 +1,117 @@
+"""Task routing with affinity: memory, tool cache, load."""
+
+from devsper.cluster.node_info import NodeInfo
+from devsper.types.task import Task
+
+
+def _parse_version(version: str) -> tuple[int, int]:
+    """Return (major, minor) from version string like 1.10 or 1.9.0."""
+    parts = version.replace("-", ".").split(".")
+    try:
+        major = int(parts[0]) if parts else 0
+        minor = int(parts[1]) if len(parts) > 1 else 0
+    except (ValueError, IndexError):
+        major, minor = 0, 0
+    return (major, minor)
+
+
+def _clamp(value: float, lo: float, hi: float) -> float:
+    return max(lo, min(hi, value))
+
+
+class TaskRouter:
+    """Route ready tasks to the best available worker. Scoring: memory_affinity, tool_affinity, load_score."""
+
+    # Weights
+    MEMORY_WEIGHT = 0.35
+    TOOL_WEIGHT = 0.25
+    LOAD_WEIGHT = 0.40
+
+    def __init__(self, controller_version: str = "1.9.0") -> None:
+        self._controller_version = controller_version
+        self._ctrl_major, self._ctrl_minor = _parse_version(controller_version)
+        self._round_robin_index: int = 0  # spread tasks when scores tie
+
+    def route(
+        self,
+        task: Task,
+        workers: list[NodeInfo],
+        worker_stats: dict[str, dict],
+    ) -> NodeInfo | None:
+        """Score all workers; return highest-scored; when scores tie, round-robin so tasks spread."""
+        if not workers:
+            return None
+        eligible = [w for w in workers if self._version_compatible(w)]
+        if not eligible:
+            return None
+        scored = [(self._score(task, w, worker_stats.get(w.node_id, {})), w) for w in eligible]
+        best_score = max(s[0] for s in scored)
+        if best_score < 0:
+            return None
+        # Treat scores within 1e-6 as tied so we round-robin and spread (avoids one worker getting all + 429)
+        eps = 1e-6
+        tied = [w for s, w in scored if abs(s - best_score) <= eps]
+        # Round-robin among tied workers so multiple tasks go to different workers
+        idx = self._round_robin_index % len(tied)
+        self._round_robin_index += 1
+        return tied[idx]
+
+    def _version_compatible(self, worker: NodeInfo) -> bool:
+        maj, min_ = _parse_version(worker.version)
+        return (maj == self._ctrl_major and min_ == self._ctrl_minor)
+
+    def _score(
+        self,
+        task: Task,
+        worker: NodeInfo,
+        stats: dict,
+    ) -> float:
+        memory_affinity = self._memory_affinity(task, stats)
+        tool_affinity = self._tool_affinity(task, stats)
+        load_score = self._load_score(worker, stats)
+        return (
+            memory_affinity * self.MEMORY_WEIGHT
+            + tool_affinity * self.TOOL_WEIGHT
+            + load_score * self.LOAD_WEIGHT
+        )
+
+    def _memory_affinity(self, task: Task, stats: dict) -> float:
+        deps = list(task.dependencies or [])
+        if not deps:
+            return 0.0
+        completed_here = list(stats.get("completed_task_ids", []))
+        hit = sum(1 for d in deps if d in completed_here)
+        return hit / len(deps)
+
+    def _tool_affinity(self, task: Task, stats: dict) -> float:
+        tools_needed = self._tools_for_task(task)
+        if not tools_needed:
+            return 0.0
+        cached = set(stats.get("cached_tools", []))
+        hit = sum(1 for t in tools_needed if t in cached)
+        return hit / len(tools_needed)
+
+    def _tools_for_task(self, task: Task) -> list[str]:
+        try:
+            from devsper.tools.selector import get_tools_for_task
+            from devsper.tools.scoring import get_default_score_store
+            score_store = get_default_score_store()
+        except Exception:
+            score_store = None
+        try:
+            tools = get_tools_for_task(
+                task.description or "",
+                role=getattr(task, "role", None),
+                score_store=score_store,
+            )
+            return [t.name for t in tools]
+        except Exception:
+            return []
+
+    def _load_score(self, worker: NodeInfo, stats: dict) -> float:
+        active = int(stats.get("active_tasks", 0))
+        avg_duration = float(stats.get("avg_task_duration_seconds", 0.0))
+        max_workers = worker.max_workers or 1
+        weighted_load = active * avg_duration
+        load_ratio = weighted_load / (max_workers * 60.0)
+        return 1.0 - _clamp(load_ratio, 0.0, 1.0)

devsper/cluster/state_backend.py

@@ -0,0 +1,105 @@
+"""Shared state backend for scheduler snapshots (Redis or filesystem)."""
+
+import json
+import os
+from abc import ABC, abstractmethod
+
+
+class StateBackend(ABC):
+    """Abstract backend for saving/loading scheduler snapshots."""
+
+    @abstractmethod
+    async def save_snapshot(self, run_id: str, snapshot: dict) -> None:
+        ...
+
+    @abstractmethod
+    async def load_snapshot(self, run_id: str) -> dict | None:
+        ...
+
+    @abstractmethod
+    async def delete_snapshot(self, run_id: str) -> None:
+        ...
+
+    @abstractmethod
+    async def list_snapshots(self) -> list[str]:
+        ...
+
+
+class RedisStateBackend(StateBackend):
+    """Store snapshots in Redis. Key: devsper:snapshot:{run_id}, set index: devsper:snapshots."""
+
+    def __init__(self, redis_client: object) -> None:
+        self._redis = redis_client
+        self._key_prefix = "devsper:snapshot:"
+        self._index_key = "devsper:snapshots"
+
+    async def save_snapshot(self, run_id: str, snapshot: dict) -> None:
+        key = f"{self._key_prefix}{run_id}"
+        await self._redis.set(key, json.dumps(snapshot))
+        await self._redis.sadd(self._index_key, run_id)
+
+    async def load_snapshot(self, run_id: str) -> dict | None:
+        key = f"{self._key_prefix}{run_id}"
+        raw = await self._redis.get(key)
+        if raw is None:
+            return None
+        if isinstance(raw, bytes):
+            raw = raw.decode("utf-8")
+        return json.loads(raw)
+
+    async def delete_snapshot(self, run_id: str) -> None:
+        key = f"{self._key_prefix}{run_id}"
+        await self._redis.delete(key)
+        await self._redis.srem(self._index_key, run_id)
+
+    async def list_snapshots(self) -> list[str]:
+        members = await self._redis.smembers(self._index_key)
+        return [m.decode("utf-8") if isinstance(m, bytes) else str(m) for m in members]
+
+
+class FilesystemStateBackend(StateBackend):
+    """Single-node: write snapshots to events_dir as {run_id}.snapshot.json. Atomic write."""
+
+    def __init__(self, events_dir: str) -> None:
+        self._events_dir = events_dir
+
+    def _path(self, run_id: str) -> str:
+        return os.path.join(self._events_dir, f"{run_id}.snapshot.json")
+
+    async def save_snapshot(self, run_id: str, snapshot: dict) -> None:
+        path = self._path(run_id)
+        tmp = path + ".tmp"
+        os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump(snapshot, f, indent=0)
+        os.replace(tmp, path)
+
+    async def load_snapshot(self, run_id: str) -> dict | None:
+        path = self._path(run_id)
+        if not os.path.isfile(path):
+            return None
+        with open(path, "r", encoding="utf-8") as f:
+            return json.load(f)
+
+    async def delete_snapshot(self, run_id: str) -> None:
+        path = self._path(run_id)
+        if os.path.isfile(path):
+            os.remove(path)
+
+    async def list_snapshots(self) -> list[str]:
+        if not os.path.isdir(self._events_dir):
+            return []
+        out: list[str] = []
+        for name in os.listdir(self._events_dir):
+            if name.endswith(".snapshot.json"):
+                out.append(name.removesuffix(".snapshot.json"))
+        return out
+
+
+def get_state_backend(config: object, redis_client: object | None = None) -> StateBackend:
+    """Return StateBackend from config. Redis if bus.backend == redis else filesystem."""
+    backend = getattr(getattr(config, "bus", None), "backend", "memory")
+    if backend == "redis" and redis_client is not None:
+        return RedisStateBackend(redis_client)
+    events_dir = getattr(config, "events_dir", ".devsper/events")
+    return FilesystemStateBackend(events_dir)

devsper/compliance/__init__.py

@@ -0,0 +1,5 @@
+"""Compliance: PII redaction and audit (v2.0)."""
+
+from devsper.compliance.pii import PIIRedactor, PIIDetection, RedactionResult
+
+__all__ = ["PIIRedactor", "PIIDetection", "RedactionResult"]

devsper/compliance/pii.py

@@ -0,0 +1,147 @@
+"""
+PIIRedactor: scan and redact PII from text before storage.
+Detectors: regex (EMAIL, PHONE, SSN, CREDIT_CARD, IP, API_KEY) + optional spaCy NER (NAME, ADDRESS).
+"""
+
+import re
+from dataclasses import dataclass, field
+
+
+@dataclass
+class PIIDetection:
+    pii_type: str
+    start: int
+    end: int
+    confidence: float
+
+
+@dataclass
+class RedactionResult:
+    redacted_text: str
+    detections: list[PIIDetection] = field(default_factory=list)
+    pii_found: bool = False
+
+
+# Regex patterns (non-capturing for substitution)
+PATTERNS = {
+    "EMAIL": re.compile(
+        r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}",
+        re.IGNORECASE,
+    ),
+    "PHONE": re.compile(
+        r"\+?[\d\s\-()]{10,20}\d",
+    ),
+    "SSN": re.compile(
+        r"\b\d{3}[-\s]?\d{2}[-\s]?\d{4}\b",
+    ),
+    "CREDIT_CARD": re.compile(
+        r"\b(?:\d[-\s]*){13,19}\d\b",
+    ),
+    "IP_ADDRESS": re.compile(
+        r"\b(?:\d{1,3}\.){3}\d{1,3}\b|\[?[0-9a-fA-F:.]+\]?",
+    ),
+    "API_KEY": re.compile(
+        r"\b(?:sk|pk)[-_][a-zA-Z0-9]{20,}\b",
+    ),
+}
+
+
+def _luhn_ok(digits: str) -> bool:
+    s = digits.replace(" ", "").replace("-", "")
+    if len(s) < 13:
+        return False
+    try:
+        nums = [int(c) for c in s]
+    except ValueError:
+        return False
+    total = 0
+    for i, d in enumerate(reversed(nums)):
+        if i % 2 == 1:
+            d *= 2
+            if d > 9:
+                d -= 9
+        total += d
+    return total % 10 == 0
+
+
+class PIIRedactor:
+    """Redact PII from text. Configurable pii_types; optional spaCy for NAME/ADDRESS when gdpr_mode."""
+
+    def __init__(
+        self,
+        pii_types: list[str] | None = None,
+        gdpr_mode: bool = False,
+    ) -> None:
+        self.pii_types = pii_types or list(PATTERNS.keys())
+        self.gdpr_mode = gdpr_mode
+        self._ner_available: bool | None = None
+
+    def _ner_detect(self, text: str) -> list[PIIDetection]:
+        out: list[PIIDetection] = []
+        if self._ner_available is False:
+            return out
+        try:
+            import spacy
+            if self._ner_available is None:
+                try:
+                    nlp = spacy.load("en_core_web_sm")
+                except Exception:
+                    nlp = spacy.blank("en")
+                setattr(self, "_ner", nlp)
+                self._ner_available = True
+        except ImportError:
+            self._ner_available = False
+            return out
+        nlp = getattr(self, "_ner", None)
+        if nlp is None:
+            return out
+        doc = nlp(text)
+        for ent in doc.ents:
+            if ent.label_ in ("PERSON", "GPE", "LOC") and (self.gdpr_mode or ent.label_ == "PERSON"):
+                out.append(PIIDetection(
+                    pii_type="NAME" if ent.label_ == "PERSON" else "ADDRESS",
+                    start=ent.start_char,
+                    end=ent.end_char,
+                    confidence=0.9,
+                ))
+        return out
+
+    def redact(self, text: str) -> RedactionResult:
+        if not text:
+            return RedactionResult(redacted_text="", detections=[], pii_found=False)
+        detections: list[PIIDetection] = []
+        ner_done = False
+        for pii_type in self.pii_types:
+            if pii_type not in PATTERNS and pii_type not in ("NAME", "ADDRESS"):
+                continue
+            if pii_type in ("NAME", "ADDRESS") or self.gdpr_mode:
+                if not ner_done:
+                    detections.extend(self._ner_detect(text))
+                    ner_done = True
+                continue
+            pattern = PATTERNS[pii_type]
+            for m in pattern.finditer(text):
+                snippet = m.group(0)
+                if pii_type == "CREDIT_CARD" and not _luhn_ok(snippet):
+                    continue
+                detections.append(PIIDetection(
+                    pii_type=pii_type,
+                    start=m.start(),
+                    end=m.end(),
+                    confidence=0.95,
+                ))
+        detections.sort(key=lambda d: (d.start, -d.end))
+        merged: list[PIIDetection] = []
+        for d in detections:
+            if merged and d.start < merged[-1].end:
+                continue
+            merged.append(d)
+        result = list(text)
+        for d in reversed(merged):
+            result[d.start:d.end] = f"[REDACTED:{d.pii_type}]"
+        redacted_text = "".join(result)
+        return RedactionResult(
+            redacted_text=redacted_text,
+            detections=merged,
+            pii_found=len(merged) > 0,
+        )

devsper/config/__init__.py

@@ -0,0 +1,52 @@
+"""
+devsper configuration: TOML + env, Pydantic-validated.
+
+Priority: env > project config > user config > defaults.
+Config locations: ./devsper.toml, ./workflow.devsper.toml, ~/.config/devsper/config.toml,
+and legacy .devsper/config.toml.
+"""
+
+from devsper.config.resolver import resolve_config
+from devsper.config.schema import (
+    A2AConfig,
+    devsperConfigModel,
+    KnowledgeConfig,
+    MCPConfig,
+    MemoryConfig,
+    ModelsConfig,
+    NodesConfig,
+    ProviderAzureConfig,
+    ProvidersConfig,
+    SwarmConfig,
+    TelemetryConfig,
+    ToolsConfig,
+)
+
+# Backward compatibility: old code expects devsperConfig and get_config()
+devsperConfig = devsperConfigModel
+
+
+def get_config(config_path: str | None = None) -> devsperConfigModel:
+    """
+    Load and resolve configuration.
+    Returns object with .worker_model, .planner_model, .events_dir, .data_dir,
+    and .swarm, .models, .memory, .tools, .telemetry, .providers.
+    """
+    return resolve_config(config_path=config_path)
+
+
+__all__ = [
+    "A2AConfig",
+    "get_config",
+    "devsperConfig",
+    "devsperConfigModel",
+    "KnowledgeConfig",
+    "MCPConfig",
+    "MemoryConfig",
+    "ModelsConfig",
+    "ProviderAzureConfig",
+    "ProvidersConfig",
+    "SwarmConfig",
+    "TelemetryConfig",
+    "ToolsConfig",
+]

devsper/config/config_loader.py

@@ -0,0 +1,121 @@
+"""Discover and load TOML config from standard locations."""
+
+from pathlib import Path
+
+import tomllib
+
+
+def _load_toml(path: Path) -> dict:
+    if not path.is_file():
+        return {}
+    try:
+        with open(path, "rb") as f:
+            return tomllib.load(f)
+    except Exception:
+        return {}
+
+
+def user_config_path() -> Path:
+    return Path.home() / ".config" / "devsper" / "config.toml"
+
+
+def project_config_paths() -> list[Path]:
+    """Return candidate project config paths in order: devsper.toml, workflow.devsper.toml, .devsper/config.toml (legacy)."""
+    cwd = Path.cwd()
+    paths = [
+        cwd / "devsper.toml",
+        cwd / "workflow.devsper.toml",
+        cwd / ".devsper" / "config.toml",
+        cwd.parent / "devsper.toml",
+        cwd.parent / "workflow.devsper.toml",
+        cwd.parent / ".devsper" / "config.toml",
+    ]
+    return paths
+
+
+def load_user_config() -> dict:
+    return _load_toml(user_config_path())
+
+
+def load_project_config() -> dict:
+    """Load first existing project config file (priority: devsper.toml > workflow.devsper.toml > .devsper/config.toml)."""
+    for p in project_config_paths():
+        data = _load_toml(p)
+        if data:
+            return data
+    return {}
+
+
+def _extract_legacy_defaults(data: dict) -> dict:
+    """Map legacy [default] or top-level keys into a flat dict for merging."""
+    out: dict = {}
+    default_block = data.get("default")
+    if isinstance(default_block, dict):
+        out = dict(default_block)
+    # Top-level string/int/float/bool keys (legacy)
+    for k, v in data.items():
+        if k in ("default", "workflow", "swarm", "models", "memory", "tools", "telemetry", "providers"):
+            continue
+        if isinstance(v, (str, int, float, bool)):
+            out[k] = v
+    return out
+
+
+def normalize_toml_to_flat(data: dict) -> dict:
+    """
+    Normalize TOML data into a structure the resolver expects.
+    Supports new format [swarm], [models], etc. and legacy [default] / top-level.
+    """
+    result: dict = {}
+    legacy = _extract_legacy_defaults(data)
+    if legacy:
+        result["events_dir"] = legacy.get("events_dir", "")
+        result["data_dir"] = legacy.get("data_dir", "")
+        result["worker_model"] = legacy.get("worker_model", "")
+        result["planner_model"] = legacy.get("planner_model", "")
+
+    if "swarm" in data and isinstance(data["swarm"], dict):
+        result["swarm"] = data["swarm"]
+    if "models" in data and isinstance(data["models"], dict):
+        result["models"] = data["models"]
+    elif legacy:
+        if legacy.get("worker_model"):
+            result.setdefault("models", {})["worker"] = legacy["worker_model"]
+        if legacy.get("planner_model"):
+            result.setdefault("models", {})["planner"] = legacy["planner_model"]
+    if "memory" in data and isinstance(data["memory"], dict):
+        result["memory"] = data["memory"]
+    if "knowledge" in data and isinstance(data["knowledge"], dict):
+        result["knowledge"] = data["knowledge"]
+    if "tools" in data and isinstance(data["tools"], dict):
+        result["tools"] = data["tools"]
+        result["tools"] = data["tools"]
+    if "telemetry" in data and isinstance(data["telemetry"], dict):
+        result["telemetry"] = data["telemetry"]
+    if "cache" in data and isinstance(data["cache"], dict):
+        result["cache"] = data["cache"]
+    if "bus" in data and isinstance(data["bus"], dict):
+        result["bus"] = data["bus"]
+    if "nodes" in data and isinstance(data["nodes"], dict):
+        result["nodes"] = data["nodes"]
+    if "providers" in data and isinstance(data["providers"], dict):
+        result["providers"] = data["providers"]
+    # v1.10.5: MCP and A2A
+    if "mcp" in data and isinstance(data["mcp"], dict):
+        result["mcp"] = data["mcp"]
+    if "a2a" in data and isinstance(data["a2a"], dict):
+        result["a2a"] = data["a2a"]
+    # v2.1: hitl and [[hitl.policies]]
+    if "hitl" in data and isinstance(data["hitl"], dict):
+        result["hitl"] = dict(data["hitl"])
+        policies = data["hitl"].get("policies")
+        if isinstance(policies, list):
+            result["hitl"]["policies"] = []
+            for p in policies:
+                if isinstance(p, dict):
+                    pol = dict(p)
+                    triggers = p.get("triggers")
+                    if isinstance(triggers, list):
+                        pol["triggers"] = [t if isinstance(t, dict) else {"type": "confidence_below", "threshold": 0.5} for t in triggers]
+                    result["hitl"]["policies"].append(pol)
+    return result