devsper 2.1.6__py3-none-any.whl

devsper/swarm/planner.py

@@ -0,0 +1,197 @@
+"""
+Planner: convert one task into multiple ordered subtasks.
+
+Lifecycle: PLANNER_STARTED → TASK_CREATED (×N) → PLANNER_FINISHED.
+Subtasks have sequential dependencies; each subtask gets a short unique ID.
+"""
+
+import re
+import secrets
+from datetime import datetime, timezone
+
+from devsper.types.task import Task
+from devsper.types.event import Event, events
+from devsper.agents.roles import infer_role_from_description
+from devsper.utils.event_logger import EventLog
+from devsper.utils.models import generate
+
+
+PLANNER_PROMPT = """Break the following task into the minimal number of smaller steps needed (use only as many as the task requires—often 1–3 for simple tasks, more for complex ones).
+
+Task:
+{task_description}
+{kg_section}
+
+Each step must be a concrete action that can be done with tools (e.g. list directory, read file, run a command). Do NOT use vague steps like "open a terminal", "navigate to X", or "access the environment"—instead say exactly what to do: "List the contents of /tmp", "Read and summarize file X", etc. Steps will be executed by agents with filesystem and other tools; they do not need to "open" anything.
+
+Return a numbered list.
+
+Example format:
+1. List the contents of /tmp
+2. Summarize the file listing
+"""
+
+EXPAND_TASKS_PROMPT = """A task just completed in a workflow.
+
+Completed task: {task_description}
+Result (preview): {result_preview}
+
+Suggest 0 to 3 additional follow-up tasks that would extend or build on this work.
+Return a numbered list only. One task per line.
+Example:
+1. Compare methods
+2. Identify trends
+"""
+
+
+NUMBERED_LINE = re.compile(r"^\s*\d+[.)]\s*(.+)$", re.MULTILINE)
+
+# Skip decomposition for short, single-step prompts (e.g. "What is 2+2?")
+MAX_SIMPLE_TASK_LEN = 200
+MULTI_STEP_PATTERN = re.compile(
+    r"\b(then|and then|first|second|step\s*\d|finally|after that|next,?)\b|\n|^\s*\d+[.)]\s+",
+    re.IGNORECASE | re.MULTILINE,
+)
+
+
+def _is_simple_task(description: str) -> bool:
+    """True if the task looks like a single question/request, not a multi-step workflow."""
+    if not description or len(description.strip()) > MAX_SIMPLE_TASK_LEN:
+        return False
+    return MULTI_STEP_PATTERN.search(description.strip()) is None
+
+
+def _short_id() -> str:
+    """Return a short, URL-safe task id (8 hex chars)."""
+    return secrets.token_hex(4)
+
+
+class Planner:
+    """Converts one Task into a list of subtasks with sequential dependencies."""
+
+    def __init__(
+        self,
+        model_name: str = "gpt-4o",
+        event_log: EventLog | None = None,
+        strategy=None,
+        prompt_suffix: str = "",
+        knowledge_graph=None,
+        guide_planning: bool = False,
+        min_confidence: float = 0.30,
+        parallel: bool = False,
+    ):
+        self.model_name = model_name
+        self.event_log = event_log or EventLog()
+        self.strategy = strategy
+        self.prompt_suffix = prompt_suffix or ""
+        self.knowledge_graph = knowledge_graph
+        self.guide_planning = guide_planning
+        self.min_confidence = min_confidence
+        self.parallel = parallel
+
+    def plan(self, task: Task) -> list[Task]:
+        """Break task into subtasks (strategy DAG or LLM). Emit planner lifecycle events."""
+        self._emit(events.PLANNER_STARTED, {"task_id": task.id})
+
+        if self.strategy is not None:
+            subtasks = self.strategy.plan(task)
+            if subtasks:
+                for st in subtasks:
+                    if getattr(st, "role", None) is None:
+                        st.role = infer_role_from_description(st.description or "")
+                    if self.parallel:
+                        st.dependencies = []
+                    self._emit(events.TASK_CREATED, {"task_id": st.id, "description": st.description})
+                self._emit(events.PLANNER_FINISHED, {"task_id": task.id, "subtask_count": len(subtasks)})
+                return subtasks
+
+        # Simple single-step prompt: skip LLM decomposition (avoids overcomplication)
+        if _is_simple_task(task.description or ""):
+            if getattr(task, "role", None) is None:
+                task.role = infer_role_from_description(task.description or "")
+            self._emit(events.TASK_CREATED, {"task_id": task.id, "description": task.description})
+            self._emit(events.PLANNER_FINISHED, {"task_id": task.id, "subtask_count": 1})
+            return [task]
+
+        task_description = (task.description or "") + self.prompt_suffix
+        kg_section = ""
+        if self.guide_planning and self.knowledge_graph is not None:
+            from devsper.knowledge.query import query_for_planning, format_planning_context
+            planning_ctx = query_for_planning(self.knowledge_graph, task_description)
+            if planning_ctx.confidence > self.min_confidence:
+                kg_section = "\n\n## Relevant prior knowledge:\n" + format_planning_context(planning_ctx) + "\n\nUse this to avoid re-discovering known facts."
+                self._emit(
+                    events.PLANNER_KG_CONTEXT_INJECTED,
+                    {
+                        "concept_count": len(planning_ctx.relevant_concepts),
+                        "finding_count": len(planning_ctx.prior_findings),
+                        "confidence": planning_ctx.confidence,
+                    },
+                )
+        prompt = PLANNER_PROMPT.format(task_description=task_description, kg_section=kg_section or "")
+        raw = generate(self.model_name, prompt)
+        steps = self._parse_numbered_list(raw)
+
+        subtasks: list[Task] = []
+        task_ids: list[str] = []
+        for i, description in enumerate(steps, start=1):
+            task_id = _short_id()
+            task_ids.append(task_id)
+            deps = [] if self.parallel else ([task_ids[i - 2]] if i > 1 else [])
+            role = infer_role_from_description(description.strip())
+            subtask = Task(id=task_id, description=description.strip(), dependencies=deps, role=role)
+            subtasks.append(subtask)
+            self._emit(
+                events.TASK_CREATED,
+                {"task_id": task_id, "description": subtask.description},
+            )
+
+        self._emit(events.PLANNER_FINISHED, {"task_id": task.id, "subtask_count": len(subtasks)})
+        return subtasks
+
+    def expand_tasks(self, completed_task: Task, context: list[Task] | None = None) -> list[Task]:
+        """
+        After a task completes, optionally generate additional tasks (dynamic DAG growth).
+        New tasks depend on the completed task. Emits TASK_CREATED for each.
+        """
+        result_preview = (completed_task.result or "")[:500]
+        prompt = EXPAND_TASKS_PROMPT.format(
+            task_description=completed_task.description,
+            result_preview=result_preview,
+        )
+        raw = generate(self.model_name, prompt)
+        steps = self._parse_numbered_list(raw)
+        if not steps:
+            return []
+
+        new_tasks: list[Task] = []
+        for i, description in enumerate(steps, start=1):
+            task_id = _short_id()
+            role = infer_role_from_description(description.strip())
+            subtask = Task(
+                id=task_id,
+                description=description.strip(),
+                dependencies=[completed_task.id],
+                role=role,
+            )
+            new_tasks.append(subtask)
+            self._emit(events.TASK_CREATED, {"task_id": task_id, "description": subtask.description})
+
+        return new_tasks
+
+    def _parse_numbered_list(self, text: str) -> list[str]:
+        """Parse '1. step\\n2. step' into ['step', 'step', ...]."""
+        steps = []
+        for line in text.strip().splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            m = NUMBERED_LINE.match(line)
+            if m:
+                steps.append(m.group(1).strip())
+        return steps
+
+    def _emit(self, event_type: events, payload: dict) -> None:
+        self.event_log.append_event(
+            Event(timestamp=datetime.now(timezone.utc), type=event_type, payload=payload)
+        )

devsper/swarm/prefetcher.py

@@ -0,0 +1,91 @@
+"""
+Speculative pre-fetching: pre-warm memory context and tool selection for likely successor tasks.
+v1.7.
+"""
+
+import asyncio
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from devsper.tools.base import Tool
+
+
+@dataclass
+class PrefetchResult:
+    memory_context: str
+    tools: list  # list[Tool]
+    computed_at: datetime
+
+
+class TaskPrefetcher:
+    """
+    While a task is running, pre-warm its likely successors:
+    fetch memory context and select tools in background.
+    Results are cached and consumed by the agent when the task actually starts.
+    """
+
+    def __init__(
+        self,
+        memory_router,
+        tool_selector,
+        score_store=None,
+        max_age_seconds: float = 30.0,
+    ):
+        self._memory_router = memory_router
+        self._tool_selector = tool_selector
+        self._score_store = score_store
+        self._max_age_seconds = max_age_seconds
+        self._warmup_cache: dict[str, PrefetchResult] = {}
+        self._lock = asyncio.Lock()
+
+    async def prefetch(self, task) -> None:
+        """Run in background via asyncio.create_task()."""
+        task_id = getattr(task, "id", "")
+        description = getattr(task, "description", "") or ""
+        loop = asyncio.get_running_loop()
+
+        def _get_memory() -> str:
+            if not self._memory_router or not description:
+                return ""
+            try:
+                return self._memory_router.get_memory_context(description)
+            except Exception:
+                return ""
+
+        def _get_tools():
+            if not self._tool_selector:
+                return []
+            try:
+                return self._tool_selector(
+                    description,
+                    role=getattr(task, "role", None),
+                    score_store=self._score_store,
+                )
+            except Exception:
+                return []
+
+        memory_ctx = await loop.run_in_executor(None, _get_memory)
+        tools = await loop.run_in_executor(None, _get_tools)
+        async with self._lock:
+            self._warmup_cache[task_id] = PrefetchResult(
+                memory_context=memory_ctx,
+                tools=tools,
+                computed_at=datetime.now(timezone.utc),
+            )
+
+    def consume(self, task_id: str) -> PrefetchResult | None:
+        """
+        Called by Agent at start — returns pre-warmed context or None.
+        Result older than max_age_seconds is treated as stale and not used.
+        """
+        result = self._warmup_cache.pop(task_id, None)
+        if result is None:
+            return None
+        age_seconds = (
+            datetime.now(timezone.utc) - result.computed_at
+        ).total_seconds()
+        if age_seconds > self._max_age_seconds:
+            return None
+        return result

devsper/swarm/scheduler.py

@@ -0,0 +1,153 @@
+"""
+Scheduler: manage the task DAG and determine which tasks are runnable.
+
+Supports: add_tasks, get_ready_tasks, get_speculative_tasks, mark_completed,
+confirm_speculative_for, discard_speculative_for, is_finished.
+v1.9: Single source of truth — get_task, get_all_tasks, get_results, snapshot, restore.
+"""
+
+from datetime import datetime, timezone
+import networkx as nx
+
+from devsper.types.task import Task, TaskStatus
+from devsper.types.exceptions import TaskNotFoundError
+from devsper.swarm.speculation import (
+    get_speculative_candidates,
+    confirm_speculative,
+    discard_speculative,
+)
+
+
+class Scheduler:
+    """Manages a DAG of tasks. Tracks dependencies and exposes runnable tasks. Single source of truth for task state."""
+
+    def __init__(self, run_id: str = "") -> None:
+        self._graph: nx.DiGraph = nx.DiGraph()
+        self._tasks: dict[str, Task] = {}
+        self.run_id = run_id
+
+    def get_task(self, task_id: str) -> Task:
+        """Return task by id. Raises TaskNotFoundError if not found."""
+        if task_id not in self._tasks:
+            raise TaskNotFoundError(f"Task not found: {task_id!r}")
+        return self._tasks[task_id]
+
+    def get_all_tasks(self) -> list[Task]:
+        """Return all tasks."""
+        return list(self._tasks.values())
+
+    def add_tasks(self, tasks: list[Task]) -> None:
+        """Add tasks and build the internal dependency graph."""
+        for task in tasks:
+            self._tasks[task.id] = task
+            self._graph.add_node(task.id)
+        for task in tasks:
+            for dep in task.dependencies:
+                if dep not in self._tasks:
+                    raise ValueError(f"Unknown dependency: {dep!r}")
+                self._graph.add_edge(dep, task.id)
+        if not nx.is_directed_acyclic_graph(self._graph):
+            raise ValueError("Task graph contains a cycle")
+
+    def get_ready_tasks(self) -> list[Task]:
+        """Return tasks that are runnable: PENDING and all dependencies completed."""
+        ready: list[Task] = []
+        for task_id, task in self._tasks.items():
+            if task.status != TaskStatus.PENDING:
+                continue
+            deps = list(self._graph.predecessors(task_id))
+            if all(self._tasks[dep].status == TaskStatus.COMPLETED for dep in deps):
+                ready.append(task)
+        return ready
+
+    def mark_completed(self, task_id: str, result: str = "") -> None:
+        """Mark a task completed and set its result. Dependent tasks can become runnable."""
+        if task_id in self._tasks:
+            t = self._tasks[task_id]
+            t.status = TaskStatus.COMPLETED
+            t.result = result
+
+    def mark_failed(self, task_id: str, error: str = "") -> None:
+        """Mark a task failed and set its error."""
+        if task_id in self._tasks:
+            t = self._tasks[task_id]
+            t.status = TaskStatus.FAILED
+            t.error = error
+
+    def is_finished(self) -> bool:
+        """Return True when every task is completed or failed (no pending/running left)."""
+        return all(
+            t.status in (TaskStatus.COMPLETED, TaskStatus.FAILED)
+            for t in self._tasks.values()
+        )
+
+    def get_results(self) -> dict[str, str]:
+        """Return task_id -> result for completed tasks only."""
+        return {
+            task_id: (t.result or "")
+            for task_id, t in self._tasks.items()
+            if t.status == TaskStatus.COMPLETED and t.result is not None
+        }
+
+    def snapshot(self) -> dict:
+        """Full serializable state for checkpointing and node sync."""
+        edges = list(self._graph.edges())
+        tasks_data = [t.to_dict() for t in self._tasks.values()]
+        completed_count = sum(1 for t in self._tasks.values() if t.status == TaskStatus.COMPLETED)
+        failed_count = sum(1 for t in self._tasks.values() if t.status == TaskStatus.FAILED)
+        return {
+            "run_id": self.run_id,
+            "tasks": tasks_data,
+            "edges": [[str(u), str(v)] for u, v in edges],
+            "completed_count": completed_count,
+            "failed_count": failed_count,
+            "snapshot_at": datetime.now(timezone.utc).isoformat(),
+        }
+
+    @classmethod
+    def restore(cls, snapshot: dict) -> "Scheduler":
+        """Reconstruct Scheduler from snapshot dict."""
+        run_id = snapshot.get("run_id", "")
+        s = cls(run_id=run_id)
+        tasks_data = snapshot.get("tasks", [])
+        for tdata in tasks_data:
+            task = Task.from_dict(tdata)
+            s._tasks[task.id] = task
+            s._graph.add_node(task.id)
+        for edge in snapshot.get("edges", []):
+            if len(edge) >= 2:
+                u, v = str(edge[0]), str(edge[1])
+                if u in s._tasks and v in s._tasks:
+                    s._graph.add_edge(u, v)
+        return s
+
+    def get_completed_tasks(self) -> list[Task]:
+        """Return all completed tasks (for learning/memory storage)."""
+        return [
+            t
+            for t in self._tasks.values()
+            if t.status == TaskStatus.COMPLETED and t.result is not None
+        ]
+
+    def get_speculative_tasks(self) -> list[Task]:
+        """Return tasks that can be run speculatively (one dep running, rest completed)."""
+        candidates = get_speculative_candidates(self._tasks, self._graph)
+        for t in candidates:
+            t.speculative = True
+        return candidates
+
+    def get_successors(self, task_id: str) -> list[str]:
+        """Return task ids that depend on the given task."""
+        return list(self._graph.successors(task_id))
+
+    def confirm_speculative_for(self, completed_task_id: str) -> None:
+        """When completed_task_id finishes, confirm its speculative successors (keep results)."""
+        for sid in self.get_successors(completed_task_id):
+            if sid in self._tasks:
+                confirm_speculative(self._tasks[sid])
+
+    def discard_speculative_for(self, failed_task_id: str) -> None:
+        """When failed_task_id fails, discard results of speculative successors."""
+        for sid in self.get_successors(failed_task_id):
+            if sid in self._tasks:
+                discard_speculative(self._tasks[sid])

devsper/swarm/speculation.py

@@ -0,0 +1,47 @@
+"""
+Speculative execution: predict likely next tasks and execute them early.
+
+When planner produces DAG A → B → C:
+- Run A
+- Speculatively schedule B and C (they depend only on A)
+- When A completes: confirm B and C (use speculative results) or discard
+"""
+
+from devsper.types.task import Task, TaskStatus
+
+
+def get_speculative_candidates(
+    tasks: dict[str, Task],
+    graph,
+) -> list[Task]:
+    """
+    Return tasks that are candidates for speculative execution: all dependencies
+    are either COMPLETED or exactly one is RUNNING (and we bet it will complete).
+    graph must have .predecessors(node_id) returning an iterable of dependency ids.
+    """
+    candidates: list[Task] = []
+    for task_id, task in tasks.items():
+        if task.status != TaskStatus.PENDING:
+            continue
+        deps = list(graph.predecessors(task_id))
+        if not deps:
+            continue
+        completed = sum(1 for d in deps if tasks[d].status == TaskStatus.COMPLETED)
+        running = sum(1 for d in deps if tasks[d].status == TaskStatus.RUNNING)
+        # Speculative: all deps done except one which is running
+        if running == 1 and completed == len(deps) - 1:
+            candidates.append(task)
+    return candidates
+
+
+def confirm_speculative(task: Task) -> None:
+    """Mark a speculatively run task as confirmed (keep result)."""
+    if task.speculative and task.result is not None:
+        task.status = TaskStatus.COMPLETED
+
+
+def discard_speculative(task: Task) -> None:
+    """Discard speculative result; task will be re-run when deps are confirmed."""
+    if task.speculative:
+        task.status = TaskStatus.PENDING
+        task.result = None