devsper 2.1.6__py3-none-any.whl

devsper/intelligence/strategies/research_strategy.py

@@ -0,0 +1,34 @@
+"""Research strategy: DAG for literature review (corpus -> topic -> citation -> review)."""
+
+import secrets
+
+from devsper.types.task import Task
+
+from devsper.intelligence.strategies.base import Strategy
+
+
+def _short_id() -> str:
+    return secrets.token_hex(4)
+
+
+class ResearchStrategy(Strategy):
+    """Research pipeline: corpus_builder -> topic_extraction -> citation_graph -> literature_review."""
+
+    def plan(self, root_task: Task) -> list[Task]:
+        steps = [
+            ("corpus_builder", "Build a corpus of relevant papers and sources for the topic."),
+            ("topic_extraction", "Extract main topics and themes from the corpus."),
+            ("citation_graph", "Build citation graph and identify key references."),
+            ("literature_review", "Write a structured literature review synthesizing findings."),
+        ]
+        task_ids: list[str] = []
+        tasks: list[Task] = []
+        for i, (step_id, desc) in enumerate(steps):
+            tid = _short_id()
+            task_ids.append(tid)
+            deps = [task_ids[i - 1]] if i > 0 else []
+            # Include root context in first step
+            description = f"{root_task.description}\n\nStep: {desc}" if i == 0 else desc
+            t = Task(id=tid, description=description, dependencies=deps)
+            tasks.append(t)
+        return tasks

devsper/intelligence/strategy_selector.py

@@ -0,0 +1,84 @@
+"""
+Strategy selector: choose execution strategy (research, code analysis, data analysis).
+"""
+
+import re
+from enum import Enum
+
+from devsper.types.task import Task
+
+
+class ExecutionStrategy(str, Enum):
+    RESEARCH = "research"
+    CODE_ANALYSIS = "code_analysis"
+    DATA_ANALYSIS = "data_analysis"
+    DOCUMENT = "document"
+    EXPERIMENT = "experiment"
+    GENERAL = "general"
+
+
+RESEARCH_KEYWORDS = [
+    "research", "paper", "literature", "survey", "cite", "citation",
+    "diffusion", "transformer", "methodology", "findings", "review",
+]
+CODE_KEYWORDS = [
+    "code", "codebase", "repository", "refactor", "lint", "test",
+    "api", "architecture", "module", "function", "class", "implement",
+]
+DATA_KEYWORDS = [
+    "data", "dataset", "csv", "analysis", "experiment", "metric",
+    "statistic", "plot", "visualization", "pipeline", "training",
+]
+DOCUMENT_KEYWORDS = [
+    "document", "pdf", "docx", "ingest", "extract", "corpus",
+    "paper", "report", "write", "summary",
+]
+EXPERIMENT_KEYWORDS = [
+    "experiment", "sweep", "parameter", "benchmark", "ablation",
+    "compare", "metric", "evaluate", "run",
+]
+
+
+class StrategySelector:
+    """
+    Select execution strategy from a root task description to tune planner/executor behavior.
+    """
+
+    def __init__(self) -> None:
+        self._keyword_lists = {
+            ExecutionStrategy.RESEARCH: RESEARCH_KEYWORDS,
+            ExecutionStrategy.CODE_ANALYSIS: CODE_KEYWORDS,
+            ExecutionStrategy.DATA_ANALYSIS: DATA_KEYWORDS,
+            ExecutionStrategy.DOCUMENT: DOCUMENT_KEYWORDS,
+            ExecutionStrategy.EXPERIMENT: EXPERIMENT_KEYWORDS,
+        }
+
+    def select(self, task: Task | str) -> ExecutionStrategy:
+        """
+        Return the best strategy for the given task (or task description string).
+        """
+        text = task.description if isinstance(task, Task) else str(task)
+        text = (text or "").lower()
+        scores = {s: 0 for s in ExecutionStrategy}
+        scores[ExecutionStrategy.GENERAL] = 0
+        for strategy, keywords in self._keyword_lists.items():
+            for kw in keywords:
+                if kw in text:
+                    scores[strategy] += 1
+        candidates = [s for s in ExecutionStrategy if s != ExecutionStrategy.GENERAL]
+        best = max(candidates, key=lambda s: scores[s])
+        return best if scores[best] > 0 else ExecutionStrategy.GENERAL
+
+    def suggest_planner_prompt_suffix(self, strategy: ExecutionStrategy) -> str:
+        """Return optional prompt suffix to bias the planner for this strategy."""
+        if strategy == ExecutionStrategy.RESEARCH:
+            return " Focus on: literature search, paper summaries, citation context, methodology comparison."
+        if strategy == ExecutionStrategy.CODE_ANALYSIS:
+            return " Focus on: structure, dependencies, tests, refactors, documentation."
+        if strategy == ExecutionStrategy.DATA_ANALYSIS:
+            return " Focus on: data loading, stats, visualizations, experiments, metrics."
+        if strategy == ExecutionStrategy.DOCUMENT:
+            return " Focus on: document ingest, extraction, linking, and reporting."
+        if strategy == ExecutionStrategy.EXPERIMENT:
+            return " Focus on: setup, run, comparison, and experiment reporting."
+        return ""

devsper/intelligence/synthesis.py

@@ -0,0 +1,132 @@
+"""
+v1.8: Cross-run synthesis — answer questions by querying across ALL memory (all runs).
+"""
+
+from collections.abc import Iterator
+from datetime import datetime
+
+from devsper.knowledge.query import query_for_planning, format_planning_context, PlanningContext
+from devsper.memory.memory_index import MemoryIndex
+from devsper.memory.memory_types import MemoryRecord
+from devsper.knowledge.knowledge_graph import KnowledgeGraph
+from devsper.utils.models import generate
+
+
+def _cosine_sim(a: list[float], b: list[float]) -> float:
+    if not a or not b or len(a) != len(b):
+        return 0.0
+    dot = sum(x * y for x, y in zip(a, b))
+    na = sum(x * x for x in a) ** 0.5
+    nb = sum(x * x for x in b) ** 0.5
+    if na == 0 or nb == 0:
+        return 0.0
+    return dot / (na * nb)
+
+
+def _deduplicate_by_similarity(records: list[MemoryRecord], threshold: float = 0.95) -> list[MemoryRecord]:
+    """Remove records with cosine similarity > threshold to an already-kept record."""
+    if not records:
+        return []
+    out: list[MemoryRecord] = []
+    for r in records:
+        if r.embedding is None:
+            out.append(r)
+            continue
+        keep = True
+        for o in out:
+            if o.embedding is None:
+                continue
+            if _cosine_sim(r.embedding, o.embedding) > threshold:
+                keep = False
+                break
+        if keep:
+            out.append(r)
+    return out
+
+
+def _short_run_id(run_id: str) -> str:
+    """Short run id for citations (e.g. events_2025-03-09... -> 2025-03-09)."""
+    if not run_id:
+        return "unknown"
+    if "_" in run_id:
+        parts = run_id.split("_")
+        if len(parts) >= 2:
+            return parts[1][:12] if len(parts[1]) > 12 else parts[1]
+    return run_id[:12] if len(run_id) > 12 else run_id
+
+
+class CrossRunSynthesizer:
+    """
+    Answers questions by querying across ALL memory (all runs),
+    not just the current session.
+    """
+
+    def __init__(
+        self,
+        memory_index: MemoryIndex,
+        knowledge_graph: KnowledgeGraph | None,
+        worker_model: str,
+    ):
+        self.memory_index = memory_index
+        self.knowledge_graph = knowledge_graph
+        self.worker_model = worker_model
+
+    def synthesize(
+        self,
+        query: str,
+        max_sources: int = 20,
+        stream: bool = True,
+        use_kg: bool = True,
+        since: datetime | None = None,
+    ) -> Iterator[str] | str:
+        """
+        1. Query memory_index across all runs (no run_id filter): top-20 by similarity
+        2. Optionally query knowledge graph: query_for_planning(kg, query)
+        3. Deduplicate: remove memory records with cosine similarity > 0.95 to each other
+        4. Build synthesis prompt with all sources
+        5. Stream LLM response (or return full string if stream=False)
+        """
+        memories = self.memory_index.query_across_runs(query, top_k=max_sources)
+        if since is not None:
+            memories = [m for m in memories if m.timestamp >= since]
+        memories = _deduplicate_by_similarity(memories, threshold=0.95)
+        memories = memories[:max_sources]
+
+        kg_ctx: PlanningContext | None = None
+        if use_kg and self.knowledge_graph is not None:
+            kg_ctx = query_for_planning(self.knowledge_graph, query)
+
+        prompt = self._build_synthesis_prompt(query, memories, kg_ctx)
+        if stream:
+            gen = generate(self.worker_model, prompt, stream=True)
+            for chunk in gen:
+                yield chunk
+        else:
+            return generate(self.worker_model, prompt, stream=False) or ""
+
+    def _build_synthesis_prompt(
+        self,
+        query: str,
+        memories: list[MemoryRecord],
+        kg_ctx: PlanningContext | None,
+    ) -> str:
+        """Build system + user prompt for synthesis."""
+        kg_block = ""
+        if kg_ctx and (kg_ctx.relevant_concepts or kg_ctx.prior_findings or kg_ctx.related_methods):
+            kg_block = "Knowledge Graph Facts:\n" + format_planning_context(kg_ctx) + "\n\n"
+        unique_runs = list(dict.fromkeys(getattr(m, "run_id", "") or "" for m in memories))
+        unique_runs = [r for r in unique_runs if r]
+        memory_block = "\n".join(
+            f"[run:{_short_run_id(getattr(m, 'run_id', ''))}] {(m.content or '')[:300]}"
+            for m in memories
+        )
+        user = f"""Query: {query}
+
+{kg_block}Memory Sources ({len(memories)} records across {len(unique_runs)} runs):
+{memory_block}"""
+        system = """You are synthesizing research findings across multiple past sessions.
+Answer the query using ONLY the provided sources.
+Cite sources as [run:RUN_ID_SHORT] inline.
+If sources conflict, note the conflict explicitly.
+Do not speculate beyond the sources."""
+        return f"System:\n{system}\n\nUser:\n{user}"

devsper/intelligence/task_optimizer.py

@@ -0,0 +1,92 @@
+"""
+Task optimizer: merge redundant tasks, detect parallel opportunities, remove unnecessary tasks.
+"""
+
+import re
+from devsper.types.task import Task
+
+
+def _normalize_for_sim(s: str) -> str:
+    return re.sub(r"\s+", " ", (s or "").lower().strip())
+
+
+class TaskOptimizer:
+    """
+    Optimize a task graph: merge redundant tasks, identify tasks that can run in parallel,
+    drop no-op or duplicate steps.
+    """
+
+    def __init__(self, min_similarity_chars: int = 20) -> None:
+        self.min_similarity_chars = min_similarity_chars
+
+    def optimize(self, tasks: list[Task]) -> list[Task]:
+        """
+        Return a new list of tasks with redundancies merged, dependencies updated,
+        and optional parallelization hints. Does not mutate input tasks.
+        """
+        if not tasks:
+            return []
+        seen_norm: dict[str, Task] = {}
+        for t in tasks:
+            norm = _normalize_for_sim(t.description)[:200]
+            if norm in seen_norm and len(norm) >= self.min_similarity_chars:
+                continue
+            seen_norm[norm] = t
+        deduped = list(seen_norm.values())
+        merged = self._merge_trivial(deduped)
+        return self._rebuild_deps(merged)
+
+    def _merge_trivial(self, tasks: list[Task]) -> list[Task]:
+        out: list[Task] = []
+        for t in tasks:
+            desc = (t.description or "").strip()
+            if len(desc) < 15 and out:
+                prev = out[-1]
+                out[-1] = Task(
+                    id=prev.id,
+                    description=prev.description + "; " + desc,
+                    dependencies=prev.dependencies,
+                )
+            else:
+                out.append(t)
+        return out
+
+    def _rebuild_deps(self, tasks: list[Task]) -> list[Task]:
+        """Rebuild task list with sequential ids and deps (task_1 -> task_2 -> ...)."""
+        id_map = {t.id: i for i, t in enumerate(tasks, start=1)}
+        result: list[Task] = []
+        for i, t in enumerate(tasks, start=1):
+            new_id = f"task_{i}"
+            deps = [f"task_{id_map[d]}" for d in t.dependencies if d in id_map]
+            if not deps and i > 1:
+                deps = [f"task_{i - 1}"]
+            result.append(
+                Task(id=new_id, description=t.description, dependencies=deps)
+            )
+        return result
+
+    def detect_parallel_opportunities(self, tasks: list[Task]) -> list[list[str]]:
+        """
+        Return groups of task ids that could run in parallel (same dependencies).
+        Each group is a list of task ids that all depend on the same set and could be run together.
+        """
+        by_dep_key: dict[tuple, list[str]] = {}
+        for t in tasks:
+            key = tuple(sorted(t.dependencies))
+            by_dep_key.setdefault(key, []).append(t.id)
+        return [ids for ids in by_dep_key.values() if len(ids) > 1]
+
+    def remove_unnecessary(self, tasks: list[Task], predicate=None) -> list[Task]:
+        """
+        Remove tasks for which predicate(task) is True (e.g. no-op descriptions).
+        predicate defaults to: description too short or placeholder-like.
+        """
+        if predicate is None:
+            def predicate(t: Task) -> bool:
+                d = (t.description or "").strip().lower()
+                if len(d) < 5:
+                    return True
+                if d in ("n/a", "none", "todo", "tbd", "-", "..."):
+                    return True
+                return False
+        return [t for t in tasks if not predicate(t)]

devsper/knowledge/__init__.py

@@ -0,0 +1,5 @@
+"""Swarm knowledge graph: relationships between memory (documents, concepts, datasets, methods)."""
+
+from devsper.knowledge.knowledge_graph import KnowledgeGraph
+
+__all__ = ["KnowledgeGraph"]

devsper/knowledge/extractor.py

@@ -0,0 +1,204 @@
+"""
+v1.8: Post-run knowledge extraction from task results into the knowledge graph.
+Heuristic extraction (no LLM); fast, non-blocking.
+"""
+
+import re
+import time
+from dataclasses import dataclass
+from datetime import datetime, timezone
+
+from devsper.knowledge.knowledge_graph import (
+    KnowledgeGraph,
+    NODE_CONCEPT,
+    NODE_DATASET,
+    NODE_DOCUMENT,
+    NODE_METHOD,
+    EDGE_USES,
+    EDGE_EXTENDS,
+    EDGE_OUTPERFORMS,
+    EDGE_CITES,
+)
+from devsper.types.task import Task, TaskStatus
+from devsper.types.event import Event, events
+
+
+@dataclass
+class KGNode:
+    """Lightweight node for extractor output."""
+    id: str
+    kind: str
+    label: str
+    confidence: float
+
+
+@dataclass
+class KGEdge:
+    """Lightweight edge for extractor output."""
+    from_id: str
+    to_id: str
+    edge_type: str
+
+
+# Patterns for documents: explicit citations
+_DOC_PATTERN = re.compile(
+    r"(?:according to|paper:|article:|from)\s*[:\s]*([^\n.]+?)(?:\.|$)|(https?://[^\s]+)",
+    re.IGNORECASE,
+)
+# Capitalized multi-word (2-3 words)
+_CONCEPT_PATTERN = re.compile(r"\b([A-Z][a-z]+(?:\s+[A-Z][a-z0-9]+){1,2})\b")
+# Technical: digits+letters, camelCase
+_TECH_PATTERN = re.compile(r"\b([a-z]+[A-Z][a-zA-Z0-9]*|[A-Z][a-z]+[A-Z][a-zA-Z0-9]*)\b")
+# Dataset/corpus/benchmark near a proper noun
+_DATASET_PATTERN = re.compile(
+    r"\b(dataset|corpus|benchmark)\s+[\[']?([A-Za-z0-9\-_]+)[\]']?|\b([A-Z][A-Za-z0-9\-]+(?:-\d+)?)\s+(?:dataset|corpus|benchmark)\b",
+    re.IGNORECASE,
+)
+# "using X", "via X", "with X" method
+_METHOD_PATTERN = re.compile(
+    r"\b(?:using|via|with)\s+([A-Z][A-Za-z0-9\s]+?)(?:\s+to|\s+for|,|\.|$)",
+    re.IGNORECASE,
+)
+
+# Relationship patterns (entity co-occurrence in same sentence)
+_REL_USES = re.compile(r"(\w+(?:\s+\w+)?)\s+(?:uses?|leverages?)\s+(\w+(?:\s+\w+)?)", re.IGNORECASE)
+_REL_EXTENDS = re.compile(r"(\w+(?:\s+\w+)?)\s+(?:is\s+based\s+on|extends?)\s+(\w+(?:\s+\w+)?)", re.IGNORECASE)
+_REL_OUTPERFORMS = re.compile(r"(\w+(?:\s+\w+)?)\s+(?:outperforms?|is\s+better\s+than)\s+(\w+(?:\s+\w+)?)", re.IGNORECASE)
+_REL_CITES = re.compile(r"(\w+(?:\s+\w+)?)\s+(?:cites?|references?)\s+(\w+(?:\s+\w+)?)", re.IGNORECASE)
+
+
+def _normalize_id(kind: str, label: str) -> str:
+    label_clean = re.sub(r"\s+", " ", label.strip())
+    return f"{kind}:{label_clean}"
+
+
+class KnowledgeExtractor:
+    """
+    Post-run background pass: extract entities, facts, and relationships
+    from task results and add to knowledge graph.
+    """
+
+    def __init__(self, min_confidence: float = 0.60):
+        self.min_confidence = min_confidence
+
+    async def extract_from_run(
+        self,
+        run_id: str,
+        tasks: list[Task],
+        kg: KnowledgeGraph,
+        event_log=None,
+    ) -> None:
+        """Called after SWARM_FINISHED. Run in background, non-blocking."""
+        start = time.perf_counter()
+        nodes_added = 0
+        edges_added = 0
+        for task in tasks:
+            if task.status != TaskStatus.COMPLETED or not task.result:
+                continue
+            entities = self._extract_entities(task.result)
+            for e in entities:
+                if e.confidence >= self.min_confidence:
+                    kg.add_or_update_node(e.id, e.kind, e.label, confidence=e.confidence)
+                    nodes_added += 1
+            relationships = self._extract_relationships(task.result, entities)
+            for rel in relationships:
+                kg.add_edge(rel.from_id, rel.to_id, rel.edge_type)
+                edges_added += 1
+        kg.save()
+        duration = time.perf_counter() - start
+        if event_log is not None:
+            event_log.append_event(
+                Event(
+                    timestamp=datetime.now(timezone.utc),
+                    type=events.KNOWLEDGE_EXTRACTED,
+                    payload={
+                        "run_id": run_id,
+                        "nodes_added": nodes_added,
+                        "edges_added": edges_added,
+                        "duration_seconds": round(duration, 2),
+                    },
+                )
+            )
+        return None
+
+    def _extract_entities(self, text: str) -> list[KGNode]:
+        """Heuristic extraction. Confidence: URL/explicit 0.95, repeated 3+ 0.80, single 0.60."""
+        if not text or not text.strip():
+            return []
+        nodes: list[KGNode] = []
+        seen: dict[str, float] = {}
+
+        # Documents: URLs, "According to", "paper:", "article:"
+        for m in _DOC_PATTERN.finditer(text):
+            g1, g2 = m.group(1), m.group(2)
+            raw = (g1 or g2 or "").strip()
+            if not raw:
+                continue
+            if raw.startswith("http"):
+                node_id = _normalize_id(NODE_DOCUMENT, raw[:80])
+                nodes.append(KGNode(id=node_id, kind=NODE_DOCUMENT, label=raw[:200], confidence=0.95))
+            else:
+                node_id = _normalize_id(NODE_DOCUMENT, raw[:100])
+                nodes.append(KGNode(id=node_id, kind=NODE_DOCUMENT, label=raw[:200], confidence=0.95))
+
+        # Concepts: capitalized phrases, technical terms
+        sentences = re.split(r"[.!?]\s+", text)
+        concept_mentions: dict[str, int] = {}
+        for sent in sentences:
+            for m in _CONCEPT_PATTERN.finditer(sent):
+                c = m.group(1).strip()
+                if len(c) < 3 or c.lower() in ("the", "this", "that"):
+                    continue
+                concept_mentions[c] = concept_mentions.get(c, 0) + 1
+            for m in _TECH_PATTERN.finditer(sent):
+                c = m.group(1).strip()
+                if len(c) < 2:
+                    continue
+                concept_mentions[c] = concept_mentions.get(c, 0) + 1
+        for label, count in concept_mentions.items():
+            conf = 0.80 if count >= 3 else 0.60
+            node_id = _normalize_id(NODE_CONCEPT, label)
+            if node_id not in seen or seen[node_id] < conf:
+                seen[node_id] = conf
+                nodes.append(KGNode(id=node_id, kind=NODE_CONCEPT, label=label, confidence=conf))
+
+        # Datasets
+        for m in _DATASET_PATTERN.finditer(text):
+            name = (m.group(2) or m.group(3) or "").strip()
+            if name and len(name) >= 2:
+                node_id = _normalize_id(NODE_DATASET, name)
+                nodes.append(KGNode(id=node_id, kind=NODE_DATASET, label=name, confidence=0.75))
+
+        # Methods: "using X", "via X"
+        for m in _METHOD_PATTERN.finditer(text):
+            name = m.group(1).strip()
+            if len(name) >= 2:
+                node_id = _normalize_id(NODE_METHOD, name)
+                nodes.append(KGNode(id=node_id, kind=NODE_METHOD, label=name, confidence=0.70))
+
+        return nodes
+
+    def _extract_relationships(
+        self,
+        text: str,
+        entities: list[KGNode],
+    ) -> list[KGEdge]:
+        """Typed edges between detected entities; pattern match on co-occurrence."""
+        edges: list[KGEdge] = []
+        entity_labels = {e.label.lower(): e for e in entities}
+        entity_labels.update({e.id.split(":", 1)[-1].lower(): e for e in entities})
+
+        def try_edge(pat: re.Pattern, edge_type: str) -> None:
+            for m in pat.finditer(text):
+                a, b = m.group(1).strip(), m.group(2).strip()
+                a_lower, b_lower = a.lower(), b.lower()
+                node_a = entity_labels.get(a_lower) or entity_labels.get(a)
+                node_b = entity_labels.get(b_lower) or entity_labels.get(b)
+                if node_a and node_b and node_a.id != node_b.id:
+                    edges.append(KGEdge(from_id=node_a.id, to_id=node_b.id, edge_type=edge_type))
+
+        try_edge(_REL_USES, EDGE_USES)
+        try_edge(_REL_EXTENDS, EDGE_EXTENDS)
+        try_edge(_REL_OUTPERFORMS, EDGE_OUTPERFORMS)
+        try_edge(_REL_CITES, EDGE_CITES)
+        return edges