devsper 2.1.6__py3-none-any.whl

devsper/tools/filesystem/read_file.py

@@ -0,0 +1,35 @@
+"""Read contents of a file from the filesystem."""
+
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class ReadFileTool(Tool):
+    """Read and return the full text content of a file."""
+
+    name = "read_file"
+    description = "Read the contents of a file. Returns the raw text."
+    input_schema = {
+        "type": "object",
+        "properties": {"path": {"type": "string", "description": "Path to the file"}},
+        "required": ["path"],
+    }
+
+    def run(self, **kwargs) -> str:
+        path = kwargs.get("path")
+        if not path or not isinstance(path, str):
+            return "Error: path must be a non-empty string"
+        p = Path(path)
+        if not p.exists():
+            return f"Error: file not found: {path}"
+        if not p.is_file():
+            return f"Error: not a file: {path}"
+        try:
+            return p.read_text(encoding="utf-8", errors="replace")
+        except Exception as e:
+            return f"Error reading file: {e}"
+
+
+register(ReadFileTool())

devsper/tools/filesystem/search_files.py

@@ -0,0 +1,60 @@
+"""Search for files by name pattern or content."""
+
+import re
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class SearchFilesTool(Tool):
+    """Search for files: by filename pattern (glob) or by text content in files."""
+
+    name = "search_files"
+    description = "Search for files by path/name pattern or by content. Returns matching file paths."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "path": {"type": "string", "description": "Root directory to search"},
+            "pattern": {"type": "string", "description": "Glob pattern for filename (e.g. *.py) or regex for content"},
+            "content_search": {"type": "boolean", "description": "If true, pattern is used as regex to search file contents"},
+        },
+        "required": ["path", "pattern"],
+    }
+
+    def run(self, **kwargs) -> str:
+        path = kwargs.get("path")
+        pattern = kwargs.get("pattern")
+        content_search = kwargs.get("content_search", False)
+        if not path or not isinstance(path, str):
+            return "Error: path must be a non-empty string"
+        if not pattern or not isinstance(pattern, str):
+            return "Error: pattern must be a non-empty string"
+        p = Path(path)
+        if not p.exists() or not p.is_dir():
+            return f"Error: path must be an existing directory: {path}"
+        try:
+            if content_search:
+                try:
+                    rx = re.compile(pattern)
+                except re.error:
+                    return f"Error: invalid regex pattern: {pattern}"
+                results = []
+                for f in p.rglob("*"):
+                    if f.is_file():
+                        try:
+                            text = f.read_text(encoding="utf-8", errors="replace")
+                            if rx.search(text):
+                                results.append(str(f))
+                        except (OSError, UnicodeDecodeError):
+                            pass
+                return "\n".join(sorted(results)) if results else "No matches found"
+            else:
+                matches = sorted(p.rglob(pattern))
+                files = [str(m) for m in matches if m.is_file()]
+                return "\n".join(files) if files else "No matches found"
+        except Exception as e:
+            return f"Error searching: {e}"
+
+
+register(SearchFilesTool())

devsper/tools/filesystem/write_file.py

@@ -0,0 +1,41 @@
+"""Write content to a file."""
+
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class WriteFileTool(Tool):
+    """Write text content to a file. Overwrites if the file exists."""
+
+    name = "write_file"
+    description = "Write content to a file. Creates parent dirs if needed. Overwrites existing file."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "path": {"type": "string", "description": "Path to the file"},
+            "content": {"type": "string", "description": "Content to write"},
+        },
+        "required": ["path", "content"],
+    }
+
+    def run(self, **kwargs) -> str:
+        path = kwargs.get("path")
+        content = kwargs.get("content")
+        if not path or not isinstance(path, str):
+            return "Error: path must be a non-empty string"
+        if content is None:
+            return "Error: content is required"
+        if not isinstance(content, str):
+            content = str(content)
+        p = Path(path).resolve()
+        try:
+            p.parent.mkdir(parents=True, exist_ok=True)
+            p.write_text(content, encoding="utf-8")
+            return f"Wrote {len(content)} characters to {p}"
+        except Exception as e:
+            return f"Error writing file: {e}"
+
+
+register(WriteFileTool())

devsper/tools/flagship/__init__.py

@@ -0,0 +1,15 @@
+"""Flagship high-value tools: docproc corpus, research graph, repo map, experiment runner, distributed document analysis."""
+
+from devsper.tools.flagship.docproc_corpus_pipeline import DocprocCorpusPipelineTool
+from devsper.tools.flagship.research_graph_builder import ResearchGraphBuilderTool
+from devsper.tools.flagship.repository_semantic_map import RepositorySemanticMapTool
+from devsper.tools.flagship.swarm_experiment_runner import SwarmExperimentRunnerTool
+from devsper.tools.flagship.distributed_document_analysis import DistributedDocumentAnalysisTool
+
+__all__ = [
+    "DocprocCorpusPipelineTool",
+    "ResearchGraphBuilderTool",
+    "RepositorySemanticMapTool",
+    "SwarmExperimentRunnerTool",
+    "DistributedDocumentAnalysisTool",
+]

devsper/tools/flagship/distributed_document_analysis.py

@@ -0,0 +1,77 @@
+"""Analyze hundreds of documents using swarm parallelism: batch → swarm tasks → aggregate insights."""
+
+import json
+from pathlib import Path
+from unittest.mock import patch
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+from devsper.types.task import Task
+from devsper.swarm.scheduler import Scheduler
+from devsper.swarm.executor import Executor
+from devsper.agents.agent import Agent
+from devsper.utils.event_logger import EventLog
+from devsper.tools.documents._docproc import run_docproc_to_markdown, DOCPROC_EXTENSIONS
+
+
+class DistributedDocumentAnalysisTool(Tool):
+    """
+    Analyze many documents using swarm parallelism: split into batches, spawn swarm tasks,
+    collect summaries, aggregate into a single research report.
+    """
+
+    name = "distributed_document_analysis"
+    description = "Analyze hundreds of documents using swarm parallelism; returns aggregated research report."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "documents": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": "List of document paths to analyze",
+            },
+        },
+        "required": ["documents"],
+    }
+
+    def run(self, **kwargs) -> str:
+        documents = kwargs.get("documents")
+        if not documents or not isinstance(documents, list):
+            return "Error: documents must be a non-empty list of paths"
+        paths = [p for p in documents if isinstance(p, str) and p.strip()][:30]
+
+        tasks = []
+        for i, path in enumerate(paths):
+            task_id = f"doc_{i}"
+            deps = [f"doc_{i - 1}"] if i > 0 else []
+            tasks.append(Task(id=task_id, description=f"Summarize the document at: {path}", dependencies=deps))
+
+        scheduler = Scheduler()
+        scheduler.add_tasks(tasks)
+        log = EventLog()
+
+        def _mock_generate(model_name: str, prompt: str) -> str:
+            for p in paths:
+                if p in prompt:
+                    content, _ = run_docproc_to_markdown(p)
+                    if content:
+                        return (content[:300] + "...") if len(content) > 300 else content
+                    return f"Summary of document at {p} (no content extracted)."
+            return "Summary unavailable."
+
+        with patch("devsper.agents.agent.generate", side_effect=_mock_generate):
+            agent = Agent(model_name="mock", event_log=log)
+            executor = Executor(scheduler=scheduler, agent=agent, worker_count=4, event_log=log)
+            executor.run_sync()
+
+        results = scheduler.get_results()
+        summaries = [results.get(f"doc_{i}", "") for i in range(len(paths))]
+        aggregated = {
+            "total_documents": len(paths),
+            "summaries": [{"path": p, "summary": s[:500]} for p, s in zip(paths, summaries)],
+            "insights": f"Processed {len(paths)} documents with swarm parallelism. Total summaries: {len(summaries)}.",
+        }
+        return json.dumps(aggregated, indent=2)
+
+
+register(DistributedDocumentAnalysisTool())

devsper/tools/flagship/docproc_corpus_pipeline.py

@@ -0,0 +1,91 @@
+"""Convert a directory of PDFs/DOCX/PPTX into a structured research corpus (discover → extract → markdown → sections → JSON)."""
+
+import json
+import re
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+from devsper.tools.documents._docproc import run_docproc_to_markdown, DOCPROC_EXTENSIONS
+
+
+def _extract_sections(markdown: str) -> list[dict]:
+    """Extract sections from markdown (headers and following content)."""
+    sections = []
+    current = {"title": "", "content": "", "word_count": 0}
+    for line in markdown.splitlines():
+        if line.startswith("#"):
+            if current["title"] or current["content"]:
+                current["word_count"] = len(current["content"].split())
+                sections.append(current)
+            level = len(line) - len(line.lstrip("#"))
+            title = line.lstrip("#").strip()
+            current = {"title": title, "content": "", "word_count": 0}
+        else:
+            current["content"] = (current["content"] + "\n" + line).strip()
+    if current["title"] or current["content"]:
+        current["word_count"] = len(current["content"].split())
+        sections.append(current)
+    return sections
+
+
+def _extract_citations(text: str) -> list[str]:
+    """Heuristic citation extraction."""
+    refs = []
+    refs.extend(re.findall(r"\[\d+(?:\s*[-–,]\s*\d+)*\]", text))
+    refs.extend(re.findall(r"\([A-Z][a-z]+(?:\s+et\s+al\.?)?,?\s*\d{4}\)", text))
+    return list(dict.fromkeys(refs))
+
+
+class DocprocCorpusPipelineTool(Tool):
+    """
+    Convert a directory of PDF/DOCX/PPTX into a structured research dataset:
+    discover files → docproc extraction → markdown → sections → structured JSON corpus.
+    """
+
+    name = "docproc_corpus_pipeline"
+    description = "Convert a directory of PDFs/DOCX/PPTX into a structured research dataset (markdown, sections, word counts, citations)."
+    input_schema = {
+        "type": "object",
+        "properties": {"directory": {"type": "string", "description": "Path to directory containing documents"}},
+        "required": ["directory"],
+    }
+
+    def run(self, **kwargs) -> str:
+        directory = kwargs.get("directory")
+        if not directory or not isinstance(directory, str):
+            return "Error: directory must be a non-empty string"
+        root = Path(directory).resolve()
+        if not root.exists() or not root.is_dir():
+            return f"Error: directory not found: {directory}"
+
+        files = []
+        for ext in DOCPROC_EXTENSIONS:
+            files.extend(root.rglob(f"*{ext}"))
+        files = [f for f in files if f.is_file()][:50]
+
+        corpus = []
+        for path in files:
+            content, err = run_docproc_to_markdown(str(path))
+            if err:
+                corpus.append({"path": str(path.name), "error": err, "title": path.stem, "sections": [], "word_count": 0, "citations": []})
+                continue
+            md = content or ""
+            sections = _extract_sections(md)
+            citations = _extract_citations(md)
+            word_count = sum(s.get("word_count", 0) for s in sections) or len(md.split())
+            title = path.stem
+            if sections and sections[0].get("title"):
+                title = sections[0]["title"][:200]
+            corpus.append({
+                "title": title,
+                "path": str(path.name),
+                "sections": [{"title": s["title"], "word_count": s["word_count"]} for s in sections],
+                "word_count": word_count,
+                "citations": citations[:30],
+            })
+
+        return json.dumps({"documents": corpus, "total_files": len(files)}, indent=2)
+
+
+register(DocprocCorpusPipelineTool())

devsper/tools/flagship/repository_semantic_map.py

@@ -0,0 +1,99 @@
+"""Generate a semantic architecture map of a codebase: modules, dependencies, entrypoints, summary."""
+
+import ast
+import json
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class RepositorySemanticMapTool(Tool):
+    """
+    Generate a semantic architecture map of a codebase: scan files, parse AST,
+    extract modules, build dependency graph, summarize responsibilities.
+    Output: modules, dependencies, entrypoints, architecture_summary.
+    """
+
+    name = "repository_semantic_map"
+    description = "Generate a semantic architecture map of a codebase: modules, dependencies, entrypoints, architecture_summary."
+    input_schema = {
+        "type": "object",
+        "properties": {"repo_path": {"type": "string", "description": "Path to repository root"}},
+        "required": ["repo_path"],
+    }
+
+    def _imports(self, p: Path, root: Path) -> list[str]:
+        rel = str(p.relative_to(root)).replace("\\", "/").replace(".py", "").replace("/", ".")
+        imports = []
+        try:
+            tree = ast.parse(p.read_text(encoding="utf-8", errors="replace"))
+            for node in ast.walk(tree):
+                if isinstance(node, ast.Import):
+                    for alias in node.names:
+                        imports.append(alias.name.split(".")[0])
+                elif isinstance(node, ast.ImportFrom):
+                    if node.module:
+                        imports.append(node.module.split(".")[0])
+        except (SyntaxError, OSError):
+            pass
+        return list(dict.fromkeys(imports))
+
+    def _responsibility(self, p: Path) -> str:
+        try:
+            tree = ast.parse(p.read_text(encoding="utf-8", errors="replace"))
+            doc = ast.get_docstring(tree)
+            if doc:
+                return doc.split("\n")[0][:120].strip()
+            first = p.read_text(encoding="utf-8", errors="replace").strip().split("\n")
+            for line in first[:5]:
+                if line.strip().startswith("class ") or line.strip().startswith("def "):
+                    return line.strip()[:80]
+        except (SyntaxError, OSError):
+            pass
+        return ""
+
+    def run(self, **kwargs) -> str:
+        repo_path = kwargs.get("repo_path")
+        if not repo_path or not isinstance(repo_path, str):
+            return "Error: repo_path must be a non-empty string"
+        root = Path(repo_path).resolve()
+        if not root.exists() or not root.is_dir():
+            return f"Error: repo_path not found: {repo_path}"
+
+        modules = []
+        dependencies = []
+        entrypoints = []
+        max_files = 150
+
+        py_files = list(root.rglob("*.py"))
+        for p in py_files[:max_files]:
+            if not p.is_file():
+                continue
+            rel = str(p.relative_to(root)).replace("\\", "/")
+            mod = rel.replace(".py", "").replace("/", ".")
+            if "__pycache__" in mod or mod.startswith("."):
+                continue
+            resp = self._responsibility(p)
+            modules.append({"module": mod, "file": rel, "responsibility": resp})
+            for imp in self._imports(p, root):
+                if imp and imp != mod.split(".")[0]:
+                    dependencies.append({"from": mod, "to": imp})
+            if "main" in p.name or "__main__" in rel or "run" in p.stem.lower():
+                entrypoints.append(mod)
+
+        entrypoints = list(dict.fromkeys(entrypoints))[:20]
+        architecture_summary = (
+            f"Modules: {len(modules)}. Dependencies: {len(dependencies)}. "
+            f"Entrypoints: {', '.join(entrypoints[:5]) or 'none detected'}."
+        )
+
+        return json.dumps({
+            "modules": modules[:80],
+            "dependencies": dependencies[:200],
+            "entrypoints": entrypoints,
+            "architecture_summary": architecture_summary,
+        }, indent=2)
+
+
+register(RepositorySemanticMapTool())

devsper/tools/flagship/research_graph_builder.py

@@ -0,0 +1,111 @@
+"""Build a knowledge graph from research documents: entities, citations, methods, datasets."""
+
+import json
+import re
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+ENTITY_PATTERN = re.compile(r"\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)\b")
+CITE_PATTERN = re.compile(r"\[[\d\s,–-]+\]|\([A-Z][a-z]+(?:\s+et\s+al\.?)?,?\s*\d{4}\)")
+METHOD_PATTERN = re.compile(r"\b(?:method|approach|algorithm|model|framework|technique)s?\s+[:\s]+([^.!?\n]{10,80})", re.I)
+DATASET_PATTERN = re.compile(r"\b(?:dataset|corpus|benchmark)s?\s+[:\s]*([A-Za-z0-9\-]+(?:\s+[A-Za-z0-9\-]+)*)", re.I)
+
+
+def _extract_entities(text: str, max_n: int = 100) -> list[str]:
+    candidates = ENTITY_PATTERN.findall(text)
+    seen = set()
+    out = []
+    for c in candidates:
+        c = c.strip()
+        if len(c) > 2 and c not in seen and not c.isdigit():
+            seen.add(c)
+            out.append(c)
+            if len(out) >= max_n:
+                break
+    return out
+
+
+def _extract_citations(text: str, max_n: int = 50) -> list[str]:
+    return list(dict.fromkeys(CITE_PATTERN.findall(text)))[:max_n]
+
+
+def _extract_methods(text: str, max_n: int = 30) -> list[str]:
+    return [m.group(1).strip()[:80] for m in METHOD_PATTERN.finditer(text)][:max_n]
+
+
+def _extract_datasets(text: str, max_n: int = 20) -> list[str]:
+    return [m.group(1).strip() for m in DATASET_PATTERN.finditer(text)][:max_n]
+
+
+class ResearchGraphBuilderTool(Tool):
+    """
+    Build a graph of research knowledge from documents: entities, citations, methods, datasets.
+    Output: nodes, edges, entity types (knowledge graph JSON).
+    """
+
+    name = "research_graph_builder"
+    description = "Build a knowledge graph from research documents: extract entities, citations, methods, datasets."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "documents": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": "List of document texts to analyze",
+            },
+        },
+        "required": ["documents"],
+    }
+
+    def run(self, **kwargs) -> str:
+        documents = kwargs.get("documents")
+        if not documents or not isinstance(documents, list):
+            return "Error: documents must be a non-empty list of strings"
+        texts = [d for d in documents if isinstance(d, str) and d.strip()][:50]
+        if not texts:
+            return "Error: no valid document strings provided"
+
+        nodes = []
+        edges = []
+        entity_types = {}
+
+        for i, text in enumerate(texts):
+            doc_id = f"doc_{i}"
+            nodes.append({"id": doc_id, "type": "document"})
+            entities = _extract_entities(text)
+            for e in entities:
+                nid = f"entity_{e.replace(' ', '_')[:40]}"
+                if not any(n["id"] == nid for n in nodes):
+                    nodes.append({"id": nid, "type": "entity", "label": e})
+                    entity_types[nid] = "entity"
+                edges.append({"source": doc_id, "target": nid, "type": "mentions"})
+
+            for c in _extract_citations(text):
+                cid = f"cite_{hash(c) % 10**8}"
+                nodes.append({"id": cid, "type": "citation", "label": c[:60]})
+                edges.append({"source": doc_id, "target": cid, "type": "cites"})
+
+            for m in _extract_methods(text):
+                mid = f"method_{hash(m) % 10**8}"
+                nodes.append({"id": mid, "type": "method", "label": m[:80]})
+                entity_types[mid] = "method"
+                edges.append({"source": doc_id, "target": mid, "type": "describes_method"})
+
+            for d in _extract_datasets(text):
+                did = f"dataset_{d.replace(' ', '_')[:30]}"
+                if not any(n["id"] == did for n in nodes):
+                    nodes.append({"id": did, "type": "dataset", "label": d})
+                    entity_types[did] = "dataset"
+                edges.append({"source": doc_id, "target": did, "type": "uses_dataset"})
+
+        by_id = {n["id"]: n for n in nodes}
+        nodes = list(by_id.values())
+
+        return json.dumps({
+            "nodes": nodes,
+            "edges": edges,
+            "entity_types": entity_types,
+        }, indent=2)
+
+
+register(ResearchGraphBuilderTool())

devsper/tools/flagship/swarm_experiment_runner.py

@@ -0,0 +1,86 @@
+"""Run parameter sweeps or experiments using the swarm runtime; executor divides runs across workers."""
+
+import json
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+from devsper.types.task import Task
+from devsper.swarm.scheduler import Scheduler
+from devsper.swarm.executor import Executor
+from devsper.agents.agent import Agent
+from devsper.utils.event_logger import EventLog
+
+
+def _run_experiment_once(
+    task_description: str,
+    worker_model: str,
+    worker_count: int,
+) -> float:
+    """Run a single swarm-style run and return a numeric metric (e.g. result length sum)."""
+    from unittest.mock import patch
+    tasks = [
+        Task(id=f"run_1", description=task_description, dependencies=[]),
+    ]
+    scheduler = Scheduler()
+    scheduler.add_tasks(tasks)
+    log = EventLog()
+    with patch("devsper.agents.agent.generate", return_value="Experiment result output."):
+        agent = Agent(model_name=worker_model, event_log=log)
+        executor = Executor(scheduler=scheduler, agent=agent, worker_count=worker_count, event_log=log)
+        executor.run_sync()
+    results = scheduler.get_results()
+    total = sum(len(r or "") for r in results.values()) or 1.0
+    return float(total)
+
+
+class SwarmExperimentRunnerTool(Tool):
+    """
+    Run parameter sweeps or experiments using the swarm runtime.
+    Executor divides runs across workers. Output: mean, std, best configuration.
+    """
+
+    name = "swarm_experiment_runner"
+    description = "Run parameter sweeps or experiments using the swarm runtime; returns experiment statistics (mean, std, best)."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "parameters": {"type": "object", "description": "Experiment parameters (e.g. task description, config)"},
+            "runs": {"type": "integer", "description": "Number of runs (default 3)"},
+        },
+        "required": ["parameters"],
+    }
+
+    def run(self, **kwargs) -> str:
+        parameters = kwargs.get("parameters")
+        runs = kwargs.get("runs", 3)
+        if not isinstance(parameters, dict):
+            return "Error: parameters must be an object"
+        if not isinstance(runs, int) or runs < 1:
+            runs = 3
+        runs = min(runs, 20)
+
+        task_description = parameters.get("task", parameters.get("prompt", "Run experiment."))
+        if not isinstance(task_description, str):
+            task_description = str(parameters)[:500]
+
+        metrics = []
+        for _ in range(runs):
+            m = _run_experiment_once(task_description, worker_model="mock", worker_count=2)
+            metrics.append(m)
+
+        n = len(metrics)
+        mean = sum(metrics) / n
+        variance = sum((x - mean) ** 2 for x in metrics) / n if n else 0
+        std = variance ** 0.5
+        best_idx = max(range(n), key=lambda i: metrics[i])
+        best_config = {"run_index": best_idx, "metric": metrics[best_idx]}
+
+        return json.dumps({
+            "mean": round(mean, 4),
+            "std": round(std, 4),
+            "runs": n,
+            "best_configuration": best_config,
+            "metrics": [round(x, 4) for x in metrics],
+        }, indent=2)
+
+
+register(SwarmExperimentRunnerTool())

devsper/tools/knowledge/__init__.py

@@ -0,0 +1,10 @@
+"""Knowledge pipeline tools: corpus, topics, citations, knowledge graphs, timelines."""
+
+from devsper.tools.knowledge.corpus_builder import CorpusBuilderTool
+from devsper.tools.knowledge.document_corpus_summary import DocumentCorpusSummaryTool
+from devsper.tools.knowledge.document_topic_extractor import DocumentTopicExtractorTool
+from devsper.tools.knowledge.citation_graph_builder import CitationGraphBuilderTool
+from devsper.tools.knowledge.knowledge_graph_extractor import KnowledgeGraphExtractorTool
+from devsper.tools.knowledge.concept_frequency_analyzer import ConceptFrequencyAnalyzerTool
+from devsper.tools.knowledge.timeline_extractor import TimelineExtractorTool
+from devsper.tools.knowledge.cross_document_entity_linker import CrossDocumentEntityLinkerTool