devsper 2.1.6__py3-none-any.whl

devsper/tools/research/research_question_generator.py

@@ -0,0 +1,39 @@
+"""Generate simple research questions from a topic (template-based, no LLM)."""
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class ResearchQuestionGeneratorTool(Tool):
+    """Generate example research questions for a topic using fixed templates."""
+
+    name = "research_question_generator"
+    description = "Generate sample research questions for a topic (template-based)."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "topic": {"type": "string", "description": "Research topic or domain"},
+            "count": {"type": "integer", "description": "Number of questions (default 3)"},
+        },
+        "required": ["topic"],
+    }
+
+    def run(self, **kwargs) -> str:
+        topic = kwargs.get("topic")
+        count = kwargs.get("count", 3)
+        if not topic or not isinstance(topic, str):
+            return "Error: topic must be a non-empty string"
+        if not isinstance(count, int) or count < 1:
+            count = 3
+        templates = [
+            f"What are the main challenges in {topic}?",
+            f"How has {topic} evolved in recent years?",
+            f"What are the best practices for applying {topic}?",
+            f"What is the relationship between {topic} and related fields?",
+            f"What open problems remain in {topic}?",
+        ]
+        chosen = templates[: min(count, len(templates))]
+        return "Suggested research questions:\n" + "\n".join(f"- {q}" for q in chosen)
+
+
+register(ResearchQuestionGeneratorTool())

devsper/tools/research/topic_cluster.py

@@ -0,0 +1,46 @@
+"""Simple topic clustering by common words across titles/texts."""
+
+import re
+from collections import Counter
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class TopicClusterTool(Tool):
+    """Cluster a list of titles or short texts by common keywords (simple word overlap)."""
+
+    name = "topic_cluster"
+    description = "Find common topics across a list of text snippets. Returns top shared words."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "texts": {"type": "array", "description": "List of strings (e.g. paper titles or abstracts)"},
+            "top_n": {"type": "integer", "description": "Number of top words to return (default 10)"},
+        },
+        "required": ["texts"],
+    }
+
+    def run(self, **kwargs) -> str:
+        texts = kwargs.get("texts")
+        top_n = kwargs.get("top_n", 10)
+        if not isinstance(texts, list):
+            return "Error: texts must be an array of strings"
+        if not isinstance(top_n, int) or top_n < 1:
+            top_n = 10
+        stop = {"the", "a", "an", "and", "or", "of", "in", "on", "to", "for", "with", "by", "is", "are", "be", "as", "at"}
+        word_counts = Counter()
+        for t in texts:
+            if not isinstance(t, str):
+                continue
+            words = re.findall(r"\b[a-zA-Z]{3,}\b", t.lower())
+            for w in words:
+                if w not in stop:
+                    word_counts[w] += 1
+        common = word_counts.most_common(top_n)
+        if not common:
+            return "No significant keywords found."
+        return "Top shared terms:\n" + "\n".join(f"{w}: {c}" for w, c in common)
+
+
+register(TopicClusterTool())

devsper/tools/research/web_search.py

@@ -0,0 +1,47 @@
+"""Web search using DuckDuckGo when duckduckgo-search is available."""
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class WebSearchTool(Tool):
+    """Search the web. Uses DuckDuckGo when the duckduckgo-search package is installed."""
+
+    name = "web_search"
+    description = "Search the web for a query. Returns titles and snippets. Requires duckduckgo-search."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "query": {"type": "string", "description": "Search query"},
+            "max_results": {"type": "integer", "description": "Max results to return (default 5)"},
+        },
+        "required": ["query"],
+    }
+
+    def run(self, **kwargs) -> str:
+        query = kwargs.get("query")
+        max_results = kwargs.get("max_results", 5)
+        if not query or not isinstance(query, str):
+            return "Error: query must be a non-empty string"
+        if not isinstance(max_results, int) or max_results < 1:
+            max_results = 5
+        try:
+            from duckduckgo_search import DDGS
+            with DDGS() as ddgs:
+                results = list(ddgs.text(query, max_results=max_results))
+            if not results:
+                return "No results found."
+            lines = []
+            for i, r in enumerate(results, 1):
+                title = r.get("title", "")
+                body = r.get("body", "")
+                href = r.get("href", "")
+                lines.append(f"{i}. {title}\n   {body}\n   {href}")
+            return "\n\n".join(lines)
+        except ImportError:
+            return "Error: Install duckduckgo-search (pip install duckduckgo-search) for web search."
+        except Exception as e:
+            return f"Error: {e}"
+
+
+register(WebSearchTool())

devsper/tools/research/wikipedia_lookup.py

@@ -0,0 +1,50 @@
+"""Look up a topic on Wikipedia using the JSON API."""
+
+import json
+import urllib.parse
+import urllib.request
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+
+
+class WikipediaLookupTool(Tool):
+    """Fetch a Wikipedia article summary for a topic."""
+
+    name = "wikipedia_lookup"
+    description = "Get a short Wikipedia summary for a topic. Uses the Wikipedia API."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "topic": {"type": "string", "description": "Topic or article title to look up"},
+        },
+        "required": ["topic"],
+    }
+
+    def run(self, **kwargs) -> str:
+        topic = kwargs.get("topic")
+        if not topic or not isinstance(topic, str):
+            return "Error: topic must be a non-empty string"
+        try:
+            url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode({
+                "action": "query",
+                "titles": topic.strip(),
+                "prop": "extracts",
+                "exintro": True,
+                "explaintext": True,
+                "format": "json",
+            })
+            req = urllib.request.Request(url, headers={"User-Agent": "devsper/1.0"})
+            with urllib.request.urlopen(req, timeout=10) as resp:
+                data = json.loads(resp.read().decode())
+            pages = data.get("query", {}).get("pages", {})
+            page = next(iter(pages.values()), None)
+            if not page or page.get("missing"):
+                return f"No Wikipedia article found for: {topic}"
+            extract = page.get("extract", "").strip()
+            return extract or "No extract available."
+        except Exception as e:
+            return f"Error: {e}"
+
+
+register(WikipediaLookupTool())

devsper/tools/research_advanced/__init__.py

@@ -0,0 +1,14 @@
+"""Advanced research tools: literature review, similarity, gaps, methodology, citations."""
+
+from devsper.tools.research_advanced.literature_review_generator import LiteratureReviewGeneratorTool
+from devsper.tools.research_advanced.paper_similarity_search import PaperSimilaritySearchTool
+from devsper.tools.research_advanced.research_gap_finder import ResearchGapFinderTool
+from devsper.tools.research_advanced.methodology_extractor import MethodologyExtractorTool
+from devsper.tools.research_advanced.paper_contribution_extractor import PaperContributionExtractorTool
+from devsper.tools.research_advanced.paper_dataset_identifier import PaperDatasetIdentifierTool
+from devsper.tools.research_advanced.citation_context_extractor import CitationContextExtractorTool
+from devsper.tools.research_advanced.research_topic_mapper import ResearchTopicMapperTool
+from devsper.tools.research_advanced.paper_trend_analyzer import PaperTrendAnalyzerTool
+from devsper.tools.research_advanced.paper_method_comparator import PaperMethodComparatorTool
+from devsper.tools.research_advanced.parallel_document_analyzer import ParallelDocumentAnalyzerTool
+from devsper.tools.research_advanced.swarm_literature_review import SwarmLiteratureReviewTool

devsper/tools/research_advanced/citation_context_extractor.py

@@ -0,0 +1,60 @@
+"""Extract the surrounding context (sentence) for each citation in the text."""
+
+import re
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+from devsper.tools.documents._docproc import run_docproc_to_markdown
+
+CITE_PATTERNS = [
+    re.compile(r"\([^)]*?\b(?:et\s+al\.?|&\s*[^)]+)[^)]*?\d{4}[^)]*\)", re.I),
+    re.compile(r"\[\d+(?:\s*[-–,]\s*\d+)*\]"),
+    re.compile(r"\([A-Z][a-z]+(?:\s+et\s+al\.?)?,?\s*\d{4}\)"),
+]
+
+
+class CitationContextExtractorTool(Tool):
+    """
+    For each citation pattern found, extract the containing sentence as context.
+    """
+
+    name = "citation_context_extractor"
+    description = "Extract the sentence context around each citation in a document."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "file_path": {"type": "string", "description": "Path to the document"},
+            "max_contexts": {"type": "integer", "description": "Max citation contexts (default 25)"},
+        },
+        "required": ["file_path"],
+    }
+
+    def run(self, **kwargs) -> str:
+        file_path = kwargs.get("file_path")
+        max_contexts = kwargs.get("max_contexts", 25)
+        if not file_path or not isinstance(file_path, str):
+            return "Error: file_path must be a non-empty string"
+        if not isinstance(max_contexts, int) or max_contexts < 1:
+            max_contexts = 25
+        content, err = run_docproc_to_markdown(file_path)
+        if err:
+            return err
+        text = content or ""
+        sentences = re.split(r"(?<=[.!?])\s+", text)
+        results = []
+        seen = set()
+        for sent in sentences:
+            for pat in CITE_PATTERNS:
+                if pat.search(sent) and sent not in seen:
+                    seen.add(sent)
+                    results.append(sent.strip()[:400])
+                    break
+            if len(results) >= max_contexts:
+                break
+        if not results:
+            return "No citation contexts found."
+        return "Citation contexts:\n\n" + "\n\n".join(results)
+
+
+register(CitationContextExtractorTool())

devsper/tools/research_advanced/literature_review_generator.py

@@ -0,0 +1,79 @@
+"""Generate a structured literature review outline from multiple paper abstracts or texts."""
+
+import re
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+from devsper.tools.documents._docproc import run_docproc_to_markdown
+
+STOP = frozenset("a an the and or but in on at to for of with by from as is was are were been be have has had do does did will would could should may might must can this that it its".split())
+
+
+class LiteratureReviewGeneratorTool(Tool):
+    """
+    Produce a literature review structure: common themes, key terms, and doc summaries from multiple papers.
+    """
+
+    name = "literature_review_generator"
+    description = "Generate a structured literature review from multiple paper texts or document paths."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "file_paths": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": "Paths to papers (PDF, DOCX, etc.)",
+            },
+            "texts": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": "Alternatively, raw text/abstracts",
+            },
+            "top_themes": {"type": "integer", "description": "Number of theme keywords (default 15)"},
+        },
+        "required": [],
+    }
+
+    def _get_texts(self, file_paths: list | None, texts: list | None) -> list[str]:
+        out = []
+        if texts and isinstance(texts, list):
+            for t in texts:
+                if isinstance(t, str) and t.strip():
+                    out.append(t.strip())
+        if file_paths and isinstance(file_paths, list):
+            for path in file_paths:
+                if not isinstance(path, str) or not path.strip():
+                    continue
+                content, err = run_docproc_to_markdown(path.strip())
+                if not err and content:
+                    out.append(content.strip())
+        return out
+
+    def run(self, **kwargs) -> str:
+        file_paths = kwargs.get("file_paths")
+        texts = kwargs.get("texts")
+        top_themes = kwargs.get("top_themes", 15)
+        if not isinstance(top_themes, int) or top_themes < 1:
+            top_themes = 15
+        all_texts = self._get_texts(file_paths, texts)
+        if not all_texts:
+            return "Error: provide file_paths or texts (non-empty)"
+        combined = " ".join(all_texts).lower()
+        words = [w for w in re.findall(r"[a-z]+", combined) if len(w) >= 4 and w not in STOP]
+        from collections import Counter
+
+        counts = Counter(words)
+        themes = [w for w, _ in counts.most_common(top_themes)]
+        lines = [
+            "Literature review outline",
+            "=" * 40,
+            "Common themes (keywords): " + ", ".join(themes),
+            "",
+            f"Documents/sources: {len(all_texts)}",
+            "Per-source length (chars): " + ", ".join(str(len(t)) for t in all_texts),
+        ]
+        return "\n".join(lines)
+
+
+register(LiteratureReviewGeneratorTool())

devsper/tools/research_advanced/methodology_extractor.py

@@ -0,0 +1,58 @@
+"""Extract methodology-related sentences from paper text using keyword heuristics."""
+
+import re
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+from devsper.tools.documents._docproc import run_docproc_to_markdown
+
+METHOD_KEYWORDS = re.compile(
+    r"\b(method|methods|methodology|experiment|dataset|evaluation|approach|algorithm|framework|procedure|protocol|design|implementation|training|model)\b",
+    re.I,
+)
+
+
+class MethodologyExtractorTool(Tool):
+    """
+    Extract sentences that likely describe methodology (method, experiment, dataset, etc.).
+    """
+
+    name = "methodology_extractor"
+    description = "Extract methodology-related sentences from a paper (PDF, DOCX, etc.)."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "file_path": {"type": "string", "description": "Path to the document"},
+            "max_sentences": {"type": "integer", "description": "Max sentences (default 20)"},
+        },
+        "required": ["file_path"],
+    }
+
+    def run(self, **kwargs) -> str:
+        file_path = kwargs.get("file_path")
+        max_sentences = kwargs.get("max_sentences", 20)
+        if not file_path or not isinstance(file_path, str):
+            return "Error: file_path must be a non-empty string"
+        if not isinstance(max_sentences, int) or max_sentences < 1:
+            max_sentences = 20
+        content, err = run_docproc_to_markdown(file_path)
+        if err:
+            return err
+        text = content or ""
+        sentences = re.split(r"(?<=[.!?])\s+", text)
+        selected = []
+        for s in sentences:
+            s = s.strip()
+            if len(s) < 20:
+                continue
+            if METHOD_KEYWORDS.search(s):
+                selected.append(s[:500])
+                if len(selected) >= max_sentences:
+                    break
+        if not selected:
+            return "No methodology sentences found (look for method/experiment/dataset/evaluation)."
+        return "Methodology sentences:\n\n" + "\n\n".join(selected)
+
+
+register(MethodologyExtractorTool())

devsper/tools/research_advanced/paper_contribution_extractor.py

@@ -0,0 +1,50 @@
+"""Extract contribution-like statements from paper text (we propose, our contribution, etc.)."""
+
+import re
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+from devsper.tools.documents._docproc import run_docproc_to_markdown
+
+CONTRIBUTION_PAT = re.compile(
+    r"[^.!?]*(?:we\s+propose|our\s+contribution|this\s+paper\s+(?:presents|introduces|shows)|we\s+introduce|we\s+present|we\s+show|main\s+contribution)[^.!?]*[.!?]",
+    re.I,
+)
+
+
+class PaperContributionExtractorTool(Tool):
+    """
+    Extract sentences that state contributions (we propose, our contribution, this paper presents, etc.).
+    """
+
+    name = "paper_contribution_extractor"
+    description = "Extract contribution statements from a paper."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "file_path": {"type": "string", "description": "Path to the document"},
+            "max_sentences": {"type": "integer", "description": "Max sentences (default 15)"},
+        },
+        "required": ["file_path"],
+    }
+
+    def run(self, **kwargs) -> str:
+        file_path = kwargs.get("file_path")
+        max_sentences = kwargs.get("max_sentences", 15)
+        if not file_path or not isinstance(file_path, str):
+            return "Error: file_path must be a non-empty string"
+        if not isinstance(max_sentences, int) or max_sentences < 1:
+            max_sentences = 15
+        content, err = run_docproc_to_markdown(file_path)
+        if err:
+            return err
+        text = content or ""
+        matches = CONTRIBUTION_PAT.findall(text)
+        selected = [m.strip()[:400] for m in matches[:max_sentences]]
+        if not selected:
+            return "No contribution sentences found (look for 'we propose', 'our contribution', etc.)."
+        return "Contribution statements:\n\n" + "\n\n".join(selected)
+
+
+register(PaperContributionExtractorTool())

devsper/tools/research_advanced/paper_dataset_identifier.py

@@ -0,0 +1,49 @@
+"""Identify dataset names mentioned in paper text using common patterns."""
+
+import re
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+from devsper.tools.documents._docproc import run_docproc_to_markdown
+
+DATASET_PAT = re.compile(
+    r"\b([A-Z][A-Za-z0-9\-]+(?:\s+[A-Z][A-Za-z0-9\-]+)*)\s+(?:dataset|benchmark|corpus)\b|\b(?:on|using|with)\s+([A-Z][A-Za-z0-9\-]+(?:\s+[A-Z][A-Za-z0-9\-]+)*)\s+(?:dataset|benchmark)?",
+    re.I,
+)
+
+
+class PaperDatasetIdentifierTool(Tool):
+    """
+    Find dataset and benchmark names mentioned in a paper.
+    """
+
+    name = "paper_dataset_identifier"
+    description = "Identify dataset and benchmark names mentioned in a paper."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "file_path": {"type": "string", "description": "Path to the document"},
+        },
+        "required": ["file_path"],
+    }
+
+    def run(self, **kwargs) -> str:
+        file_path = kwargs.get("file_path")
+        if not file_path or not isinstance(file_path, str):
+            return "Error: file_path must be a non-empty string"
+        content, err = run_docproc_to_markdown(file_path)
+        if err:
+            return err
+        text = content or ""
+        found = set()
+        for m in DATASET_PAT.finditer(text):
+            for g in m.groups():
+                if g and len(g) > 1:
+                    found.add(g.strip())
+        if not found:
+            return "No dataset/benchmark names identified (look for 'X dataset', 'on X', 'using X')."
+        return "Datasets/benchmarks: " + ", ".join(sorted(found))
+
+
+register(PaperDatasetIdentifierTool())

devsper/tools/research_advanced/paper_method_comparator.py

@@ -0,0 +1,62 @@
+"""Compare methodology-related terms and sentences across multiple papers."""
+
+import re
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+from devsper.tools.documents._docproc import run_docproc_to_markdown
+
+METHOD_WORDS = re.compile(
+    r"\b(method|algorithm|model|framework|dataset|evaluation|training|accuracy|baseline|neural|transformer|embedding)\b",
+    re.I,
+)
+
+
+class PaperMethodComparatorTool(Tool):
+    """
+    Compare methods across papers: shared methodology terms and per-paper method term counts.
+    """
+
+    name = "paper_method_comparator"
+    description = "Compare methodology terms and sentences across multiple papers."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "file_paths": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": "Paths to papers",
+            },
+        },
+        "required": ["file_paths"],
+    }
+
+    def run(self, **kwargs) -> str:
+        file_paths = kwargs.get("file_paths")
+        if not file_paths or not isinstance(file_paths, list):
+            return "Error: file_paths must be a non-empty list"
+        doc_terms = {}
+        all_terms = set()
+        for path in file_paths:
+            if not isinstance(path, str) or not path.strip():
+                continue
+            p = Path(path.strip()).resolve()
+            if not p.exists() or not p.is_file():
+                continue
+            content, err = run_docproc_to_markdown(str(p))
+            if err:
+                continue
+            text = content or ""
+            terms = list(dict.fromkeys(METHOD_WORDS.findall(text)))
+            doc_terms[p.name] = [t.lower() for t in terms]
+            all_terms.update(t.lower() for t in terms)
+        shared = [t for t in all_terms if sum(1 for v in doc_terms.values() if t in v) > 1]
+        lines = ["Method comparison", "=" * 40]
+        for name, terms in doc_terms.items():
+            lines.append(f"{name}: " + ", ".join(terms[:15]))
+        lines.append("\nShared method terms: " + ", ".join(sorted(shared)))
+        return "\n".join(lines)
+
+
+register(PaperMethodComparatorTool())

devsper/tools/research_advanced/paper_similarity_search.py

@@ -0,0 +1,69 @@
+"""Search for papers similar to a query text using simple term overlap (no embeddings)."""
+
+import re
+from pathlib import Path
+
+from devsper.tools.base import Tool
+from devsper.tools.registry import register
+from devsper.tools.documents._docproc import run_docproc_to_markdown
+
+STOP = frozenset("a an the and or but in on at to for of with by from as is was are were been be have has had do does did will would could should may might must can this that it its".split())
+
+
+class PaperSimilaritySearchTool(Tool):
+    """
+    Rank documents by term-overlap similarity to a query (bag-of-words, no embeddings).
+    """
+
+    name = "paper_similarity_search"
+    description = "Find documents most similar to a query text using term overlap."
+    input_schema = {
+        "type": "object",
+        "properties": {
+            "query": {"type": "string", "description": "Query text or abstract"},
+            "file_paths": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": "Paths to candidate documents",
+            },
+            "top_k": {"type": "integer", "description": "Number of results (default 5)"},
+        },
+        "required": ["query", "file_paths"],
+    }
+
+    def _tokens(self, text: str, min_len: int = 3) -> set[str]:
+        words = re.findall(r"[a-z]+", text.lower())
+        return {w for w in words if len(w) >= min_len and w not in STOP}
+
+    def run(self, **kwargs) -> str:
+        query = kwargs.get("query")
+        file_paths = kwargs.get("file_paths")
+        top_k = kwargs.get("top_k", 5)
+        if not query or not isinstance(query, str):
+            return "Error: query must be a non-empty string"
+        if not file_paths or not isinstance(file_paths, list):
+            return "Error: file_paths must be a non-empty list"
+        if not isinstance(top_k, int) or top_k < 1:
+            top_k = 5
+        q_set = self._tokens(query)
+        if not q_set:
+            return "Error: query produced no tokens"
+        scores = []
+        for path in file_paths:
+            if not isinstance(path, str) or not path.strip():
+                continue
+            p = Path(path.strip()).resolve()
+            if not p.exists() or not p.is_file():
+                continue
+            content, err = run_docproc_to_markdown(str(p))
+            if err:
+                continue
+            doc_set = self._tokens(content or "")
+            overlap = len(q_set & doc_set) / len(q_set) if q_set else 0
+            scores.append((p.name, overlap))
+        scores.sort(key=lambda x: -x[1])
+        result = [f"{name}: {score:.3f}" for name, score in scores[:top_k]]
+        return "Similarity (term overlap):\n" + "\n".join(result) if result else "No documents processed."
+
+
+register(PaperSimilaritySearchTool())