tooluniverse 1.0.9.1__py3-none-any.whl → 1.0.11__py3-none-any.whl

tooluniverse/cache/result_cache_manager.py

@@ -6,9 +6,11 @@ from __future__ import annotations
 
 import logging
 import os
+import queue
+import threading
 import time
 from dataclasses import dataclass
-from typing import Any, Dict, Iterator, Optional
+from typing import Any, Dict, Iterator, Optional, Sequence
 
 from .memory_cache import LRUCache, SingleFlight
 from .sqlite_backend import CacheEntry, PersistentCache
@@ -36,6 +38,8 @@ class ResultCacheManager:
         persistence_enabled: bool = True,
         singleflight: bool = True,
         default_ttl: Optional[int] = None,
+        async_persist: Optional[bool] = None,
+        async_queue_size: int = 10000,
     ):
         self.enabled = enabled
         self.default_ttl = default_ttl
@@ -55,6 +59,7 @@ class ResultCacheManager:
                 self.persistent = None
 
         self.singleflight = SingleFlight() if singleflight else None
+        self._init_async_persistence(async_persist, async_queue_size)
 
     # ------------------------------------------------------------------
     # Helper methods
@@ -69,6 +74,33 @@ class ResultCacheManager:
     def _ttl_or_default(self, ttl: Optional[int]) -> Optional[int]:
         return ttl if ttl is not None else self.default_ttl
 
+    def _init_async_persistence(
+        self, async_persist: Optional[bool], async_queue_size: int
+    ) -> None:
+        if async_persist is None:
+            async_persist = os.getenv(
+                "TOOLUNIVERSE_CACHE_ASYNC_PERSIST", "true"
+            ).lower() in ("true", "1", "yes")
+
+        self.async_persist = (
+            async_persist and self.persistent is not None and self.enabled
+        )
+
+        self._persist_queue: Optional["queue.Queue[tuple[str, Dict[str, Any]]]"] = None
+        self._worker_thread: Optional[threading.Thread] = None
+
+        if not self.async_persist:
+            return
+
+        queue_size = max(1, async_queue_size)
+        self._persist_queue = queue.Queue(maxsize=queue_size)
+        self._worker_thread = threading.Thread(
+            target=self._async_worker,
+            name="ResultCacheWriter",
+            daemon=True,
+        )
+        self._worker_thread.start()
+
     # ------------------------------------------------------------------
     # Public API
     # ------------------------------------------------------------------
@@ -126,17 +158,15 @@ class ResultCacheManager:
         )
 
         if self.persistent:
-            try:
-                self.persistent.set(
-                    composed,
-                    value,
-                    namespace=namespace,
-                    version=version,
-                    ttl=effective_ttl,
-                )
-            except Exception as exc:
-                logger.warning("Persistent cache write failed: %s", exc)
-                self.persistent = None
+            payload = {
+                "composed": composed,
+                "value": value,
+                "namespace": namespace,
+                "version": version,
+                "ttl": effective_ttl,
+            }
+            if not self._schedule_persist("set", payload):
+                self._perform_persist_set(**payload)
 
     def delete(self, *, namespace: str, version: str, cache_key: str):
         composed = self.compose_key(namespace, version, cache_key)
@@ -162,10 +192,40 @@ class ResultCacheManager:
 
         if self.persistent:
             try:
+                self.flush()
                 self.persistent.clear(namespace=namespace)
             except Exception as exc:
                 logger.warning("Persistent cache clear failed: %s", exc)
 
+    def bulk_get(self, requests: Sequence[Dict[str, str]]) -> Dict[str, Any]:
+        """Fetch multiple cache entries at once.
+
+        Args:
+            requests: Iterable of dicts containing ``namespace``, ``version`` and ``cache_key``.
+
+        Returns
+            Mapping of composed cache keys to cached values.
+        """
+
+        if not self.enabled:
+            return {}
+
+        hits: Dict[str, Any] = {}
+        for request in requests:
+            namespace = request["namespace"]
+            version = request["version"]
+            cache_key = request["cache_key"]
+            value = self.get(
+                namespace=namespace,
+                version=version,
+                cache_key=cache_key,
+            )
+            if value is not None:
+                composed = self.compose_key(namespace, version, cache_key)
+                hits[composed] = value
+
+        return hits
+
     def stats(self) -> Dict[str, Any]:
         return {
             "enabled": self.enabled,
@@ -173,11 +233,18 @@ class ResultCacheManager:
             "persistent": (
                 self.persistent.stats() if self.persistent else {"enabled": False}
             ),
+            "async_persist": self.async_persist,
+            "pending_writes": (
+                self._persist_queue.qsize()
+                if self.async_persist and self._persist_queue is not None
+                else 0
+            ),
         }
 
     def dump(self, namespace: Optional[str] = None) -> Iterator[Dict[str, Any]]:
         if not self.persistent:
             return iter([])
+        self.flush()
         return (
             {
                 "cache_key": entry.key,
@@ -220,12 +287,100 @@ class ResultCacheManager:
         return _DummyContext()
 
     def close(self):
+        self.flush()
+        self._shutdown_async_worker()
         if self.persistent:
             try:
                 self.persistent.close()
             except Exception as exc:
                 logger.warning("Persistent cache close failed: %s", exc)
 
+    # ------------------------------------------------------------------
+    # Async persistence helpers
+    # ------------------------------------------------------------------
+
+    def flush(self):
+        if self.async_persist and self._persist_queue is not None:
+            self._persist_queue.join()
+
+    def _schedule_persist(self, op: str, payload: Dict[str, Any]) -> bool:
+        if not self.async_persist or self._persist_queue is None:
+            return False
+        try:
+            self._persist_queue.put_nowait((op, payload))
+            return True
+        except queue.Full:
+            logger.warning(
+                "Async cache queue full; falling back to synchronous persistence"
+            )
+            return False
+
+    def _async_worker(self):
+        queue_ref = self._persist_queue
+        if queue_ref is None:
+            return
+
+        while True:
+            try:
+                op, payload = queue_ref.get()
+            except Exception:
+                continue
+
+            if op == "__STOP__":
+                queue_ref.task_done()
+                break
+
+            try:
+                if op == "set":
+                    self._perform_persist_set(**payload)
+                else:
+                    logger.warning("Unknown async cache operation: %s", op)
+            except Exception as exc:
+                logger.warning("Async cache write failed: %s", exc)
+                # Disable async persistence to avoid repeated failures
+                self.async_persist = False
+            finally:
+                queue_ref.task_done()
+
+    def _perform_persist_set(
+        self,
+        *,
+        composed: str,
+        value: Any,
+        namespace: str,
+        version: str,
+        ttl: Optional[int],
+    ):
+        if not self.persistent:
+            return
+        try:
+            self.persistent.set(
+                composed,
+                value,
+                namespace=namespace,
+                version=version,
+                ttl=ttl,
+            )
+        except Exception as exc:
+            logger.warning("Persistent cache write failed: %s", exc)
+            self.persistent = None
+            raise
+
+    def _shutdown_async_worker(self) -> None:
+        if not self.async_persist or self._persist_queue is None:
+            return
+
+        try:
+            self._persist_queue.put_nowait(("__STOP__", {}))
+        except queue.Full:
+            self._persist_queue.put(("__STOP__", {}))
+
+        if self._worker_thread is not None:
+            self._worker_thread.join(timeout=5)
+
+        self._worker_thread = None
+        self._persist_queue = None
+
 
 class _DummyContext:
     def __enter__(self):

tooluniverse/cbioportal_tool.py

@@ -0,0 +1,42 @@
+import requests
+from typing import Any, Dict
+from .base_tool import BaseTool
+from .tool_registry import register_tool
+
+
+@register_tool("CBioPortalRESTTool")
+class CBioPortalRESTTool(BaseTool):
+    def __init__(self, tool_config: Dict):
+        super().__init__(tool_config)
+        self.base_url = "https://www.cbioportal.org/api"
+        self.session = requests.Session()
+        self.session.headers.update(
+            {"Accept": "application/json", "User-Agent": "ToolUniverse/1.0"}
+        )
+        self.timeout = 30
+
+    def _build_url(self, args: Dict[str, Any]) -> str:
+        url = self.tool_config["fields"]["endpoint"]
+        for k, v in args.items():
+            url = url.replace(f"{{{k}}}", str(v))
+        return url
+
+    def run(self, arguments: Dict[str, Any]) -> Dict[str, Any]:
+        try:
+            url = self._build_url(arguments)
+            response = self.session.get(url, timeout=self.timeout)
+            response.raise_for_status()
+            data = response.json()
+
+            return {
+                "status": "success",
+                "data": data,
+                "url": url,
+                "count": len(data) if isinstance(data, list) else 1,
+            }
+        except Exception as e:
+            return {
+                "status": "error",
+                "error": f"cBioPortal API error: {str(e)}",
+                "url": url,
+            }

tooluniverse/clinvar_tool.py

@@ -1,90 +1,284 @@
+"""
+ClinVar REST API Tool
+
+This tool provides access to the ClinVar database for clinical variant information,
+disease associations, and clinical significance data.
+"""
+
 import requests
+import time
+from typing import Dict, Any, Optional
 from .base_tool import BaseTool
 from .tool_registry import register_tool
 
 
-@register_tool("ClinVarTool")
-class ClinVarTool(BaseTool):
-    """
-    Local tool wrapper for ClinVar via NCBI E-utilities.
-    Uses esearch + esummary to fetch variant records.
-    """
+class ClinVarRESTTool(BaseTool):
+    """Base class for ClinVar REST API tools."""
 
     def __init__(self, tool_config):
         super().__init__(tool_config)
-        self.base = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"
+        self.base_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"
         self.session = requests.Session()
+        self.session.headers.update(
+            {"Accept": "application/json", "User-Agent": "ToolUniverse/1.0"}
+        )
+        self.timeout = 30
 
-    def run(self, arguments):
-        query = arguments.get("query")
-        retmax = arguments.get("retmax", 5)
-        if not query:
-            return {"error": "Missing required parameter: query"}
+    def _make_request(
+        self, endpoint: str, params: Optional[Dict] = None, max_retries: int = 3
+    ) -> Dict[str, Any]:
+        """Make a request to the ClinVar API with automatic retry for rate limiting."""
+        url = f"{self.base_url}{endpoint}"
 
-        # 1) esearch to get UIDs
-        search_url = f"{self.base}/esearch.fcgi"
-        search_params = {
-            "db": "clinvar",
-            "term": query,
-            "retmode": "json",
-            "retmax": retmax,
-        }
-        search_resp = self.session.get(search_url, params=search_params, timeout=20)
-        search_resp.raise_for_status()
-        search_data = search_resp.json()
-        uids = search_data.get("esearchresult", {}).get("idlist", [])
-        if not uids:
-            return []
-
-        # 2) esummary to get details
-        summary_url = f"{self.base}/esummary.fcgi"
-        summary_params = {
+        for attempt in range(max_retries + 1):
+            try:
+                response = self.session.get(url, params=params, timeout=self.timeout)
+
+                # Handle rate limiting (429 error)
+                if response.status_code == 429:
+                    retry_after = response.headers.get("Retry-After")
+                    if retry_after:
+                        wait_time = int(retry_after)
+                    else:
+                        # Default exponential backoff: 1, 2, 4 seconds
+                        wait_time = 2**attempt
+
+                    if attempt < max_retries:
+                        print(
+                            f"Rate limited (429). Waiting {wait_time} seconds before retry {attempt + 1}/{max_retries}..."
+                        )
+                        time.sleep(wait_time)
+                        continue
+                    else:
+                        return {
+                            "status": "error",
+                            "error": f"Rate limited after {max_retries} retries. Please wait before making more requests.",
+                            "url": url,
+                            "retry_after": retry_after,
+                        }
+
+                response.raise_for_status()
+
+                # ClinVar API returns XML by default, but we can request JSON
+                if params and params.get("retmode") == "json":
+                    data = response.json()
+                else:
+                    # Parse XML response
+                    data = response.text
+
+                return {
+                    "status": "success",
+                    "data": data,
+                    "url": url,
+                    "content_type": response.headers.get(
+                        "content-type", "application/xml"
+                    ),
+                    "rate_limit_info": {
+                        "limit": response.headers.get("X-RateLimit-Limit"),
+                        "remaining": response.headers.get("X-RateLimit-Remaining"),
+                    },
+                }
+
+            except requests.exceptions.RequestException as e:
+                if attempt < max_retries:
+                    wait_time = 2**attempt
+                    print(
+                        f"Request failed: {str(e)}. Retrying in {wait_time} seconds..."
+                    )
+                    time.sleep(wait_time)
+                    continue
+                else:
+                    return {
+                        "status": "error",
+                        "error": f"ClinVar API request failed after {max_retries} retries: {str(e)}",
+                        "url": url,
+                    }
+
+        return {"status": "error", "error": "Maximum retries exceeded", "url": url}
+
+    def run(self, arguments: Dict[str, Any]) -> Dict[str, Any]:
+        """Execute the tool with given arguments."""
+        return self._make_request(self.endpoint, arguments)
+
+
+@register_tool("ClinVarSearchVariants")
+class ClinVarSearchVariants(ClinVarRESTTool):
+    """Search for variants in ClinVar by gene or condition."""
+
+    def __init__(self, tool_config):
+        super().__init__(tool_config)
+        self.endpoint = "/esearch.fcgi"
+
+    def run(self, arguments: Dict[str, Any]) -> Dict[str, Any]:
+        """Search variants by gene or condition."""
+        params = {
             "db": "clinvar",
-            "id": ",".join(uids),
             "retmode": "json",
+            "retmax": arguments.get("max_results", 20),
         }
-        summary_resp = self.session.get(summary_url, params=summary_params, timeout=30)
-        summary_resp.raise_for_status()
-        summary_data = summary_resp.json()
-
-        results = []
-        for uid in uids:
-            record = summary_data.get("result", {}).get(uid, {})
-            if not record:
-                continue
-
-            # Extract key fields
-            variation_set = record.get("variation_set", [])
-            gene = ""
-            chr_name = ""
-            start = None
-            stop = None
-            spdi = ""
-            if variation_set:
-                var = variation_set[0]
-                gene = record.get("genes", [{}])[0].get("symbol", "")
-                var_loc = var.get("variation_loc", [{}])[0]
-                chr_name = var_loc.get("chr", "")
-                start = var_loc.get("start")
-                stop = var_loc.get("stop")
-                spdi = var.get("canonical_spdi", "")
-
-            clinical_sig = record.get("germline_classification", {}).get(
-                "description", ""
-            )
-
-            results.append(
-                {
-                    "uid": uid,
-                    "accession": record.get("accession", ""),
-                    "title": record.get("title", ""),
-                    "gene": gene,
-                    "chr": chr_name,
-                    "start": start,
-                    "stop": stop,
-                    "spdi": spdi,
-                    "clinical_significance": clinical_sig,
+
+        # Build search query
+        query_parts = []
+
+        if "gene" in arguments:
+            query_parts.append(f"{arguments['gene']}[gene]")
+
+        if "condition" in arguments:
+            query_parts.append(f"{arguments['condition']}[condition]")
+
+        if "variant_id" in arguments:
+            query_parts.append(f"{arguments['variant_id']}[variant_id]")
+
+        if not query_parts:
+            return {
+                "status": "error",
+                "error": "At least one search parameter is required",
+            }
+
+        params["term"] = " AND ".join(query_parts)
+
+        result = self._make_request(self.endpoint, params)
+
+        # Add search parameters to result and format data
+        if result.get("status") == "success":
+            result["search_params"] = {
+                "gene": arguments.get("gene"),
+                "condition": arguments.get("condition"),
+                "variant_id": arguments.get("variant_id"),
+            }
+
+            # Format search results for better usability
+            data = result.get("data", {})
+            if "esearchresult" in data:
+                esearch = data["esearchresult"]
+                formatted_results = {
+                    "total_count": int(esearch.get("count", 0)),
+                    "variant_ids": esearch.get("idlist", []),
+                    "query_translation": esearch.get("querytranslation", ""),
+                    "search_params": result["search_params"],
+                    "summary": f"Found {esearch.get('count', 0)} variants matching the search criteria",
                 }
-            )
+                result["formatted_results"] = formatted_results
+
+        return result
+
+
+@register_tool("ClinVarGetVariantDetails")
+class ClinVarGetVariantDetails(ClinVarRESTTool):
+    """Get detailed variant information by ClinVar ID."""
+
+    def __init__(self, tool_config):
+        super().__init__(tool_config)
+        self.endpoint = "/esummary.fcgi"
+
+    def run(self, arguments: Dict[str, Any]) -> Dict[str, Any]:
+        """Get variant details by ClinVar ID."""
+        variant_id = arguments.get("variant_id", "")
+        if not variant_id:
+            return {"status": "error", "error": "variant_id is required"}
+
+        params = {"db": "clinvar", "id": variant_id, "retmode": "json"}
+
+        result = self._make_request(self.endpoint, params)
+
+        # Add variant_id to result and format data
+        if result.get("status") == "success":
+            result["variant_id"] = variant_id
+
+            # Format the data for better usability
+            data = result.get("data", {})
+            if "result" in data and variant_id in data["result"]:
+                variant_data = data["result"][variant_id]
+
+                # Extract key information
+                formatted_data = {
+                    "variant_id": variant_id,
+                    "accession": variant_data.get("accession", ""),
+                    "title": variant_data.get("title", ""),
+                    "obj_type": variant_data.get("obj_type", ""),
+                    "genes": [
+                        gene.get("symbol", "") for gene in variant_data.get("genes", [])
+                    ],
+                    "clinical_significance": variant_data.get(
+                        "germline_classification", {}
+                    ).get("description", ""),
+                    "review_status": variant_data.get(
+                        "germline_classification", {}
+                    ).get("review_status", ""),
+                    "chromosome": variant_data.get("chr_sort", ""),
+                    "location": variant_data.get("variation_set", [{}])[0]
+                    .get("variation_loc", [{}])[0]
+                    .get("band", ""),
+                    "variation_name": variant_data.get("variation_set", [{}])[0].get(
+                        "variation_name", ""
+                    ),
+                    "raw_data": variant_data,  # Keep original data for advanced users
+                }
+
+                result["formatted_data"] = formatted_data
+
+        return result
+
+
+@register_tool("ClinVarGetClinicalSignificance")
+class ClinVarGetClinicalSignificance(ClinVarRESTTool):
+    """Get clinical significance information for variants."""
+
+    def __init__(self, tool_config):
+        super().__init__(tool_config)
+        self.endpoint = "/esummary.fcgi"
+
+    def run(self, arguments: Dict[str, Any]) -> Dict[str, Any]:
+        """Get clinical significance by variant ID."""
+        variant_id = arguments.get("variant_id", "")
+        if not variant_id:
+            return {"status": "error", "error": "variant_id is required"}
+
+        params = {"db": "clinvar", "id": variant_id, "retmode": "json"}
+
+        result = self._make_request(self.endpoint, params)
+
+        # Add variant_id to result and format clinical significance data
+        if result.get("status") == "success":
+            result["variant_id"] = variant_id
+
+            # Format the clinical significance data
+            data = result.get("data", {})
+            if "result" in data and variant_id in data["result"]:
+                variant_data = data["result"][variant_id]
+
+                # Extract clinical significance information
+                germline_class = variant_data.get("germline_classification", {})
+                clinical_impact = variant_data.get("clinical_impact_classification", {})
+                oncogenicity = variant_data.get("oncogenicity_classification", {})
+
+                formatted_data = {
+                    "variant_id": variant_id,
+                    "germline_classification": {
+                        "description": germline_class.get("description", ""),
+                        "review_status": germline_class.get("review_status", ""),
+                        "last_evaluated": germline_class.get("last_evaluated", ""),
+                        "fda_recognized": germline_class.get(
+                            "fda_recognized_database", ""
+                        ),
+                        "traits": [
+                            trait.get("trait_name", "")
+                            for trait in germline_class.get("trait_set", [])
+                        ],
+                    },
+                    "clinical_impact": {
+                        "description": clinical_impact.get("description", ""),
+                        "review_status": clinical_impact.get("review_status", ""),
+                        "last_evaluated": clinical_impact.get("last_evaluated", ""),
+                    },
+                    "oncogenicity": {
+                        "description": oncogenicity.get("description", ""),
+                        "review_status": oncogenicity.get("review_status", ""),
+                        "last_evaluated": oncogenicity.get("last_evaluated", ""),
+                    },
+                    "raw_data": variant_data,  # Keep original data for advanced users
+                }
+
+                result["formatted_data"] = formatted_data
 
-        return results
+        return result

tooluniverse/compose_scripts/drug_safety_analyzer.py

@@ -13,7 +13,7 @@ def compose(arguments, tooluniverse, call_tool):
         tooluniverse: ToolUniverse instance
         call_tool: Function to call other tools
 
-    Returns:
+    Returns
         dict: Comprehensive drug safety analysis result
     """
     drug_name = arguments["drug_name"]

tooluniverse/compose_scripts/multi_agent_literature_search.py

@@ -46,7 +46,7 @@ def compose(arguments, tooluniverse, call_tool, stream_callback=None):
         call_tool (function): Function to call other tools
         stream_callback (callable, optional): Callback function for streaming output
 
-    Returns:
+    Returns
         dict: The result of the multi-agent search
     """
     query = arguments.get("query", "")