tooluniverse 1.0.7__py3-none-any.whl → 1.0.8__py3-none-any.whl

tooluniverse/cache/result_cache_manager.py

@@ -0,0 +1,235 @@
+"""
+Result cache manager that coordinates in-memory and persistent storage.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import time
+from dataclasses import dataclass
+from typing import Any, Dict, Iterator, Optional
+
+from .memory_cache import LRUCache, SingleFlight
+from .sqlite_backend import CacheEntry, PersistentCache
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class CacheRecord:
+    value: Any
+    expires_at: Optional[float]
+    namespace: str
+    version: str
+
+
+class ResultCacheManager:
+    """Facade around memory + persistent cache layers."""
+
+    def __init__(
+        self,
+        *,
+        memory_size: int = 256,
+        persistent_path: Optional[str] = None,
+        enabled: bool = True,
+        persistence_enabled: bool = True,
+        singleflight: bool = True,
+        default_ttl: Optional[int] = None,
+    ):
+        self.enabled = enabled
+        self.default_ttl = default_ttl
+
+        self.memory = LRUCache(max_size=memory_size)
+        persistence_path = persistent_path
+        if persistence_path is None:
+            cache_dir = os.environ.get("TOOLUNIVERSE_CACHE_DIR")
+            if cache_dir:
+                persistence_path = os.path.join(cache_dir, "tooluniverse_cache.sqlite")
+        self.persistent = None
+        if persistence_enabled and persistence_path:
+            try:
+                self.persistent = PersistentCache(persistence_path, enable=True)
+            except Exception as exc:
+                logger.warning("Failed to initialize persistent cache: %s", exc)
+                self.persistent = None
+
+        self.singleflight = SingleFlight() if singleflight else None
+
+    # ------------------------------------------------------------------
+    # Helper methods
+    # ------------------------------------------------------------------
+    @staticmethod
+    def compose_key(namespace: str, version: str, cache_key: str) -> str:
+        return f"{namespace}::{version}::{cache_key}"
+
+    def _now(self) -> float:
+        return time.time()
+
+    def _ttl_or_default(self, ttl: Optional[int]) -> Optional[int]:
+        return ttl if ttl is not None else self.default_ttl
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+    def get(self, *, namespace: str, version: str, cache_key: str) -> Optional[Any]:
+        if not self.enabled:
+            return None
+
+        composed = self.compose_key(namespace, version, cache_key)
+        record = self.memory.get(composed)
+        if record:
+            if record.expires_at and record.expires_at <= self._now():
+                self.memory.delete(composed)
+            else:
+                return record.value
+
+        entry = self._get_from_persistent(composed)
+        if entry:
+            expires_at = entry.created_at + entry.ttl if entry.ttl else None
+            self.memory.set(
+                composed,
+                CacheRecord(
+                    value=entry.value,
+                    expires_at=expires_at,
+                    namespace=namespace,
+                    version=version,
+                ),
+            )
+            return entry.value
+        return None
+
+    def set(
+        self,
+        *,
+        namespace: str,
+        version: str,
+        cache_key: str,
+        value: Any,
+        ttl: Optional[int] = None,
+    ):
+        if not self.enabled:
+            return
+
+        effective_ttl = self._ttl_or_default(ttl)
+        expires_at = self._now() + effective_ttl if effective_ttl else None
+        composed = self.compose_key(namespace, version, cache_key)
+
+        self.memory.set(
+            composed,
+            CacheRecord(
+                value=value,
+                expires_at=expires_at,
+                namespace=namespace,
+                version=version,
+            ),
+        )
+
+        if self.persistent:
+            try:
+                self.persistent.set(
+                    composed,
+                    value,
+                    namespace=namespace,
+                    version=version,
+                    ttl=effective_ttl,
+                )
+            except Exception as exc:
+                logger.warning("Persistent cache write failed: %s", exc)
+                self.persistent = None
+
+    def delete(self, *, namespace: str, version: str, cache_key: str):
+        composed = self.compose_key(namespace, version, cache_key)
+        self.memory.delete(composed)
+        if self.persistent:
+            try:
+                self.persistent.delete(composed)
+            except Exception as exc:
+                logger.warning("Persistent cache delete failed: %s", exc)
+
+    def clear(self, namespace: Optional[str] = None):
+        if namespace:
+            # Clear matching namespace in memory
+            keys_to_remove = [
+                key
+                for key, record in self.memory.items()
+                if hasattr(record, "namespace") and record.namespace == namespace
+            ]
+            for key in keys_to_remove:
+                self.memory.delete(key)
+        else:
+            self.memory.clear()
+
+        if self.persistent:
+            try:
+                self.persistent.clear(namespace=namespace)
+            except Exception as exc:
+                logger.warning("Persistent cache clear failed: %s", exc)
+
+    def stats(self) -> Dict[str, Any]:
+        return {
+            "enabled": self.enabled,
+            "memory": self.memory.stats(),
+            "persistent": (
+                self.persistent.stats() if self.persistent else {"enabled": False}
+            ),
+        }
+
+    def dump(self, namespace: Optional[str] = None) -> Iterator[Dict[str, Any]]:
+        if not self.persistent:
+            return iter([])
+        return (
+            {
+                "cache_key": entry.key,
+                "namespace": entry.namespace,
+                "version": entry.version,
+                "ttl": entry.ttl,
+                "created_at": entry.created_at,
+                "last_accessed": entry.last_accessed,
+                "hit_count": entry.hit_count,
+                "value": entry.value,
+            }
+            for entry in self._iter_persistent(namespace=namespace)
+        )
+
+    def _get_from_persistent(self, composed_key: str) -> Optional[CacheEntry]:
+        if not self.persistent:
+            return None
+        try:
+            return self.persistent.get(composed_key)
+        except Exception as exc:
+            logger.warning("Persistent cache read failed: %s", exc)
+            self.persistent = None
+            return None
+
+    def _iter_persistent(self, namespace: Optional[str]):
+        if not self.persistent:
+            return iter([])
+        try:
+            return self.persistent.iter_entries(namespace=namespace)
+        except Exception as exc:
+            logger.warning("Persistent cache iterator failed: %s", exc)
+            return iter([])
+
+    # ------------------------------------------------------------------
+    # Context manager for singleflight
+    # ------------------------------------------------------------------
+    def singleflight_guard(self, composed_key: str):
+        if self.singleflight:
+            return self.singleflight.acquire(composed_key)
+        return _DummyContext()
+
+    def close(self):
+        if self.persistent:
+            try:
+                self.persistent.close()
+            except Exception as exc:
+                logger.warning("Persistent cache close failed: %s", exc)
+
+
+class _DummyContext:
+    def __enter__(self):
+        return None
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return False

tooluniverse/cache/sqlite_backend.py

@@ -0,0 +1,257 @@
+"""
+SQLite-backed persistent cache for ToolUniverse.
+
+The cache stores serialized tool results with TTL and version metadata.
+Designed to be a drop-in persistent layer behind the in-memory cache.
+"""
+
+from __future__ import annotations
+
+import os
+import pickle
+import sqlite3
+import threading
+import time
+from dataclasses import dataclass
+from typing import Any, Dict, Iterator, Optional
+
+
+@dataclass
+class CacheEntry:
+    key: str
+    value: Any
+    namespace: str
+    version: str
+    ttl: Optional[int]
+    created_at: float
+    last_accessed: float
+    hit_count: int
+
+
+class PersistentCache:
+    """SQLite-backed cache layer with TTL support."""
+
+    def __init__(self, path: str, *, enable: bool = True):
+        self.enabled = enable
+        self.path = path
+        self._lock = threading.RLock()
+        self._conn: Optional[sqlite3.Connection] = None
+
+        if self.enabled:
+            self._init_storage()
+
+    def _init_storage(self):
+        directory = os.path.dirname(self.path)
+        if directory:
+            os.makedirs(directory, exist_ok=True)
+        self._conn = sqlite3.connect(
+            self.path,
+            timeout=30,
+            check_same_thread=False,
+            isolation_level=None,  # autocommit
+        )
+        self._conn.execute("PRAGMA journal_mode=WAL;")
+        self._conn.execute("PRAGMA synchronous=NORMAL;")
+        self._conn.execute("PRAGMA foreign_keys=ON;")
+        self._ensure_schema()
+        self.cleanup_expired()
+
+    def _ensure_schema(self):
+        assert self._conn is not None
+        self._conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS cache_entries (
+                cache_key TEXT PRIMARY KEY,
+                namespace TEXT NOT NULL,
+                version TEXT,
+                value BLOB NOT NULL,
+                ttl INTEGER,
+                created_at REAL NOT NULL,
+                last_accessed REAL NOT NULL,
+                expires_at REAL,
+                hit_count INTEGER NOT NULL DEFAULT 0
+            )
+            """
+        )
+        self._conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_cache_namespace ON cache_entries(namespace)"
+        )
+        self._conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_cache_expires ON cache_entries(expires_at)"
+        )
+
+    def _serialize(self, value: Any) -> bytes:
+        return pickle.dumps(value, protocol=pickle.HIGHEST_PROTOCOL)
+
+    def _deserialize(self, payload: bytes) -> Any:
+        return pickle.loads(payload)
+
+    def close(self):
+        if self._conn:
+            self._conn.close()
+            self._conn = None
+
+    def cleanup_expired(self):
+        if not self.enabled or not self._conn:
+            return
+        with self._lock:
+            now = time.time()
+            self._conn.execute(
+                "DELETE FROM cache_entries WHERE expires_at IS NOT NULL AND expires_at <= ?",
+                (now,),
+            )
+
+    def get(self, cache_key: str) -> Optional[CacheEntry]:
+        if not self.enabled or not self._conn:
+            return None
+        with self._lock:
+            cur = self._conn.execute(
+                """
+                SELECT cache_key, namespace, version, value, ttl, created_at,
+                       last_accessed, expires_at, hit_count
+                FROM cache_entries WHERE cache_key = ?
+                """,
+                (cache_key,),
+            )
+            row = cur.fetchone()
+            if not row:
+                return None
+
+            expires_at = row[7]
+            if expires_at is not None and expires_at <= time.time():
+                self._conn.execute(
+                    "DELETE FROM cache_entries WHERE cache_key = ?", (cache_key,)
+                )
+                return None
+
+            entry = CacheEntry(
+                key=row[0],
+                namespace=row[1],
+                version=row[2] or "",
+                value=self._deserialize(row[3]),
+                ttl=row[4],
+                created_at=row[5],
+                last_accessed=row[6],
+                hit_count=row[8],
+            )
+
+            self._conn.execute(
+                """
+                UPDATE cache_entries
+                SET last_accessed = ?, hit_count = hit_count + 1
+                WHERE cache_key = ?
+                """,
+                (time.time(), cache_key),
+            )
+            return entry
+
+    def set(
+        self,
+        cache_key: str,
+        value: Any,
+        *,
+        namespace: str,
+        version: str,
+        ttl: Optional[int],
+    ):
+        if not self.enabled or not self._conn:
+            return
+        with self._lock:
+            now = time.time()
+            expires_at = now + ttl if ttl else None
+            payload = self._serialize(value)
+            self._conn.execute(
+                """
+                INSERT INTO cache_entries(cache_key, namespace, version, value, ttl,
+                                          created_at, last_accessed, expires_at, hit_count)
+                VALUES(?, ?, ?, ?, ?, ?, ?, ?, 0)
+                ON CONFLICT(cache_key) DO UPDATE SET
+                    namespace=excluded.namespace,
+                    version=excluded.version,
+                    value=excluded.value,
+                    ttl=excluded.ttl,
+                    created_at=excluded.created_at,
+                    last_accessed=excluded.last_accessed,
+                    expires_at=excluded.expires_at,
+                    hit_count=excluded.hit_count
+                """,
+                (
+                    cache_key,
+                    namespace,
+                    version,
+                    payload,
+                    ttl,
+                    now,
+                    now,
+                    expires_at,
+                ),
+            )
+
+    def delete(self, cache_key: str):
+        if not self.enabled or not self._conn:
+            return
+        with self._lock:
+            self._conn.execute(
+                "DELETE FROM cache_entries WHERE cache_key = ?", (cache_key,)
+            )
+
+    def clear(self, namespace: Optional[str] = None):
+        if not self.enabled or not self._conn:
+            return
+        with self._lock:
+            if namespace:
+                self._conn.execute(
+                    "DELETE FROM cache_entries WHERE namespace = ?", (namespace,)
+                )
+            else:
+                self._conn.execute("DELETE FROM cache_entries")
+
+    def iter_entries(self, namespace: Optional[str] = None) -> Iterator[CacheEntry]:
+        if not self.enabled or not self._conn:
+            return iter([])
+        with self._lock:
+            if namespace:
+                cur = self._conn.execute(
+                    """
+                    SELECT cache_key, namespace, version, value, ttl,
+                           created_at, last_accessed, hit_count
+                    FROM cache_entries WHERE namespace = ?
+                    """,
+                    (namespace,),
+                )
+            else:
+                cur = self._conn.execute(
+                    """
+                    SELECT cache_key, namespace, version, value, ttl,
+                           created_at, last_accessed, hit_count
+                    FROM cache_entries
+                    """
+                )
+            rows = cur.fetchall()
+
+        for row in rows:
+            yield CacheEntry(
+                key=row[0],
+                namespace=row[1],
+                version=row[2] or "",
+                value=self._deserialize(row[3]),
+                ttl=row[4],
+                created_at=row[5],
+                last_accessed=row[6],
+                hit_count=row[7],
+            )
+
+    def stats(self) -> Dict[str, Any]:
+        if not self.enabled or not self._conn:
+            return {"enabled": False}
+        with self._lock:
+            cur = self._conn.execute(
+                "SELECT COUNT(*), SUM(LENGTH(value)) FROM cache_entries"
+            )
+            count, total_bytes = cur.fetchone()
+            return {
+                "enabled": True,
+                "entries": count or 0,
+                "approx_bytes": total_bytes or 0,
+                "path": self.path,
+            }

tooluniverse/clinvar_tool.py

@@ -0,0 +1,90 @@
+import requests
+from .base_tool import BaseTool
+from .tool_registry import register_tool
+
+
+@register_tool("ClinVarTool")
+class ClinVarTool(BaseTool):
+    """
+    Local tool wrapper for ClinVar via NCBI E-utilities.
+    Uses esearch + esummary to fetch variant records.
+    """
+
+    def __init__(self, tool_config):
+        super().__init__(tool_config)
+        self.base = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"
+        self.session = requests.Session()
+
+    def run(self, arguments):
+        query = arguments.get("query")
+        retmax = arguments.get("retmax", 5)
+        if not query:
+            return {"error": "Missing required parameter: query"}
+
+        # 1) esearch to get UIDs
+        search_url = f"{self.base}/esearch.fcgi"
+        search_params = {
+            "db": "clinvar",
+            "term": query,
+            "retmode": "json",
+            "retmax": retmax,
+        }
+        search_resp = self.session.get(search_url, params=search_params, timeout=20)
+        search_resp.raise_for_status()
+        search_data = search_resp.json()
+        uids = search_data.get("esearchresult", {}).get("idlist", [])
+        if not uids:
+            return []
+
+        # 2) esummary to get details
+        summary_url = f"{self.base}/esummary.fcgi"
+        summary_params = {
+            "db": "clinvar",
+            "id": ",".join(uids),
+            "retmode": "json",
+        }
+        summary_resp = self.session.get(summary_url, params=summary_params, timeout=30)
+        summary_resp.raise_for_status()
+        summary_data = summary_resp.json()
+
+        results = []
+        for uid in uids:
+            record = summary_data.get("result", {}).get(uid, {})
+            if not record:
+                continue
+
+            # Extract key fields
+            variation_set = record.get("variation_set", [])
+            gene = ""
+            chr_name = ""
+            start = None
+            stop = None
+            spdi = ""
+            if variation_set:
+                var = variation_set[0]
+                gene = record.get("genes", [{}])[0].get("symbol", "")
+                var_loc = var.get("variation_loc", [{}])[0]
+                chr_name = var_loc.get("chr", "")
+                start = var_loc.get("start")
+                stop = var_loc.get("stop")
+                spdi = var.get("canonical_spdi", "")
+
+            clinical_sig = record.get("germline_classification", {}).get(
+                "description", ""
+            )
+
+            results.append(
+                {
+                    "uid": uid,
+                    "accession": record.get("accession", ""),
+                    "title": record.get("title", ""),
+                    "gene": gene,
+                    "chr": chr_name,
+                    "start": start,
+                    "stop": stop,
+                    "spdi": spdi,
+                    "clinical_significance": clinical_sig,
+                }
+            )
+
+        return results

tooluniverse/custom_tool.py

@@ -26,6 +26,11 @@ class CustomTool(BaseTool):
         self.execute_function = None
         if self.code_file and os.path.exists(self.code_file):
             self._load_external_code()
+        elif (
+            "implementation" in tool_config
+            and "source_code" in tool_config["implementation"]
+        ):
+            self._load_embedded_code(tool_config["implementation"])
 
     def _load_external_code(self):
         """Load the execute_tool function from external Python file"""
@@ -46,6 +51,29 @@ class CustomTool(BaseTool):
         except Exception as e:
             print(f"Error loading external code from {self.code_file}: {e}")
 
+    def _load_embedded_code(self, implementation: Dict):
+        """Load the execute_tool function from embedded source code"""
+        try:
+            source_code = implementation.get("source_code", "")
+            main_function = implementation.get("main_function", "execute_tool")
+
+            # Create a temporary module to execute the code
+            import types
+
+            module = types.ModuleType("embedded_tool_module")
+
+            # Execute the source code in the module namespace
+            exec(source_code, module.__dict__)
+
+            # Get the main function
+            if hasattr(module, main_function):
+                self.execute_function = getattr(module, main_function)
+            else:
+                print(f"Warning: No {main_function} function found in embedded code")
+
+        except Exception as e:
+            print(f"Error loading embedded code: {e}")
+
     def run(self, arguments: Any = None) -> Dict[str, Any]:
         """
         Execute the custom tool

tooluniverse/data/arxiv_tools.json

@@ -27,10 +27,7 @@
         }
       },
       "required": [
-        "query",
-        "limit",
-        "sort_by",
-        "sort_order"
+        "query"
       ]
     },
     "return_schema": {

tooluniverse/data/core_tools.json

@@ -32,10 +32,7 @@
       },
       "required": [
         "query",
-        "limit",
-        "year_from",
-        "year_to",
-        "language"
+        "limit"
       ]
     },
     "return_schema": {