spice-mcp 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

spice_mcp/mcp/server.py

@@ -22,7 +22,6 @@ try:
 except Exception:
     pass
 
-from ..adapters.dune import extract as dune_extract
 from ..adapters.dune import urls as dune_urls
 from ..adapters.dune.admin import DuneAdminAdapter
 from ..adapters.dune.client import DuneAdapter
@@ -34,6 +33,7 @@ from ..logging.query_history import QueryHistory
 from ..service_layer.discovery_service import DiscoveryService
 from ..service_layer.query_admin_service import QueryAdminService
 from ..service_layer.query_service import QueryService
+from ..service_layer.verification_service import VerificationService
 from .tools.execute_query import ExecuteQueryTool
 
 logger = logging.getLogger(__name__)
@@ -48,6 +48,7 @@ QUERY_ADMIN_SERVICE: QueryAdminService | None = None
 DISCOVERY_SERVICE: DiscoveryService | None = None
 SPELLBOOK_EXPLORER: SpellbookExplorer | None = None
 HTTP_CLIENT: HttpClient | None = None
+VERIFICATION_SERVICE: VerificationService | None = None
 
 EXECUTE_QUERY_TOOL: ExecuteQueryTool | None = None
 
@@ -58,7 +59,7 @@ app = FastMCP("spice-mcp")
 def _ensure_initialized() -> None:
     """Initialize configuration and tool instances if not already initialized."""
     global CONFIG, QUERY_HISTORY, DUNE_ADAPTER, QUERY_SERVICE, DISCOVERY_SERVICE, QUERY_ADMIN_SERVICE
-    global EXECUTE_QUERY_TOOL, HTTP_CLIENT, SPELLBOOK_EXPLORER
+    global EXECUTE_QUERY_TOOL, HTTP_CLIENT, SPELLBOOK_EXPLORER, VERIFICATION_SERVICE
 
     if CONFIG is not None and EXECUTE_QUERY_TOOL is not None:
         return
@@ -96,6 +97,15 @@ def _ensure_initialized() -> None:
     
     # Initialize Spellbook explorer (lazy, clones repo on first use)
     SPELLBOOK_EXPLORER = SpellbookExplorer()
+    
+    # Initialize verification service with persistent cache
+    from pathlib import Path
+    cache_dir = Path.home() / ".spice_mcp"
+    cache_dir.mkdir(exist_ok=True)
+    VERIFICATION_SERVICE = VerificationService(
+        cache_path=cache_dir / "table_verification_cache.json",
+        dune_adapter=DUNE_ADAPTER,
+    )
 
     EXECUTE_QUERY_TOOL = ExecuteQueryTool(CONFIG, QUERY_SERVICE, QUERY_HISTORY)
 
@@ -144,9 +154,10 @@ def compute_health_status() -> dict[str, Any]:
         if tmpl:
             tid = dune_urls.get_query_id(tmpl)
             url = dune_urls.url_templates["query"].format(query_id=tid)
+            from ..adapters.dune.user_agent import get_user_agent as get_dune_user_agent
             headers = {
                 "X-Dune-API-Key": os.getenv("DUNE_API_KEY", ""),
-                "User-Agent": dune_extract.get_user_agent(),
+                "User-Agent": get_dune_user_agent(),
             }
             client = HTTP_CLIENT or HttpClient(Config.from_env().http)
             resp = client.request("GET", url, headers=headers, timeout=5.0)
@@ -169,9 +180,10 @@ def dune_query_info(query: str) -> dict[str, Any]:
     try:
         qid = dune_urls.get_query_id(query)
         url = dune_urls.url_templates["query"].format(query_id=qid)
+        from ..adapters.dune.user_agent import get_user_agent as get_dune_user_agent
         headers = {
             "X-Dune-API-Key": dune_urls.get_api_key(),
-            "User-Agent": dune_extract.get_user_agent(),
+            "User-Agent": get_dune_user_agent(),
         }
         client = HTTP_CLIENT or HttpClient(Config.from_env().http)
         resp = client.request("GET", url, headers=headers, timeout=10.0)
@@ -197,13 +209,7 @@ def dune_query_info(query: str) -> dict[str, Any]:
         })
 
 
-@app.tool(
-    name="dune_query",
-    title="Run Dune Query",
-    description="Execute Dune queries and return agent-optimized preview.",
-    tags={"dune", "query"},
-)
-def dune_query(
+def _dune_query_impl(
     query: str,
     parameters: dict[str, Any] | None = None,
     refresh: bool = False,
@@ -217,13 +223,41 @@ def dune_query(
     extras: dict[str, Any] | None = None,
     timeout_seconds: float | None = None,
 ) -> dict[str, Any]:
+    """Internal implementation of dune_query to avoid FastMCP overload detection."""
     _ensure_initialized()
     assert EXECUTE_QUERY_TOOL is not None
+    
+    # Normalize parameters: handle case where MCP client passes JSON string
+    # This can happen if FastMCP's schema generation doesn't match client expectations
+    normalized_parameters = parameters
+    if isinstance(parameters, str):
+        try:
+            import json
+            normalized_parameters = json.loads(parameters)
+        except (json.JSONDecodeError, TypeError):
+            return error_response(
+                ValueError(f"parameters must be a dict or JSON string, got {type(parameters).__name__}"),
+                context={
+                    "tool": "dune_query",
+                    "query": query,
+                    "parameters_type": type(parameters).__name__,
+                }
+            )
+    
+    # Normalize extras similarly
+    normalized_extras = extras
+    if isinstance(extras, str):
+        try:
+            import json
+            normalized_extras = json.loads(extras)
+        except (json.JSONDecodeError, TypeError):
+            normalized_extras = None
+    
     try:
         # Execute query synchronously
         return EXECUTE_QUERY_TOOL.execute(
             query=query,
-            parameters=parameters,
+            parameters=normalized_parameters,
             refresh=refresh,
             max_age=max_age,
             limit=limit,
@@ -232,7 +266,7 @@ def dune_query(
             sort_by=sort_by,
             columns=columns,
             format=format,
-            extras=extras,
+            extras=normalized_extras,
             timeout_seconds=timeout_seconds,
         )
     except Exception as e:
@@ -244,6 +278,60 @@ def dune_query(
         })
 
 
+@app.tool(
+    name="dune_query",
+    title="Run Dune Query",
+    description="Execute Dune queries and return agent-optimized preview.",
+    tags={"dune", "query"},
+)
+def dune_query(
+    query: str,
+    parameters: dict[str, Any] | None = None,
+    refresh: bool = False,
+    max_age: float | None = None,
+    limit: int | None = None,
+    offset: int | None = None,
+    sample_count: int | None = None,
+    sort_by: str | None = None,
+    columns: list[str] | None = None,
+    format: Literal["preview", "raw", "metadata", "poll"] = "preview",
+    extras: dict[str, Any] | None = None,
+    timeout_seconds: float | None = None,
+) -> dict[str, Any]:
+    """Execute Dune queries (by ID, URL, or raw SQL) and return agent-optimized preview.
+    
+    ⚠️ IMPORTANT: ALWAYS use dune_discover FIRST to find verified table names.
+    Do not guess table names or query information_schema directly.
+    
+    The query parameter accepts:
+    - Query IDs (e.g., "123456")
+    - Query URLs (e.g., "https://dune.com/queries/123456")
+    - Raw SQL using tables discovered via dune_discover
+    
+    For Spellbook models, use the 'dune_table' field returned by dune_discover.
+    Example: dune_discover(keyword="walrus") → returns dune_table="sui_walrus.base_table"
+             Then use: dune_query(query="SELECT * FROM sui_walrus.base_table LIMIT 10")
+    
+    This wrapper ensures FastMCP doesn't detect overloads in imported functions.
+    """
+    # Always ensure parameters is explicitly passed (even if None) to avoid FastMCP
+    # overload detection when the keyword is omitted
+    return _dune_query_impl(
+        query=query,
+        parameters=parameters,
+        refresh=refresh,
+        max_age=max_age,
+        limit=limit,
+        offset=offset,
+        sample_count=sample_count,
+        sort_by=sort_by,
+        columns=columns,
+        format=format,
+        extras=extras,
+        timeout_seconds=timeout_seconds,
+    )
+
+
 @app.tool(
     name="dune_health_check",
     title="Health Check",
@@ -254,22 +342,6 @@ def dune_health_check() -> dict[str, Any]:
     return compute_health_status()
 
 
-def _dune_find_tables_impl(
-    keyword: str | None = None,
-    schema: str | None = None,
-    limit: int = 50,
-) -> dict[str, Any]:
-    _ensure_initialized()
-    assert DISCOVERY_SERVICE is not None
-    out: dict[str, Any] = {}
-    if keyword:
-        out["schemas"] = DISCOVERY_SERVICE.find_schemas(keyword)
-    if schema:
-        tables = DISCOVERY_SERVICE.list_tables(schema, limit=limit)
-        out["tables"] = [summary.table for summary in tables]
-    return out
-
-
 def _unified_discover_impl(
     keyword: str | list[str] | None = None,
     schema: str | None = None,
@@ -315,6 +387,8 @@ def _unified_discover_impl(
                     "table": summary.table,
                     "fully_qualified_name": f"{schema}.{summary.table}",
                     "source": "dune",
+                    "dune_table": f"{schema}.{summary.table}",
+                    "verified": True,
                 }
                 for summary in tables
             ]
@@ -350,11 +424,57 @@ def _unified_discover_impl(
                     "table": model["table"],
                     "fully_qualified_name": model["fully_qualified_name"],
                     "source": "spellbook",
+                    # Include resolved Dune table names
+                    "dune_schema": model.get("dune_schema"),
+                    "dune_alias": model.get("dune_alias"),
+                    "dune_table": model.get("dune_table"),
                 }
                 if "columns" in model:
                     table_info["columns"] = model["columns"]
                 out["tables"].append(table_info)
     
+    # Verify Spellbook tables exist in Dune before returning
+    if source in ("spellbook", "both") and out["tables"]:
+        assert VERIFICATION_SERVICE is not None
+        # Extract Spellbook tables that need verification
+        spellbook_tables = [
+            (t["dune_schema"], t["dune_alias"])
+            for t in out["tables"]
+            if t.get("source") == "spellbook" and t.get("dune_schema") and t.get("dune_alias")
+        ]
+        
+        if spellbook_tables:
+            # Verify tables exist (uses cache, queries Dune only if needed)
+            verification_results = VERIFICATION_SERVICE.verify_tables_batch(spellbook_tables)
+
+            # Filter: drop only tables explicitly verified as False.
+            # Keep tables when verification is True or inconclusive (missing).
+            verified_tables = []
+            for t in out["tables"]:
+                if t.get("source") != "spellbook":
+                    # Keep Dune tables as-is
+                    verified_tables.append(t)
+                    continue
+                dune_fqn = t.get("dune_table")
+                if not dune_fqn:
+                    # If we couldn't resolve dune_table, keep it (conservative)
+                    verified_tables.append(t)
+                    continue
+                vr = verification_results.get(dune_fqn)
+                if vr is False:
+                    # Explicitly known to be non-existent -> drop
+                    continue
+                if vr is True:
+                    t["verified"] = True
+                # If vr is None/missing (inconclusive), keep without setting verified
+                verified_tables.append(t)
+            
+            out["tables"] = verified_tables
+            
+            # Add helpful message if no tables found
+            if not out["tables"] and len(spellbook_tables) > 0:
+                out["message"] = "No verified tables found. Try different keywords or check schema names."
+    
     # Deduplicate and sort schemas
     out["schemas"] = sorted(list(set(out["schemas"])))
     
@@ -379,11 +499,19 @@ def dune_discover(
     include_columns: bool = True,
 ) -> dict[str, Any]:
     """
-    Unified discovery tool for Dune tables and Spellbook models.
+    PRIMARY discovery tool for finding tables in Dune.
+    
+    ⚠️ IMPORTANT: ALWAYS use this tool instead of querying information_schema directly.
+    Querying information_schema is slow and causes lag. This tool uses optimized
+    native SHOW statements for fast discovery.
     
-    This tool can search both Dune's live schemas (via SQL queries) and Spellbook's
-    dbt models (via GitHub repo parsing) in a single call. You don't need to decide
-    which source to use - it can search both automatically.
+    This tool automatically:
+    - Parses dbt configs from Spellbook models to resolve actual Dune table names
+    - Verifies tables exist in Dune before returning them
+    - Returns ONLY verified, queryable tables
+    
+    All returned tables are VERIFIED to exist - you can query them immediately using
+    the 'dune_table' field.
     
     Args:
         keyword: Search term(s) - can be a string or list of strings
@@ -398,16 +526,25 @@ def dune_discover(
         Dictionary with:
         - 'schemas': List of matching schema names
         - 'tables': List of table/model objects, each with:
-          - schema: Schema name
-          - table: Table/model name
-          - fully_qualified_name: schema.table
+          - schema: Schema name (Spellbook subproject name)
+          - table: Table/model name (Spellbook model name)
+          - fully_qualified_name: schema.table (Spellbook format)
           - source: "dune" or "spellbook"
+          - dune_schema: Actual Dune schema name (for Spellbook models)
+          - dune_alias: Actual Dune table alias (for Spellbook models)
+          - dune_table: Verified, queryable Dune table name (e.g., "sui_walrus.base_table")
+          - verified: True (all returned tables are verified to exist)
           - columns: Column details (for Spellbook models, if include_columns=True)
         - 'source': The source parameter used
+        - 'message': Helpful message if no tables found
     
     Examples:
-        # Search both sources for layerzero
-        dune_discover(keyword="layerzero")
+        # Search both sources for walrus - returns verified tables only
+        dune_discover(keyword="walrus")
+        # → Returns tables with dune_table field like "sui_walrus.base_table"
+        
+        # Use the dune_table field to query immediately
+        dune_query(query="SELECT * FROM sui_walrus.base_table LIMIT 10")
         
         # Search only Spellbook
         dune_discover(keyword=["layerzero", "bridge"], source="spellbook")
@@ -435,26 +572,6 @@ def dune_discover(
         })
 
 
-@app.tool(
-    name="dune_find_tables",
-    title="Find Tables",
-    description="Search schemas and optionally list tables.",
-    tags={"dune", "schema"},
-)
-def dune_find_tables(keyword: str | None = None, schema: str | None = None, limit: int = 50) -> dict[str, Any]:
-    """
-    Search schemas and optionally list tables in Dune.
-    """
-    try:
-        return _dune_find_tables_impl(keyword=keyword, schema=schema, limit=limit)
-    except Exception as e:
-        return error_response(e, context={
-            "tool": "dune_find_tables",
-            "keyword": keyword,
-            "schema": schema,
-        })
-
-
 def _dune_describe_table_impl(schema: str, table: str) -> dict[str, Any]:
     _ensure_initialized()
     assert DISCOVERY_SERVICE is not None
@@ -529,10 +646,22 @@ def _spellbook_find_models_impl(
                     matches_keyword = any(kw.lower() in table_name for kw in keywords)
                     
                     if matches_keyword:
+                        # Get model details including resolved Dune table names
+                        models_dict = SPELLBOOK_EXPLORER._load_models()
+                        model_details = None
+                        for m in models_dict.get(schema_name, []):
+                            if m["name"] == table_summary.table:
+                                model_details = m
+                                break
+                        
                         model_info: dict[str, Any] = {
                             "schema": schema_name,
                             "table": table_summary.table,
                             "fully_qualified_name": f"{schema_name}.{table_summary.table}",
+                            # Include resolved Dune table names if available
+                            "dune_schema": model_details.get("dune_schema") if model_details else None,
+                            "dune_alias": model_details.get("dune_alias") if model_details else None,
+                            "dune_table": model_details.get("dune_table") if model_details else None,
                         }
                         
                         # Include column details if requested
@@ -564,10 +693,22 @@ def _spellbook_find_models_impl(
             out["models"] = []
         
         for table_summary in tables:
+            # Get model details including resolved Dune table names
+            models_dict = SPELLBOOK_EXPLORER._load_models()
+            model_details = None
+            for m in models_dict.get(schema, []):
+                if m["name"] == table_summary.table:
+                    model_details = m
+                    break
+            
             model_info: dict[str, Any] = {
                 "schema": schema,
                 "table": table_summary.table,
                 "fully_qualified_name": f"{schema}.{table_summary.table}",
+                # Include resolved Dune table names if available
+                "dune_schema": model_details.get("dune_schema") if model_details else None,
+                "dune_alias": model_details.get("dune_alias") if model_details else None,
+                "dune_table": model_details.get("dune_table") if model_details else None,
             }
             
             # Include column details if requested
@@ -591,47 +732,6 @@ def _spellbook_find_models_impl(
     return out
 
 
-@app.tool(
-    name="spellbook_find_models",
-    title="Search Spellbook",
-    description="Search dbt models in Spellbook GitHub repository.",
-    tags={"spellbook", "dbt", "schema"},
-)
-def spellbook_find_models(
-    keyword: str | list[str] | None = None,
-    schema: str | None = None,
-    limit: int = 50,
-    include_columns: bool = True,
-) -> dict[str, Any]:
-    """
-    Search Spellbook dbt models from GitHub repository.
-    
-    Args:
-        keyword: Search term(s) to find models - can be a string or list of strings
-        schema: Schema/subproject name to list tables from
-        limit: Maximum number of models to return
-        include_columns: Whether to include column details in results (default: True)
-    
-    Returns:
-        Dictionary with 'schemas' and 'models' keys
-    """
-    try:
-        return _spellbook_find_models_impl(
-            keyword=keyword,
-            schema=schema,
-            limit=limit,
-            include_columns=include_columns,
-        )
-    except Exception as e:
-        return error_response(e, context={
-            "tool": "spellbook_find_models",
-            "keyword": keyword,
-            "schema": schema,
-        })
-
-
-
-
 # Resources
 @app.resource(uri="spice:history/tail/{n}", name="Query History Tail", description="Tail last N lines from query history")
 def history_tail(n: str) -> str:

spice_mcp/mcp/tools/execute_query.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
 import os
-import re
 import time
 from typing import Any
 
-from ...adapters.dune import extract as dune_extract
 from ...adapters.dune import urls as dune_urls
+from ...adapters.dune.query_wrapper import execute_query as execute_dune_query
+
+# Import user_agent from separate module to avoid importing overloaded functions
+from ...adapters.dune.user_agent import get_user_agent as get_dune_user_agent
 from ...adapters.http_client import HttpClient
 from ...config import Config
 from ...core.errors import error_response
@@ -99,11 +101,13 @@ class ExecuteQueryTool(MCPTool):
     ) -> dict[str, Any]:
         t0 = time.time()
         try:
-            # Rewrite SHOW statements to portable information_schema SELECTs
-            q_use = _maybe_rewrite_show_sql(query) or query
+            # Use native SHOW statements directly - they're faster than information_schema queries
+            # See issue #10: https://github.com/Evan-Kim2028/spice-mcp/issues/10
+            # Removed rewrite to avoid performance issues with information_schema queries
+            q_use = query
             # Poll-only: return execution handle without fetching results
             if format == "poll":
-                exec_obj = dune_extract.query(
+                exec_obj = execute_dune_query(
                     q_use,
                     parameters=parameters,
                     api_key=self.config.dune.api_key,
@@ -333,7 +337,7 @@ class ExecuteQueryTool(MCPTool):
             query_id = dune_urls.get_query_id(query)
             headers = {
                 "X-Dune-API-Key": dune_urls.get_api_key(),
-                "User-Agent": dune_extract.get_user_agent(),
+                "User-Agent": get_dune_user_agent(),
             }
             resp = self._http.request(
                 "GET",
@@ -362,7 +366,7 @@ class ExecuteQueryTool(MCPTool):
                 url = dune_urls.get_execution_status_url(execution_id)
                 headers = {
                     "X-Dune-API-Key": dune_urls.get_api_key(),
-                    "User-Agent": dune_extract.get_user_agent(),
+                    "User-Agent": get_dune_user_agent(),
                 }
                 resp = self._http.request("GET", url, headers=headers, timeout=10.0)
                 data = resp.json()
@@ -393,22 +397,12 @@ def _categorize_query(q: str) -> str:
 
 
 def _maybe_rewrite_show_sql(sql: str) -> str | None:
-    s = sql.strip()
-    m = re.match(r"^SHOW\s+SCHEMAS\s+LIKE\s+'([^']+)'\s*;?$", s, flags=re.IGNORECASE)
-    if m:
-        pat = m.group(1)
-        return (
-            "SELECT schema_name AS Schema FROM information_schema.schemata "
-            f"WHERE schema_name LIKE '{pat}'"
-        )
-    if re.match(r"^SHOW\s+SCHEMAS\s*;?$", s, flags=re.IGNORECASE):
-        return "SELECT schema_name AS Schema FROM information_schema.schemata"
-
-    m = re.match(r"^SHOW\s+TABLES\s+FROM\s+([A-Za-z0-9_\.]+)\s*;?$", s, flags=re.IGNORECASE)
-    if m:
-        schema = m.group(1)
-        return (
-            "SELECT table_name AS Table FROM information_schema.tables "
-            f"WHERE table_schema = '{schema}'"
-        )
+    """DEPRECATED: This function is no longer used.
+    
+    Native SHOW statements are now used directly as they're faster than
+    information_schema queries in Dune. See issue #10 for details.
+    
+    This function is kept for backward compatibility but is not called.
+    """
+    # Function body kept for reference but not executed
     return None