smartapi-mcp 0.3.2__tar.gz → 0.4.0__tar.gz

{smartapi_mcp-0.3.2 → smartapi_mcp-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: smartapi-mcp
-Version: 0.3.2
+Version: 0.4.0
 Summary: Create MCP servers for one or multiple APIs registered in SmartAPI registry
 Author-email: BioThings Team <help@biothings.io>
 Maintainer-email: BioThings Team <help@biothings.io>
@@ -25,13 +25,13 @@ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: awslabs_openapi_mcp_server<1,>=0.2.12
-Requires-Dist: fastmcp<3,>=2.14
+Requires-Dist: awslabs_openapi_mcp_server<2,>=1.1.5
+Requires-Dist: fastmcp<4,>=3.3.1
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0.0; extra == "dev"
 Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
-Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: ruff<0.17,>=0.14; extra == "dev"
 Requires-Dist: build>=0.8.0; extra == "dev"
 Requires-Dist: twine>=4.0.0; extra == "dev"
 Provides-Extra: test
@@ -62,8 +62,9 @@ Built on top of the [AWS Labs OpenAPI MCP Server](https://github.com/awslabs/ope
 
 - Python 3.10 or higher
 - Network access to SmartAPI registry (https://smart-api.info)
-- Dependencies: `awslabs_openapi_mcp_server>=0.2.12,<1` and `fastmcp>=2.14,<3`
-  (awslabs 1.x / fastmcp 3.x are not yet supported — see the dependency notes in
+- Dependencies: `awslabs_openapi_mcp_server>=1.1.5,<2` and `fastmcp>=3.3.1,<4`
+  (fastmcp 4.x is not yet supported — it moves to the MCP 2.x SDK and `httpx2`,
+  and no awslabs release targets it yet; see the dependency notes in
   `pyproject.toml`)
 
 ## Features
@@ -247,6 +248,41 @@ inspect each BioThings spec at startup and serve any API that has extra
 endpoints with faithful per-API tools instead — at the cost of slower startup
 (it downloads the specs upfront).
 
+#### Tool search (`--tool-search`)
+
+The facade solves the tool explosion for BioThings APIs. `--tool-search` solves
+it for everything else, including per-API tools in a hybrid server: instead of
+listing every tool, the server lists two synthetic tools — `search_tools` and
+`call_tool` — and the model discovers what it needs on demand.
+
+```bash
+smartapi-mcp --api_set biothings_all --tool-search bm25
+```
+
+Every tool stays *callable* through `call_tool`; only the listing changes. Facade
+tools (`biothings_query`, `biothings_get`, …) stay listed, so the common path
+remains directly callable and only the per-API long tail is collapsed:
+
+```
+Tool search (bm25) enabled: 13 tools collapsed to 7 listed
+(5 pinned + search_tools/call_tool); max_results=5.
+All 13 tools stay callable via call_tool.
+```
+
+| Mode | Behaviour |
+| --- | --- |
+| `off` (default) | List every tool |
+| `bm25` | Rank matches by keyword relevance |
+| `regex` | Match tool names/descriptions by pattern |
+
+`--tool-search-max-results` (default 5) caps the hits per search. Both options
+have environment equivalents: `SMARTAPI_TOOL_SEARCH` and
+`TOOL_SEARCH_MAX_RESULTS`.
+
+Note that discovery-on-demand costs the model an extra round trip per unfamiliar
+tool, so leave it `off` for small sets where the full list already fits
+comfortably.
+
 #### Development/Testing Setup
 
 ```json
@@ -451,9 +487,10 @@ from smartapi_mcp import (
     load_api_spec,
     get_mcp_server,
     get_merged_mcp_server,
-    PREDEFINED_API_SETS
+    PREDEFINED_API_SETS,
 )
 
+
 async def main():
     # Get SmartAPI IDs using a query
     smartapi_ids = await get_smartapi_ids("tags.name=biothings")
@@ -463,16 +500,15 @@ async def main():
     api_spec = load_api_spec("59dce17363dce279d389100834e43648")  # MyGene.info
     print(f"Loaded API: {api_spec.get('info', {}).get('title', 'Unknown')}")
 
-    # Create MCP server for a single API
-    server = await get_mcp_server(
-        smartapi_id="59dce17363dce279d389100834e43648",
-        server_name="MyGene MCP Server"
-    )
+    # Create MCP server for a single API. The server is named after the
+    # spec's info.title; pass server_name to the merged/`build_server_for_set`
+    # entry points below if you want to choose the name yourself.
+    server = await get_mcp_server("59dce17363dce279d389100834e43648")
 
     # Create merged MCP server for multiple APIs (recommended approach)
     merged_server = await get_merged_mcp_server(
         api_set="biothings_core",  # Use predefined set
-        server_name="BioThings Core MCP Server"
+        server_name="BioThings Core MCP Server",
     )
 
     # Or with specific SmartAPI IDs
@@ -481,7 +517,7 @@ async def main():
             "59dce17363dce279d389100834e43648",  # MyGene.info
             "09c8782d9f4027712e65b95424adba79",  # MyVariant.info
         ],
-        server_name="Custom MCP Server"
+        server_name="Custom MCP Server",
     )
 
     # Show available predefined API sets
@@ -493,6 +529,7 @@ async def main():
     # Or run with HTTP transport
     # merged_server.run(transport="http", host="localhost", port=8000)
 
+
 if __name__ == "__main__":
     asyncio.run(main())
 ```

{smartapi_mcp-0.3.2 → smartapi_mcp-0.4.0}/README.md

@@ -17,8 +17,9 @@ Built on top of the [AWS Labs OpenAPI MCP Server](https://github.com/awslabs/ope
 
 - Python 3.10 or higher
 - Network access to SmartAPI registry (https://smart-api.info)
-- Dependencies: `awslabs_openapi_mcp_server>=0.2.12,<1` and `fastmcp>=2.14,<3`
-  (awslabs 1.x / fastmcp 3.x are not yet supported — see the dependency notes in
+- Dependencies: `awslabs_openapi_mcp_server>=1.1.5,<2` and `fastmcp>=3.3.1,<4`
+  (fastmcp 4.x is not yet supported — it moves to the MCP 2.x SDK and `httpx2`,
+  and no awslabs release targets it yet; see the dependency notes in
   `pyproject.toml`)
 
 ## Features
@@ -202,6 +203,41 @@ inspect each BioThings spec at startup and serve any API that has extra
 endpoints with faithful per-API tools instead — at the cost of slower startup
 (it downloads the specs upfront).
 
+#### Tool search (`--tool-search`)
+
+The facade solves the tool explosion for BioThings APIs. `--tool-search` solves
+it for everything else, including per-API tools in a hybrid server: instead of
+listing every tool, the server lists two synthetic tools — `search_tools` and
+`call_tool` — and the model discovers what it needs on demand.
+
+```bash
+smartapi-mcp --api_set biothings_all --tool-search bm25
+```
+
+Every tool stays *callable* through `call_tool`; only the listing changes. Facade
+tools (`biothings_query`, `biothings_get`, …) stay listed, so the common path
+remains directly callable and only the per-API long tail is collapsed:
+
+```
+Tool search (bm25) enabled: 13 tools collapsed to 7 listed
+(5 pinned + search_tools/call_tool); max_results=5.
+All 13 tools stay callable via call_tool.
+```
+
+| Mode | Behaviour |
+| --- | --- |
+| `off` (default) | List every tool |
+| `bm25` | Rank matches by keyword relevance |
+| `regex` | Match tool names/descriptions by pattern |
+
+`--tool-search-max-results` (default 5) caps the hits per search. Both options
+have environment equivalents: `SMARTAPI_TOOL_SEARCH` and
+`TOOL_SEARCH_MAX_RESULTS`.
+
+Note that discovery-on-demand costs the model an extra round trip per unfamiliar
+tool, so leave it `off` for small sets where the full list already fits
+comfortably.
+
 #### Development/Testing Setup
 
 ```json
@@ -406,9 +442,10 @@ from smartapi_mcp import (
     load_api_spec,
     get_mcp_server,
     get_merged_mcp_server,
-    PREDEFINED_API_SETS
+    PREDEFINED_API_SETS,
 )
 
+
 async def main():
     # Get SmartAPI IDs using a query
     smartapi_ids = await get_smartapi_ids("tags.name=biothings")
@@ -418,16 +455,15 @@ async def main():
     api_spec = load_api_spec("59dce17363dce279d389100834e43648")  # MyGene.info
     print(f"Loaded API: {api_spec.get('info', {}).get('title', 'Unknown')}")
 
-    # Create MCP server for a single API
-    server = await get_mcp_server(
-        smartapi_id="59dce17363dce279d389100834e43648",
-        server_name="MyGene MCP Server"
-    )
+    # Create MCP server for a single API. The server is named after the
+    # spec's info.title; pass server_name to the merged/`build_server_for_set`
+    # entry points below if you want to choose the name yourself.
+    server = await get_mcp_server("59dce17363dce279d389100834e43648")
 
     # Create merged MCP server for multiple APIs (recommended approach)
     merged_server = await get_merged_mcp_server(
         api_set="biothings_core",  # Use predefined set
-        server_name="BioThings Core MCP Server"
+        server_name="BioThings Core MCP Server",
     )
 
     # Or with specific SmartAPI IDs
@@ -436,7 +472,7 @@ async def main():
             "59dce17363dce279d389100834e43648",  # MyGene.info
             "09c8782d9f4027712e65b95424adba79",  # MyVariant.info
         ],
-        server_name="Custom MCP Server"
+        server_name="Custom MCP Server",
     )
 
     # Show available predefined API sets
@@ -448,6 +484,7 @@ async def main():
     # Or run with HTTP transport
     # merged_server.run(transport="http", host="localhost", port=8000)
 
+
 if __name__ == "__main__":
     asyncio.run(main())
 ```

{smartapi_mcp-0.3.2 → smartapi_mcp-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "smartapi-mcp"
-version = "0.3.2"
+version = "0.4.0"
 description = "Create MCP servers for one or multiple APIs registered in SmartAPI registry"
 readme = "README.md"
 license = "Apache-2.0"
@@ -32,12 +32,14 @@ classifiers = [
 keywords = ["mcp", "smartapi", "api", "server", "bioinformatics"]
 requires-python = ">=3.10"
 dependencies = [
-    # awslabs 1.x requires fastmcp 3.x, which removed FastMCP.get_tools() and
-    # rejects *args tool functions -- incompatible with this package for now.
-    # fastmcp is also pinned <3 directly because awslabs 0.2.x allows fastmcp>=2.14
-    # with no upper bound and would otherwise resolve to the breaking 3.x.
-    "awslabs_openapi_mcp_server>=0.2.12,<1",
-    "fastmcp>=2.14,<3",
+    # awslabs 1.x is the release line built against fastmcp 3.x; both are pinned
+    # together because the two migrated in lockstep (fastmcp 3 replaced
+    # FastMCP.get_tools()/get_prompts() with list_tools()/list_prompts(), and
+    # awslabs 1.x switched to the new names).
+    # Upper bounds: fastmcp 4.x moves to the MCP 2.x SDK and httpx2, and no
+    # awslabs release supports it yet (awslabs 1.1.5 itself pins fastmcp<4).
+    "awslabs_openapi_mcp_server>=1.1.5,<2",
+    "fastmcp>=3.3.1,<4",
 ]
 
 [project.optional-dependencies]
@@ -45,7 +47,10 @@ dev = [
     "pytest>=7.0.0",
     "pytest-asyncio>=0.21.0",
     "pytest-cov>=4.0.0",
-    "ruff>=0.1.0",
+    # Upper bound deliberately: an unpinned linter makes CI results drift
+    # with upstream releases, failing on unchanged code. Raise it
+    # intentionally, with a lint/format pass, rather than by surprise.
+    "ruff>=0.14,<0.17",
     "build>=0.8.0",
     "twine>=4.0.0",
 ]
@@ -199,8 +204,11 @@ ignore = [
     "B027",
     # Allow boolean positional values in function calls
     "FBT003",
-    # Ignore complexity
-    "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915",
+    # Ignore complexity. PLR0917 (too-many-positional-arguments) is the same
+    # complaint as PLR0913 for the build entry points, which take a set of
+    # mutually exclusive API selectors; making them keyword-only would be a
+    # breaking change to the public API.
+    "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", "PLR0917",
     # Allow print statements (useful for CLI tools)
     "T201",
     # Allow assert statements

{smartapi_mcp-0.3.2 → smartapi_mcp-0.4.0}/smartapi_mcp/__init__.py

@@ -4,7 +4,7 @@ SmartAPI MCP Server Package
 Create MCP servers for one or multiple APIs registered in SmartAPI registry.
 """
 
-__version__ = "0.3.2"
+__version__ = "0.4.0"
 __author__ = "BioThings Team"
 __email__ = "help@biothings.io"
 

{smartapi_mcp-0.3.2 → smartapi_mcp-0.4.0}/smartapi_mcp/biothings.py

@@ -12,6 +12,7 @@ the set, so the server works on every MCP client without depending on runtime
 """
 
 import asyncio
+import math
 import re
 from dataclasses import dataclass, field
 
@@ -21,6 +22,7 @@ from fastmcp import FastMCP
 from fastmcp.tools import Tool
 
 from .smartapi import (
+    CORE_BIOTHINGS_API_IDS,
     HTTP_TIMEOUT,
     get_base_server_url,
     get_smartapi_registry,
@@ -37,6 +39,20 @@ _MAX_DESC_LEN = 200
 # HTTP methods recognized when enumerating spec operations.
 _HTTP_METHODS = {"get", "post", "put", "delete", "patch", "head", "options"}
 
+# Tags that disqualify an API from the BioThings *annotation-API family* even
+# though it carries the "biothings" tag. TRAPI services (BioThings Explorer,
+# Service Provider) are built by the same team and tagged accordingly, but they
+# speak the Translator Reasoner API -- a query-graph protocol -- not the
+# BioThings annotation interface, so none of the generic facade tools apply to
+# them. They are served with faithful per-API tools instead.
+#
+# This is not a cosmetic distinction: the facade infers an entity type from the
+# first ``/{type}/{id}``-shaped path, and BTE's ``GET /asyncquery_status/{id}``
+# matches that shape. Without this exclusion, ``biothings_get`` would request
+# ``/asyncquery_status/<id>`` and return a job status as though it were an
+# annotation record -- a wrong answer with no error.
+NON_FAMILY_TAGS = frozenset({"trapi"})
+
 
 @dataclass
 class BioThingsAPIEntry:
@@ -96,11 +112,21 @@ async def build_registry(
     return build_registry_from_entries(entries)
 
 
+def is_biothings_family(entry: BioThingsAPIEntry) -> bool:
+    """Whether ``entry`` is a BioThings annotation API the facade can serve.
+
+    Requires the ``biothings`` tag and the absence of any
+    :data:`NON_FAMILY_TAGS`.
+    """
+    tags = {str(tag).strip().lower() for tag in entry.tags}
+    return "biothings" in tags and not (tags & NON_FAMILY_TAGS)
+
+
 def is_biothings_registry(registry: dict[str, BioThingsAPIEntry]) -> bool:
-    """True only if every API in the registry is tagged ``biothings``."""
+    """True only if every API in the registry is a facade-servable BioThings API."""
     if not registry:
         return False
-    return all("biothings" in entry.tags for entry in registry.values())
+    return all(is_biothings_family(entry) for entry in registry.values())
 
 
 def _resolve_endpoints(entry: BioThingsAPIEntry) -> None:
@@ -225,12 +251,146 @@ async def partition_biothings(
     return facade_entries, extra_ids
 
 
-def _score_entry(entry: BioThingsAPIEntry, tokens: list[str]) -> int:
-    """Lexical relevance: count token hits across title/description/tags."""
-    haystack = " ".join(
-        [entry.name, entry.title, entry.description, " ".join(entry.tags)]
-    ).lower()
-    return sum(haystack.count(token) for token in tokens)
+# Words too common to discriminate between APIs. Without this, a natural-language
+# intent like "get a gene annotation by its Entrez gene id" is dominated by
+# "get"/"a"/"by"/"its"/"id" rather than by "gene" and "entrez".
+_STOPWORDS = frozenset(
+    {
+        "a",
+        "an",
+        "and",
+        "are",
+        "as",
+        "at",
+        "be",
+        "by",
+        "for",
+        "from",
+        "get",
+        "how",
+        "in",
+        "into",
+        "is",
+        "it",
+        "its",
+        "of",
+        "on",
+        "or",
+        "that",
+        "the",
+        "their",
+        "them",
+        "there",
+        "these",
+        "this",
+        "to",
+        "what",
+        "when",
+        "where",
+        "which",
+        "who",
+        "with",
+        "api",
+        "apis",
+        "data",
+        "database",
+        "dataset",
+        "info",
+        "information",
+        "record",
+        "records",
+        "service",
+        "services",
+    }
+)
+
+
+def _tokenize(text: str) -> list[str]:
+    """Lower-case word tokens. Word-based, not substring-based, on purpose.
+
+    The previous scorer used ``str.count`` on the raw text, so a query term
+    like "id" also matched inside "identifier", "candidate" and "provide".
+    """
+    return re.findall(r"[a-z0-9]+", text.lower())
+
+
+# How much more a term in the API's name/title counts than one buried in its
+# description. An API *named* for a concept is a far stronger answer to a query
+# about that concept than one merely mentioning it in prose; without this,
+# scores tie constantly and the tie-break (alphabetical) decides, which
+# systematically favours the "biothings_*"-prefixed names over MyGene/MyChem.
+_NAME_FIELD_WEIGHT = 3.0
+
+# Score multiplier for the core BioThings APIs (:data:`CORE_BIOTHINGS_API_IDS`).
+#
+# These are the canonical broad-coverage services, and they are the *worst*
+# served by pure lexical scoring: being general means their descriptions carry
+# the least distinctive vocabulary, while single-source satellite APIs read as
+# highly specific. So a lexical ranker systematically under-ranks exactly the
+# APIs a user most often wants, and needs a prior to correct for it.
+#
+# Measured on 20 BioThings intents. With the registry descriptions as they were
+# before the core-API enrichment, no boost gave recall@5 16/20 (MRR 0.71) and
+# only 3 of the 6 core-API intents were answered; 1.2 gives 19/20 (MRR 0.82).
+# With the enriched descriptions live, no boost already gives 19/20 (MRR 0.92)
+# and 1.2 gives 20/20 (MRR 0.90). Larger values buy nothing on recall and cost
+# ranking quality once the metadata is good -- at 3.0 the post-enrichment MRR
+# falls to 0.81 -- so this is deliberately the smallest value that captures the
+# benefit, and it stays close to neutral as the metadata improves.
+CORE_API_BOOST = 1.2
+
+_CORE_API_IDS = frozenset(CORE_BIOTHINGS_API_IDS)
+
+
+def _entry_terms(entry: BioThingsAPIEntry) -> tuple[set[str], set[str]]:
+    """Return ``(name_terms, all_terms)`` for one API.
+
+    ``name_terms`` covers the short name, title and tags -- the curated labels;
+    ``all_terms`` adds the free-text description.
+    """
+    name_terms = set(
+        _tokenize(" ".join([entry.name, entry.title, " ".join(map(str, entry.tags))]))
+    )
+    return name_terms, name_terms | set(_tokenize(entry.description))
+
+
+def _score_entry(
+    terms: tuple[set[str], set[str]],
+    query_terms: list[str],
+    idf: dict[str, float],
+) -> float:
+    """Score one API against a query. Higher is more relevant.
+
+    ``terms`` is the ``(name_terms, all_terms)`` pair from :func:`_entry_terms`.
+
+    Three deliberate properties:
+
+    * **Binary term frequency.** A term counts once however often it appears, so
+      an API with a long, repetitive description no longer outranks a precisely
+      matching one. This was the concrete defect in the previous scorer:
+      searching "get a gene annotation by its Entrez gene id" ranked MyGeneSet
+      above MyGene, because summed substring counts reward verbosity.
+    * **IDF weighting.** A term shared by most APIs ("gene", "translator")
+      contributes almost nothing, while a rare one ("entrez", "ngd", "taxonomy")
+      dominates -- which is what makes an intent select the API it names.
+    * **Field weighting.** A hit in the name/title/tags counts
+      :data:`_NAME_FIELD_WEIGHT` times one that is only in the description. An
+      API *named* for a concept answers a query about it far better than one
+      merely mentioning it in prose, and without this, scores tie constantly and
+      the alphabetical tie-break decides -- which systematically favoured the
+      ``biothings_*``-prefixed names over MyGene/MyChem/MyVariant.
+    """
+    name_terms, all_terms = terms
+    score = 0.0
+    for term in query_terms:
+        weight = idf.get(term, 0.0)
+        if not weight:
+            continue
+        if term in name_terms:
+            score += weight * _NAME_FIELD_WEIGHT
+        elif term in all_terms:
+            score += weight
+    return score
 
 
 def rank_apis(
@@ -258,11 +418,38 @@ def rank_apis(
     if not keyword or not keyword.strip():
         return [_as_dict(registry[name]) for name in sorted(registry)]
 
-    tokens = [tok for tok in re.split(r"\W+", keyword.lower()) if tok]
-    scored = [(_score_entry(entry, tokens), entry) for entry in registry.values()]
+    query_terms = [t for t in _tokenize(keyword) if t not in _STOPWORDS]
+    if not query_terms:
+        # Nothing discriminating left (e.g. "what data is there?"): fall back to
+        # the full catalog rather than returning an arbitrary subset.
+        return [_as_dict(registry[name]) for name in sorted(registry)]
+
+    terms_by_name = {name: _entry_terms(entry) for name, entry in registry.items()}
+    total = len(registry)
+    idf = {}
+    for term in set(query_terms):
+        seen_in = sum(1 for _, all_terms in terms_by_name.values() if term in all_terms)
+        # Robertson/Sparck-Jones IDF. Chosen over the plain log(N/n) form
+        # because it stays strictly positive even for a term present in every
+        # API: with a small registry (say two APIs that both mention "gene"),
+        # a zero weight would drop every candidate and the search would answer
+        # "nothing found" for a query that in fact matches everything.
+        idf[term] = math.log(1 + (total - seen_in + 0.5) / (seen_in + 0.5))
+
+    scored = []
+    for name, entry in registry.items():
+        score = _score_entry(terms_by_name[name], query_terms, idf)
+        if entry.smartapi_id in _CORE_API_IDS:
+            # Multiplicative, not additive: a zero score stays zero, so the
+            # boost reorders results that already match and never promotes a
+            # core API into a query it has nothing to do with.
+            score *= CORE_API_BOOST
+        scored.append((score, entry))
     scored = [pair for pair in scored if pair[0] > 0]
     scored.sort(key=lambda pair: (-pair[0], pair[1].name))
-    return [{**_as_dict(entry), "score": score} for score, entry in scored[:limit]]
+    return [
+        {**_as_dict(entry), "score": round(score, 3)} for score, entry in scored[:limit]
+    ]
 
 
 def build_biothings_facade(

{smartapi_mcp-0.3.2 → smartapi_mcp-0.4.0}/smartapi_mcp/cli.py

@@ -15,7 +15,7 @@ from awslabs.openapi_mcp_server.server import get_all_counts
 from awslabs.openapi_mcp_server.utils.metrics_provider import metrics
 
 from .config import load_config
-from .server import build_server_for_set
+from .server import TOOL_SEARCH_MODES, build_server_for_set
 
 
 def main():
@@ -26,7 +26,7 @@ def main():
         "--api_set",
         help=(
             "A predefined set of SmartAPI APIs to include. One of: "
-            "'biothings_core' (5 core BioThings APIs), 'biothings_test' "
+            "'biothings_core' (the 6 core BioThings APIs), 'biothings_test' "
             "(core + SemmedDB), or 'biothings_all' (all BioThings APIs). "
             "[env: SMARTAPI_API_SET]"
         ),
@@ -88,7 +88,7 @@ def main():
     parser.add_argument(
         "--facade",
         choices=["auto", "on", "off"],
-        default="auto",
+        default=None,
         help=(
             "How to expose large BioThings sets. The facade collapses BioThings "
             "APIs into ~5 generic tools (the target API is a parameter); any "
@@ -102,7 +102,7 @@ def main():
     parser.add_argument(
         "--facade-threshold",
         type=int,
-        default=10,
+        default=None,
         help=(
             "Number of BioThings APIs in the set at which 'auto' switches to the "
             "facade (default: 10). [env: FACADE_THRESHOLD]"
@@ -118,6 +118,45 @@ def main():
             "[env: FACADE_STRICT]"
         ),
     )
+    parser.add_argument(
+        "--tool-search",
+        choices=list(TOOL_SEARCH_MODES),
+        default=None,
+        help=(
+            "How to expose the tool listing. Serving many APIs produces hundreds "
+            "of tools, which crowds out a client's context. When search is on, "
+            "clients see 'search_tools' and 'call_tool' (plus any facade tools, "
+            "which stay listed) and discover the rest on demand; every tool "
+            "remains callable via 'call_tool'. 'auto' (default) turns search on "
+            "once the server reaches --tool-search-threshold tools. 'bm25' and "
+            "'regex' force it on regardless of size; 'off' always lists "
+            "everything. Prefer 'bm25' over 'regex': regex needs a real pattern "
+            "and returns nothing if given a natural-language query. CLI "
+            "overrides the environment variable, which overrides the default. "
+            "[env: SMARTAPI_TOOL_SEARCH]"
+        ),
+    )
+    parser.add_argument(
+        "--tool-search-threshold",
+        type=int,
+        default=None,
+        help=(
+            "Tool count at which --tool-search 'auto' turns search on "
+            "(default: 15). A listed tool costs ~300-1000 tokens of client "
+            "context, so 15 is roughly a 5-15k-token ceiling on the listing. "
+            "[env: TOOL_SEARCH_THRESHOLD]"
+        ),
+    )
+    parser.add_argument(
+        "--tool-search-max-results",
+        type=int,
+        default=None,
+        help=(
+            "Maximum number of tools returned per 'search_tools' call "
+            "(default: 10). Only used when tool search is active. "
+            "[env: TOOL_SEARCH_MAX_RESULTS]"
+        ),
+    )
     parser.add_argument(
         "--log-level",
         choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
@@ -150,6 +189,8 @@ def main():
                 facade=getattr(config, "facade", "auto"),
                 facade_threshold=getattr(config, "facade_threshold", 10),
                 facade_strict=getattr(config, "facade_strict", False),
+                tool_search=getattr(config, "tool_search", "auto"),
+                tool_search_max_results=getattr(config, "tool_search_max_results", 5),
             )
         )
     except ValueError as e:

{smartapi_mcp-0.3.2 → smartapi_mcp-0.4.0}/smartapi_mcp/config.py

@@ -18,6 +18,9 @@ class Config(_config.Config):
     facade: str = "auto"
     facade_threshold: int = 10
     facade_strict: bool = False
+    tool_search: str = "auto"
+    tool_search_max_results: int = 10
+    tool_search_threshold: int = 15
 
 
 def _parse_bool(value: str) -> bool:
@@ -51,6 +54,15 @@ def load_config(args: Any = None) -> Config:
             lambda v: setattr(config, "facade_threshold", _parse_int(v, 10))
         ),
         "FACADE_STRICT": (lambda v: setattr(config, "facade_strict", _parse_bool(v))),
+        "SMARTAPI_TOOL_SEARCH": (
+            lambda v: setattr(config, "tool_search", v.strip().lower())
+        ),
+        "TOOL_SEARCH_MAX_RESULTS": (
+            lambda v: setattr(config, "tool_search_max_results", _parse_int(v, 10))
+        ),
+        "TOOL_SEARCH_THRESHOLD": (
+            lambda v: setattr(config, "tool_search_threshold", _parse_int(v, 15))
+        ),
         "SERVER_NAME": (lambda v: setattr(config, "server_name", v)),
     }
 
@@ -108,6 +120,12 @@ def load_config(args: Any = None) -> Config:
             config.facade_threshold = int(args.facade_threshold)
         if getattr(args, "facade_strict", False):
             config.facade_strict = True
+        if getattr(args, "tool_search", None):
+            config.tool_search = str(args.tool_search).strip().lower()
+        if getattr(args, "tool_search_max_results", None):
+            config.tool_search_max_results = int(args.tool_search_max_results)
+        if getattr(args, "tool_search_threshold", None):
+            config.tool_search_threshold = int(args.tool_search_threshold)
         if hasattr(args, "transport") and args.transport:
             logger.debug(
                 f"Setting MCP Server transport mode from arguments: {args.transport}"

{smartapi_mcp-0.3.2 → smartapi_mcp-0.4.0}/smartapi_mcp/server.py

@@ -6,14 +6,25 @@ Main MCP server implementation for SmartAPI integration.
 
 import hashlib
 import re
+from collections.abc import Iterable
 
 from awslabs.openapi_mcp_server import logger
 from awslabs.openapi_mcp_server.api.config import Config
 from awslabs.openapi_mcp_server.server import create_mcp_server_async
 from fastmcp import FastMCP
+from fastmcp.server.transforms.search import (
+    BM25SearchTransform,
+    RegexSearchTransform,
+    serialize_tools_for_output_markdown,
+)
 
 # Import BioThings generic-facade builder
-from .biothings import build_biothings_facade, build_registry, partition_biothings
+from .biothings import (
+    build_biothings_facade,
+    build_registry,
+    is_biothings_family,
+    partition_biothings,
+)
 
 # Import from smartapi module - avoiding circular imports
 from .smartapi import (
@@ -33,6 +44,121 @@ from .smartapi import (
 # clients that reuse the tool-name validator.
 MAX_TOOL_NAME_LEN = 64
 
+# Ways to expose a large tool catalog. "off" lists every tool; "bm25"/"regex"
+# always replace the catalog with a search interface; "auto" picks between them
+# by tool count (see :func:`apply_tool_search`).
+TOOL_SEARCH_MODES = ("auto", "off", "bm25", "regex")
+
+# Mode "auto" resolves to. BM25 handles natural-language queries; regex needs the
+# caller to author a pattern and returns nothing (silently) if handed prose.
+TOOL_SEARCH_AUTO_MODE = "bm25"
+
+# Tool count at which "auto" turns search on.
+#
+# Measured over the registry's uptime-passing set (592 tools, 92 APIs), a single
+# entry in `tools/list` -- name plus the enriched description plus the JSON input
+# schema -- averages ~3,900 characters (~975 tokens), median ~1,270 (~320), p90
+# ~7,500, with one TRAPI tool at 84,000 (~21,000 tokens). At this threshold a
+# listing therefore costs roughly 5k tokens of median-sized tools or 15k of
+# mean-sized ones, which is a reasonable ceiling to pay before search is worth
+# its extra round trip.
+#
+# Note the 65x spread: tool *count* is a crude proxy for the thing we actually
+# care about, which is payload size. A byte/token budget would be the better
+# instrument and would make this constant a floor rather than the decision.
+TOOL_SEARCH_AUTO_THRESHOLD = 15
+
+
+async def apply_tool_search(
+    server: FastMCP,
+    mode: str = "off",
+    *,
+    max_results: int = 10,
+    always_visible: Iterable[str] = (),
+    threshold: int = TOOL_SEARCH_AUTO_THRESHOLD,
+) -> FastMCP:
+    """Collapse ``server``'s tool catalog behind a search interface.
+
+    Serving many APIs from one server makes the tool list long enough to crowd
+    out a client's context: the ``biothings_all`` set is ~50 APIs at ~6 tools
+    each. A search transform replaces the listed catalog with two synthetic
+    tools -- ``search_tools`` and ``call_tool`` -- so a model discovers tools on
+    demand instead of receiving every schema upfront. Every real tool remains
+    callable through ``call_tool``; only the *listing* changes.
+
+    Names in ``always_visible`` stay listed alongside the synthetic tools.
+    :func:`build_server_for_set` pins the BioThings facade tools this way, so
+    the common path stays directly callable and only the per-API long tail is
+    collapsed. That combination is the intended arrangement: the facade answers
+    BioThings queries directly (where lexical search is weakest, because the
+    generated per-API descriptions are near-identical boilerplate), and search
+    covers the non-BioThings tail (where it works well).
+
+    ``mode`` is one of :data:`TOOL_SEARCH_MODES`:
+
+    ``"auto"``
+        Enable :data:`TOOL_SEARCH_AUTO_MODE` once the server has at least
+        ``threshold`` tools; leave smaller catalogs listed in full.
+    ``"off"``
+        Leave the catalog alone.
+    ``"bm25"`` / ``"regex"``
+        Always enable that transform, regardless of size.
+
+    ``max_results`` caps the hits per search. ``server`` is mutated in place and
+    returned.
+    """
+    if mode not in TOOL_SEARCH_MODES:
+        err_msg = (
+            f"Unknown tool search mode {mode!r}; "
+            f"expected one of: {', '.join(TOOL_SEARCH_MODES)}."
+        )
+        raise ValueError(err_msg)
+    if mode == "off":
+        return server
+
+    tool_count = len(await server.list_tools())
+    if not tool_count:
+        # Leave an empty server alone so the caller's "no tools registered"
+        # diagnostics still fire instead of counting the synthetic tools.
+        logger.warning(
+            f"Tool search ({mode}) requested but the server has no tools; "
+            "leaving the catalog unchanged."
+        )
+        return server
+
+    if mode == "auto":
+        if tool_count < threshold:
+            logger.info(
+                f"Tool search (auto): {tool_count} tools is below the "
+                f"{threshold}-tool threshold; listing them all."
+            )
+            return server
+        logger.info(
+            f"Tool search (auto): {tool_count} tools reaches the "
+            f"{threshold}-tool threshold; enabling {TOOL_SEARCH_AUTO_MODE}."
+        )
+        mode = TOOL_SEARCH_AUTO_MODE
+
+    pinned = sorted(always_visible)
+    transform_cls = BM25SearchTransform if mode == "bm25" else RegexSearchTransform
+    server.add_transform(
+        transform_cls(
+            max_results=max_results,
+            always_visible=pinned,
+            # Markdown results are roughly half the size of the default JSON
+            # serialization, which is the point when enabling search at all.
+            search_result_serializer=serialize_tools_for_output_markdown,
+        )
+    )
+    exposed = len(await server.list_tools())
+    logger.info(
+        f"Tool search ({mode}) enabled: {tool_count} tools collapsed to "
+        f"{exposed} listed ({len(pinned)} pinned + search_tools/call_tool); "
+        f"max_results={max_results}. All {tool_count} tools stay callable "
+        "via call_tool."
+    )
+    return server
+
 
 async def get_mcp_server(smartapi_id: str) -> FastMCP:
     config = Config(
@@ -72,6 +198,53 @@ def _fit_name(name: str, used: set[str]) -> str:
     return truncated
 
 
+async def build_api_servers(
+    smartapi_ids: list[str],
+) -> tuple[list[FastMCP], list[tuple[str, str]]]:
+    """Build one MCP server per SmartAPI id, skipping the ones that fail.
+
+    Returns ``(servers, failures)`` where each failure is ``(smartapi_id,
+    reason)``.
+
+    Not every registered spec can be turned into a server: some use external
+    ``$ref``s (refused by awslabs 1.x as an SSRF guard), some are invalid
+    OpenAPI, some have no ``servers`` block. Roughly one in six of the
+    registry's uptime-passing APIs fails for one of those reasons, and a single
+    one of them used to abort the whole build -- so ``--smartapi_q
+    '_status.uptime_status:pass'`` could not start at all. Serving the APIs that
+    do work, and reporting the rest, is far more useful than serving none.
+
+    Kept sequential like the code it replaces: fanning these out concurrently
+    makes the SmartAPI registry start refusing DNS/connections partway through,
+    which turns working APIs into spurious failures.
+    """
+    servers: list[FastMCP] = []
+    failures: list[tuple[str, str]] = []
+    for sid in smartapi_ids:
+        try:
+            servers.append(await get_mcp_server(sid))
+        # SystemExit is caught alongside Exception on purpose. awslabs'
+        # create_mcp_server_async reports *every* spec error by calling
+        # sys.exit(1) from inside the library, so a spec that fastmcp itself
+        # rejects (e.g. an OpenAPI 3.0 document using "type": "null") raises
+        # SystemExit rather than an Exception -- which "except Exception" does
+        # not catch, and which therefore still took down every other API in the
+        # set. Confirmed on the registry's uptime-passing set, where it killed a
+        # 27-API build at API 17. This is scoped tightly to one call, so it
+        # cannot swallow a genuine interpreter exit (and Ctrl-C raises
+        # KeyboardInterrupt, not SystemExit).
+        except (Exception, SystemExit) as exc:  # any spec problem is survivable
+            reason = f"{type(exc).__name__}: {str(exc)[:200]}"
+            failures.append((sid, reason))
+            logger.warning(f"Skipping SmartAPI {sid}: {reason}")
+    if failures:
+        logger.warning(
+            f"{len(failures)} of {len(smartapi_ids)} API(s) could not be loaded "
+            f"and were skipped; {len(servers)} loaded successfully."
+        )
+    return servers, failures
+
+
 async def _merge_servers_into(
     target: FastMCP, list_of_servers: list[FastMCP]
 ) -> FastMCP:
@@ -83,33 +256,41 @@ async def _merge_servers_into(
     # Seed with names already in the target (e.g. facade tools in the hybrid
     # path) so merged per-API tools/prompts never collide with them. Tools and
     # prompts have separate namespaces, so each gets its own set.
-    used_tool_names: set[str] = set(await target.get_tools())
-    used_prompt_names: set[str] = set(await target.get_prompts())
+    used_tool_names: set[str] = {tool.name for tool in await target.list_tools()}
+    used_prompt_names: set[str] = {
+        prompt.name for prompt in await target.list_prompts()
+    }
     for server in list_of_servers:
         api_name = re.sub(
             r"[^a-z0-9_-]", "_", getattr(server, "name", "unknown_api").lower()
         )
 
-        tools = await server.get_tools()
+        tools = await server.list_tools()
         if tools:
-            for original_name, tool in tools.items():
+            for tool in tools:
                 # Rename the tool by prefixing with API name, keeping it within
-                # the 64-char limit that MCP clients enforce.
-                prefixed = f"{api_name}_{original_name}"
+                # the 64-char limit that MCP clients enforce. Renaming in place
+                # is safe: Component.key is a property derived from .name, so
+                # the target registers the tool under its new name.
+                prefixed = f"{api_name}_{tool.name}"
                 tool.name = _fit_name(prefixed, used_tool_names)
                 used_tool_names.add(tool.name)
                 target.add_tool(tool)
         else:
-            err_msg = f"Server {server} does not have accessible tools."
-            raise AttributeError(err_msg)
+            # A spec that parses but yields no tools is a property of that one
+            # API, not a reason to lose every other API in the set.
+            logger.warning(
+                f"API '{api_name}' contributed no tools; skipping it. Its spec "
+                "parsed but produced no callable operations."
+            )
 
         # Merge prompts
-        prompts = await server.get_prompts()
+        prompts = await server.list_prompts()
         if prompts:
-            for original_name, prompt in prompts.items():
+            for prompt in prompts:
                 # Rename the prompt by prefixing with API name, keeping it
                 # within the 64-char limit that MCP clients enforce.
-                prefixed = f"{api_name}_{original_name}"
+                prefixed = f"{api_name}_{prompt.name}"
                 prompt.name = _fit_name(prefixed, used_prompt_names)
                 used_prompt_names.add(prompt.name)
                 target.add_prompt(prompt)
@@ -168,13 +349,12 @@ async def get_merged_mcp_server(
         err_msg = "No SmartAPI IDs provided or found with the given query."
         raise ValueError(err_msg)
     smartapi_exclude_ids = smartapi_exclude_ids or []
-    list_of_servers = [
-        await get_mcp_server(sid)
-        for sid in smartapi_ids
-        if sid not in smartapi_exclude_ids
-    ]
+    wanted = [sid for sid in smartapi_ids if sid not in smartapi_exclude_ids]
+    list_of_servers, _failures = await build_api_servers(wanted)
     merged_server = await merge_mcp_servers(list_of_servers, server_name)
-    logger.info(f"Merged {len(list_of_servers)} APIs into one MCP server.")
+    logger.info(
+        f"Merged {len(list_of_servers)} of {len(wanted)} APIs into one MCP server."
+    )
     return merged_server
 
 
@@ -221,6 +401,9 @@ async def build_server_for_set(
     facade: str = "auto",
     facade_threshold: int = 10,
     facade_strict: bool = False,
+    tool_search: str = "auto",
+    tool_search_max_results: int = 10,
+    tool_search_threshold: int = TOOL_SEARCH_AUTO_THRESHOLD,
 ) -> FastMCP:
     """Build the MCP server for an API set, picking the right strategy.
 
@@ -240,6 +423,15 @@ async def build_server_for_set(
     downloads). When ``True``, each BioThings spec is inspected and any API with
     extra endpoints is served with faithful per-API tools instead (slower
     startup; downloads specs upfront).
+
+    ``tool_search`` (see :data:`TOOL_SEARCH_MODES`, default ``"auto"``)
+    additionally collapses the tool listing behind a search interface, which is
+    the answer to the per-API tool explosion when the facade does not apply.
+    Facade tools are pinned so they stay listed, giving a hybrid server whose
+    BioThings half is answered by the facade and whose per-API half is
+    discovered by search. ``"auto"`` engages only once the merged server has
+    ``tool_search_threshold`` tools; ``tool_search_max_results`` caps hits per
+    search.
     """
     available_ids = await _resolve_smartapi_ids(
         smartapi_q=smartapi_q,
@@ -251,8 +443,13 @@ async def build_server_for_set(
 
     if facade != "off":
         registry = await build_registry(available_ids)
+        # TRAPI services carry the "biothings" tag but are not annotation APIs;
+        # is_biothings_family excludes them so they fall through to the
+        # non_biothings_ids branch below and get faithful per-API tools.
         biothings = {
-            name: entry for name, entry in registry.items() if "biothings" in entry.tags
+            name: entry
+            for name, entry in registry.items()
+            if is_biothings_family(entry)
         }
         if facade == "on" and not biothings:
             logger.warning(
@@ -279,26 +476,47 @@ async def build_server_for_set(
 
             if facade_entries:
                 server = build_biothings_facade(facade_entries, server_name)
+                # Capture the facade tools before merging per-API servers so
+                # tool search can pin them and collapse only the long tail.
+                # Skipped entirely when tool search is off, to keep the default
+                # path free of extra work.
+                facade_tool_names: list[str] = (
+                    [tool.name for tool in await server.list_tools()]
+                    if tool_search != "off"
+                    else []
+                )
                 if per_api_ids:
                     logger.info(
                         f"Hybrid server: facade over {len(facade_entries)} "
                         f"BioThings API(s) + per-API tools for "
                         f"{len(per_api_ids)} other API(s)."
                     )
-                    extra_servers = [await get_mcp_server(sid) for sid in per_api_ids]
+                    extra_servers, _failures = await build_api_servers(per_api_ids)
                     await _merge_servers_into(server, extra_servers)
                 else:
                     logger.info(
                         f"Using BioThings facade for {len(facade_entries)} APIs "
                         f"(server_name={server_name})."
                     )
-                return server
+                return await apply_tool_search(
+                    server,
+                    tool_search,
+                    max_results=tool_search_max_results,
+                    always_visible=facade_tool_names,
+                    threshold=tool_search_threshold,
+                )
             logger.info(
                 "No APIs qualified for the BioThings facade; using per-API tools."
             )
 
     logger.info(f"Using per-API tools for {len(available_ids)} APIs.")
-    return await get_merged_mcp_server(
+    server = await get_merged_mcp_server(
         smartapi_ids=available_ids,
         server_name=server_name,
     )
+    return await apply_tool_search(
+        server,
+        tool_search,
+        max_results=tool_search_max_results,
+        threshold=tool_search_threshold,
+    )

{smartapi_mcp-0.3.2 → smartapi_mcp-0.4.0}/smartapi_mcp/smartapi.py

@@ -106,33 +106,34 @@ def get_base_server_url(api_spec: dict) -> str:
     return base_server_url
 
 
+# The core BioThings APIs: the canonical, broad-coverage annotation services,
+# as distinct from the ~50 single-source satellite APIs. Named because it serves
+# two purposes that must not drift apart -- the ``biothings_core`` preset (which
+# APIs to serve) and discovery ranking (which APIs to prefer when several match
+# a query, see ``CORE_API_BOOST`` in :mod:`smartapi_mcp.biothings`).
+CORE_BIOTHINGS_API_IDS = [
+    "59dce17363dce279d389100834e43648",  # MyGene.info
+    "09c8782d9f4027712e65b95424adba79",  # MyVariant.info
+    "8f08d1446e0bb9c2b323713ce83e2bd3",  # MyChem.info
+    "671b45c0301c8624abbd26ae78449ca2",  # MyDisease.info
+    "85139f4dccfcefa3ac3042372066916d",  # MyGeneSet.info
+    "f7943e6167166b3ea9e4b8be08f45fa6",  # MyTaxon.info
+]
+
+# SemmedDB, added to the "test" set for its non-standard /query/ngd endpoint.
+_SEMMEDDB_ID = "1d288b3a3caf75d541ffaae3aab386c8"
+
 PREDEFINED_API_SETS = ["biothings_core", "biothings_test", "biothings_all"]
 
 
 def get_predefined_api_set(api_set: str) -> dict:
     """Return the predefined API set for the given set name."""
     if api_set == "biothings_core":
-        return {
-            "smartapi_ids": [
-                "59dce17363dce279d389100834e43648",  # MyGene.info
-                "09c8782d9f4027712e65b95424adba79",  # MyVariant.info
-                "8f08d1446e0bb9c2b323713ce83e2bd3",  # MyChem.info
-                "671b45c0301c8624abbd26ae78449ca2",  # MyDisease.info
-                "85139f4dccfcefa3ac3042372066916d",  # MyGeneSet.info
-            ]
-        }
+        return {"smartapi_ids": list(CORE_BIOTHINGS_API_IDS)}
     if api_set == "biothings_test":
-        # biothings core APIs plus the SemmedDB API, useful for testings
-        return {
-            "smartapi_ids": [
-                "59dce17363dce279d389100834e43648",  # MyGene.info
-                "09c8782d9f4027712e65b95424adba79",  # MyVariant.info
-                "8f08d1446e0bb9c2b323713ce83e2bd3",  # MyChem.info
-                "671b45c0301c8624abbd26ae78449ca2",  # MyDisease.info
-                "85139f4dccfcefa3ac3042372066916d",  # MyGeneSet.info
-                "1d288b3a3caf75d541ffaae3aab386c8",  # SemmedDB
-            ]
-        }
+        # The core APIs plus SemmedDB, whose /query/ngd endpoint exercises the
+        # non-standard-endpoint handling.
+        return {"smartapi_ids": [*CORE_BIOTHINGS_API_IDS, _SEMMEDDB_ID]}
     if api_set == "biothings_all":
         # include all biothings APIs with a few excluded
         return {