world-reference-mcp 0.2.0__py3-none-any.whl

scripts/compute_tool_fingerprints.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Regenerate ``connectors/tool_schema_fingerprints.json`` from the live tool surface.
+
+Connects an in-process FastMCP client to a plain server carrying this
+package's nine tools (no auth/network/fleet-registration wiring — those are
+concerns of ``agent_connector_sdk.mcp.server.create_mcp_server``, irrelevant
+to a tool's client-visible schema) and fingerprints exactly what a real
+client's ``tools/list`` would see, via
+``agent_connector_sdk.certify.fingerprints.tool_fingerprint`` — the same
+function ``discover()`` verifies against at extraction time.
+
+Run after changing any tool's signature or docstring-derived schema:
+
+    uv run --frozen python scripts/compute_tool_fingerprints.py --apply
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+from pathlib import Path
+
+from agent_connector_sdk.certify.fingerprints import tool_fingerprint
+from fastmcp import Client, FastMCP
+
+from world_reference_mcp.mcp.clients import Clients
+from world_reference_mcp.mcp.tools_nutrition import register_nutrition_tools
+from world_reference_mcp.mcp.tools_observations import register_observation_tools
+from world_reference_mcp.mcp.tools_taxonomy import register_taxonomy_tools
+from world_reference_mcp.mcp.tools_terms import register_term_tools
+from world_reference_mcp.mcp.tools_wikidata import register_wikidata_tools
+
+_OUTPUT = Path(__file__).resolve().parent.parent / (
+    "world_reference_mcp/connectors/tool_schema_fingerprints.json"
+)
+
+
+def _bare_server() -> FastMCP:
+    mcp: FastMCP = FastMCP("world-reference-mcp")
+    clients = Clients()
+    register_taxonomy_tools(mcp, clients)
+    register_term_tools(mcp, clients)
+    register_nutrition_tools(mcp, clients)
+    register_observation_tools(mcp, clients)
+    register_wikidata_tools(mcp, clients)
+    return mcp
+
+
+async def _fingerprints() -> dict[str, str]:
+    mcp = _bare_server()
+    async with Client(mcp) as client:
+        tools = await client.list_tools()
+    return {tool.name: tool_fingerprint(tool) for tool in tools}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--apply", action="store_true", help="write the output file")
+    args = parser.parse_args()
+    fingerprints = asyncio.run(_fingerprints())
+    document = {
+        "_comment": (
+            "Generated by scripts/compute_tool_fingerprints.py from the live "
+            "tools/list schema. Never hand-edit."
+        ),
+        "tools": dict(sorted(fingerprints.items())),
+    }
+    text = json.dumps(document, indent=2, sort_keys=False) + "\n"
+    if args.apply:
+        _OUTPUT.write_text(text)
+        print(f"wrote {_OUTPUT}")
+    else:
+        print(text)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/security_sanitizer.py

@@ -0,0 +1,318 @@
+#!/usr/bin/env python3
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+MAX_SCAN_BYTES = 8 * 1024 * 1024
+
+# Config
+# llms.txt is the deliberate root-level AI entry index (llms-txt convention,
+# like robots.txt) shipped by the docs/day-0 work — not garbage.
+# overrides.txt is the sanctioned uv dependency-override file (UV_OVERRIDE in
+# docker/Dockerfile, mirroring [tool.uv] override-dependencies) shipped by the
+# pydantic-ai v2 migration — a canonical packaging input, not scratch.
+# .security-audit-allow.txt is the OSV dependency-audit risk-acceptance ledger
+# (scripts/audit_dependencies.py, wired into the dependency-audit pre-commit hook)
+# — a committed, actively-read governance input, not scratch/garbage.
+ALLOWED_TXT_NAMES = {
+    "requirements.txt",
+    "requirements-dev.txt",
+    "llms.txt",
+    "overrides.txt",
+    ".security-audit-allow.txt",
+}
+TRANSIENT_PY_PATTERNS = [
+    re.compile(r"^test_.*\.py$"),
+    re.compile(r"^fix_.*\.py$"),
+    re.compile(r"^debug_.*\.py$"),
+    re.compile(r"^scratch_.*\.py$"),
+    re.compile(r"^temp_.*\.py$"),
+]
+
+SECRET_PATTERNS = [
+    ("GitHub PAT", re.compile(r"ghp_[A-Za-z0-9_]{36,255}")),
+    ("GitHub Fine-grained PAT", re.compile(r"github_pat_[A-Za-z0-9_]{82,255}")),
+    ("GitLab PAT", re.compile(r"glpat-[A-Za-z0-9\-]{20,255}")),
+    ("Private key", re.compile(r"-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----")),
+    ("AWS access key", re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b")),
+    ("Slack token", re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b")),
+    ("Langfuse secret", re.compile(r"\bsk-lf-[A-Za-z0-9_-]{16,}\b")),
+    (
+        "Generic Secret Assignment",
+        re.compile(
+            r"secret[A-Za-z0-9_]*\s*[:=]\s*['\"][A-Za-z0-9_\-\.\~\*]{16,255}['\"]",
+            re.IGNORECASE,
+        ),
+    ),
+    (
+        "Generic Token Assignment",
+        re.compile(
+            r"token\s*[:=]\s*['\"][A-Za-z0-9_\-\.\~\*]{16,255}['\"]", re.IGNORECASE
+        ),
+    ),
+]
+
+EXCLUDED_DIRS = {
+    ".git",
+    ".venv",
+    "venv",
+    "node_modules",
+    "build",
+    "dist",
+    "__pycache__",
+    ".tox",
+    ".specify",
+    ".mypy_cache",
+    ".pytest_cache",
+    ".ruff_cache",
+    ".cache",
+}
+EXCLUDED_EXTENSIONS = {
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".gif",
+    ".webp",
+    ".ico",
+    ".pyc",
+    ".db",
+    ".kuzu",
+    ".sqlite",
+    ".sqlite3",
+    ".zip",
+    ".tar.gz",
+    ".tgz",
+    ".bz2",
+    ".xz",
+    ".pdf",
+    ".bin",
+    ".exe",
+    ".dll",
+    ".so",
+    ".dylib",
+    ".woff",
+    ".woff2",
+    ".eot",
+    ".ttf",
+    ".mp4",
+    ".mp3",
+    ".wav",
+    ".lock",
+    ".svg",
+}
+
+# Placeholder / Mock indicators
+PLACEHOLDER_VALUES = {
+    "1234567890",
+    "abcdef12345",
+    "abc123youandme",
+    "askdfalskdvjas",
+    "test_token",
+    "test_secret",
+    "glpat-askdfalskdvjas",
+    "github_pat_12345",
+    "glpat-abc123youandme",
+    "github_pat_...",
+    "glpat-*************",
+    "ghp_*************",
+    "github_pat_*************",
+    "token_*************",
+    "secret_*************",
+    "glpat-abc",
+    "ghp_abc",
+    "github_pat_abc",
+    "${env:",
+}
+PLACEHOLDER_PREFIXES = ("your_", "your-", "dummy", "example", "mock")
+
+
+def is_placeholder(match_str: str) -> bool:
+    # Generic assignment patterns include the variable name. Judge only the
+    # quoted value so names such as ``secret_example`` cannot suppress a real
+    # credential finding.
+    assignment = re.search(r"['\"]([^'\"]+)['\"]\s*$", match_str)
+    candidate = assignment.group(1) if assignment else match_str
+    candidate_lower = candidate.strip().lower()
+    if candidate_lower in PLACEHOLDER_VALUES or candidate_lower.startswith(
+        PLACEHOLDER_PREFIXES
+    ):
+        return True
+
+    # Check if match is mostly asterisks or single repeated char
+    cleaned = candidate.replace("'", "").replace('"', "").strip()
+    if not cleaned:
+        return True
+
+    # Check if there are sequences of asterisks indicating masked values
+    if "*" in cleaned:
+        # e.g., glpat-*************
+        return True
+
+    compact = re.sub(r"[^A-Za-z0-9]", "", cleaned).lower()
+    if len(compact) >= 8 and len(set(compact)) == 1:
+        return True
+
+    return False
+
+
+def _is_excluded_relative_path(relative: Path) -> bool:
+    if relative.is_absolute() or ".." in relative.parts:
+        raise ValueError("unsafe source inventory path")
+    return any(part in EXCLUDED_DIRS for part in relative.parts)
+
+
+def _git_tracked_files(repo_path: Path):
+    result = subprocess.run(
+        ["git", "ls-files", "--cached", "--others", "--exclude-standard"],
+        cwd=str(repo_path),
+        capture_output=True,
+        text=True,
+        check=True,
+        timeout=30,
+    )
+    files = []
+    for line in result.stdout.splitlines():
+        if not line.strip():
+            continue
+        relative = Path(line.strip())
+        # Avoid files inside excluded directories
+        if not _is_excluded_relative_path(relative):
+            files.append(repo_path / relative)
+    return files
+
+
+def _walk_tracked_files(repo_path: Path):
+    # Fallback to manual recursive scan
+    files = []
+    for root, dirs, walk_files in os.walk(str(repo_path)):
+        # Hidden source/config directories (for example ``.github``) can
+        # contain credentials and must not disappear merely because Git
+        # inventory was unavailable. Exclude only known generated trees.
+        dirs[:] = [d for d in dirs if d not in EXCLUDED_DIRS]
+        for file in walk_files:
+            files.append(Path(root) / file)
+    return files
+
+
+def get_repo_files(repo_path: Path):
+    try:
+        return _git_tracked_files(repo_path)
+    except (OSError, subprocess.SubprocessError, ValueError):
+        return _walk_tracked_files(repo_path)
+
+
+def _root_naming_violations(file_path: Path):
+    violations = []
+    if file_path.suffix == ".txt":
+        if file_path.name.lower() not in ALLOWED_TXT_NAMES:
+            violations.append(
+                "Non-standard root-level text file detected: "
+                f"'{file_path.name}'. Only {sorted(ALLOWED_TXT_NAMES)} "
+                "are allowed."
+            )
+    elif file_path.suffix == ".py":
+        for pattern in TRANSIENT_PY_PATTERNS:
+            if pattern.match(file_path.name):
+                violations.append(
+                    "Transient/temporary script detected in root: "
+                    f"'{file_path.name}'. Please move it to a subfolder "
+                    "or delete it."
+                )
+                break
+    return violations
+
+
+def _should_secret_scan(file_path: Path) -> bool:
+    if file_path.suffix.lower() in EXCLUDED_EXTENSIONS:
+        return False
+    if file_path.name == "security_sanitizer.py":
+        return False
+    return True
+
+
+def _secret_violations_in_line(line: str, rel_path: Path, idx: int):
+    violations = []
+    for label, pattern in SECRET_PATTERNS:
+        for match in pattern.findall(line):
+            match_str = match[0] if isinstance(match, tuple) else match
+            if not is_placeholder(match_str):
+                violations.append(
+                    f"Potential unmasked secret ({label}) detected in "
+                    f"{rel_path}:{idx}"
+                )
+    return violations
+
+
+def _secret_scan_violations(file_path: Path, repo_path: Path):
+    rel_path = file_path.relative_to(repo_path)
+    if file_path.stat().st_size > MAX_SCAN_BYTES:
+        return [
+            f"Source file exceeds security scan boundary: '{rel_path}'",
+        ]
+
+    # Decode strictly. Silently discarding invalid bytes can splice a
+    # credential around the discarded byte and make a fail-open scan.
+    content = file_path.read_text(encoding="utf-8")
+    violations = []
+    for idx, line in enumerate(content.splitlines(), 1):
+        violations.extend(_secret_violations_in_line(line, rel_path, idx))
+    return violations
+
+
+def _scan_one_file(file_path: Path, repo_path: Path):
+    violations = []
+
+    # 1. Check root level naming constraints
+    if file_path.parent == repo_path:
+        violations.extend(_root_naming_violations(file_path))
+
+    # 2. Check for secrets
+    if not _should_secret_scan(file_path):
+        return violations
+
+    try:
+        violations.extend(_secret_scan_violations(file_path, repo_path))
+    except (OSError, UnicodeError):
+        violations.append(
+            f"Source file could not be inspected: "
+            f"'{file_path.relative_to(repo_path)}'"
+        )
+    return violations
+
+
+def scan_repository(repo_path: Path):
+    violations = []
+
+    for file_path in get_repo_files(repo_path):
+        if not file_path.is_file():
+            continue
+        if file_path.is_symlink():
+            # Git tracks the link target text, not the target contents. Never
+            # follow a repository symlink into machine-local material.
+            continue
+        violations.extend(_scan_one_file(file_path, repo_path))
+
+    return violations
+
+
+def main():
+    repo_path = Path.cwd()
+
+    print("🔒 Running Security and Garbage Sanitizer...")
+    violations = scan_repository(repo_path)
+
+    if violations:
+        print("\n❌ SECURITY AND GARBAGE VALIDATION FAILED!")
+        print("Please correct the following issues before committing:")
+        for idx, violation in enumerate(violations, 1):
+            print(f"\n[{idx}] {violation}")
+        sys.exit(1)
+
+    print("✅ All checks passed! No root garbage or unmasked secrets detected.")
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

tests/conftest.py

@@ -0,0 +1,34 @@
+"""Shared test fixtures. No live network: every HTTP call is mocked."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable
+from pathlib import Path
+
+import httpx
+import pytest
+
+FIXTURES = Path(__file__).parent / "fixtures"
+
+
+def load_fixture(name: str) -> dict:
+    """Read one recorded or synthetic fixture JSON file by name."""
+    return json.loads((FIXTURES / name).read_text())
+
+
+def mock_client(
+    base_url: str, handler: Callable[[httpx.Request], httpx.Response]
+) -> httpx.AsyncClient:
+    """An ``httpx.AsyncClient`` whose transport is a synchronous fixture handler."""
+    return httpx.AsyncClient(base_url=base_url, transport=httpx.MockTransport(handler))
+
+
+def json_response(payload: dict) -> httpx.Response:
+    """A 200 JSON response carrying ``payload``."""
+    return httpx.Response(200, json=payload)
+
+
+@pytest.fixture
+def fixture() -> Callable[[str], dict]:
+    return load_fixture

tests/test_adapter_conformance.py

@@ -0,0 +1,164 @@
+"""End-to-end ``SourceAdapter`` conformance for a real in-process MCP session.
+
+Exercises the full path an EG sync run takes — ``McpToolSourceAdapter`` ->
+``session.call_tool`` -> the real FastMCP tool -> the real API-layer
+function (monkeypatched here to a small deterministic fixture instead of a
+live NCBI call) -> back through the generic adapter's pagination, checkpoint
+and provenance machinery — using the SDK's own conformance-check functions
+(:mod:`agent_connector_sdk.testing.source_adapters`), the same functions a
+sync run's own tests would use.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from typing import Any
+
+import pytest
+from agent_connector_sdk.adapters.mcp_tool import McpToolSourceAdapter
+from agent_connector_sdk.manifest.live_contract import validate_preset_tool_contract
+from agent_connector_sdk.manifest.model import SchemaMapping
+from agent_connector_sdk.manifest.presets import ToolPreset
+from agent_connector_sdk.ports.session import TransportEndpoint
+from agent_connector_sdk.testing.results import SessionFactory
+from agent_connector_sdk.testing.source_adapters import (
+    check_capability_descriptor,
+    check_checkpoint_resume,
+    check_idempotent_rerun,
+    check_malformed_input_rejection,
+    check_pagination,
+    check_provenance_completeness,
+    sweep,
+)
+from agent_connector_sdk.transports.mcp import McpTransport
+from fastmcp import FastMCP
+
+from world_reference_mcp.api import ncbi_taxonomy
+from world_reference_mcp.mcp.clients import Clients
+from world_reference_mcp.mcp.tools_taxonomy import register_taxonomy_tools
+
+_TAXA = [
+    {
+        "ncbi_taxon_id": str(9620 + index),
+        "scientific_name": f"Vulpes species-{index}",
+        "common_name": "",
+        "rank": "species",
+        "division": "carnivores",
+        "modification_date": "2021/03/24 00:00",
+    }
+    for index in range(5)
+]
+_SMALL_PAGE_SIZE = 2
+
+
+def _small_preset() -> ToolPreset:
+    return ToolPreset.from_mapping(
+        "taxon_ncbi_small",
+        {
+            "server": "world-reference-mcp",
+            "tool": "taxonomy_ncbi_search",
+            "records_path": "items",
+            "id_field": "ncbi_taxon_id",
+            "updated_field": "modification_date",
+            "doc_type": "taxon",
+            "pagination": "offset",
+            "page_param": "retstart",
+            "page_size_param": "retmax",
+            "page_size": _SMALL_PAGE_SIZE,
+            "arguments": {"term": "Mammalia[Subtree]"},
+            "params_style": "args",
+        },
+    )
+
+
+def _bare_server() -> FastMCP:
+    mcp: FastMCP = FastMCP("world-reference-mcp-test")
+    register_taxonomy_tools(mcp, Clients())
+    return mcp
+
+
+def _session_factory(mcp: FastMCP) -> SessionFactory:
+    @asynccontextmanager
+    async def _open() -> AsyncIterator[Any]:
+        async with McpTransport().session(TransportEndpoint(in_process=mcp)) as session:
+            yield session
+
+    return _open
+
+
+async def _verified_adapter(sessions: SessionFactory) -> McpToolSourceAdapter:
+    """Build an adapter pinned to the tool's CURRENT live fingerprint.
+
+    Mirrors what a real deployment does once, at certification time: read the
+    live schema, pin it, then verify every extraction against that pin.
+    """
+    preset = _small_preset()
+    async with sessions() as probe_session:
+        contract = await probe_session.list_tools()
+    live = validate_preset_tool_contract(
+        contract, tool_name=preset.tool, presets=(preset,), expected_schema_sha256=""
+    )
+    return McpToolSourceAdapter(
+        preset,
+        connector="world-reference-mcp",
+        tool_schema_sha256=live.compatibility_sha256,
+        mapping_reference="manifest:world-reference-mcp#schema_mappings/taxon_ncbi_small",
+        schema_mappings={"taxon_ncbi_small": SchemaMapping(ontology_class="Taxon")},
+    )
+
+
+async def _good_page(
+    client: Any, *, term: str, retstart: int, retmax: int
+) -> dict[str, Any]:
+    return {
+        "items": _TAXA[retstart : retstart + retmax],
+        "offset": retstart,
+        "limit": retmax,
+        "total": len(_TAXA),
+    }
+
+
+async def _malformed_page(
+    client: Any, *, term: str, retstart: int, retmax: int
+) -> dict[str, Any]:
+    return {
+        "items": [{"scientific_name": "no id field"}],
+        "offset": 0,
+        "limit": 1,
+        "total": 1,
+    }
+
+
+@pytest.mark.asyncio
+async def test_good_source_conforms(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(ncbi_taxonomy, "search_taxa", _good_page)
+    sessions = _session_factory(_bare_server())
+    adapter = await _verified_adapter(sessions)
+
+    results = [
+        check_capability_descriptor(adapter),
+        await check_pagination(adapter, sessions),
+        await check_checkpoint_resume(adapter, sessions),
+        await check_idempotent_rerun(adapter, sessions),
+    ]
+    async with sessions() as session:
+        records = (await sweep(adapter, session)).records
+    results.append(check_provenance_completeness(records))
+
+    failed = [result for result in results if not result.passed]
+    assert not failed, failed
+    assert len(records) == len(_TAXA)
+    expected_ids = {taxon["ncbi_taxon_id"] for taxon in _TAXA}
+    assert {record.record_id for record in records} == expected_ids
+
+
+@pytest.mark.asyncio
+async def test_malformed_source_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(ncbi_taxonomy, "search_taxa", _good_page)
+    sessions = _session_factory(_bare_server())
+    adapter = await _verified_adapter(sessions)
+
+    monkeypatch.setattr(ncbi_taxonomy, "search_taxa", _malformed_page)
+    result = await check_malformed_input_rejection(adapter, sessions)
+    assert result.passed, result.detail

tests/test_api_fdc.py

@@ -0,0 +1,29 @@
+"""USDA FoodData Central API-layer test against a synthetic fixture."""
+
+from __future__ import annotations
+
+import httpx
+import pytest
+
+from tests.conftest import json_response, load_fixture, mock_client
+from world_reference_mcp.api.fdc import search_foods
+
+
+@pytest.mark.asyncio
+async def test_search_foods_maps_nutrients() -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert request.url.path == "/fdc/v1/foods/search"
+        assert request.url.params["api_key"] == "test-key"
+        return json_response(load_fixture("fdc_food_search.json"))
+
+    async with mock_client("https://api.nal.usda.gov/fdc/v1", handler) as client:
+        page = await search_foods(
+            client, api_key="test-key", query="fruit", page_number=1, page_size=50
+        )
+    assert page["total"] == 2
+    apple = page["items"][0]
+    assert apple["fdc_id"] == "1750340"
+    assert apple["description"] == "Apples, raw, with skin"
+    energy = next(n for n in apple["nutrients"] if n["nutrient_name"] == "Energy")
+    assert energy["amount"] == 52.0
+    assert energy["unit_name"] == "KCAL"

tests/test_api_gbif.py

@@ -0,0 +1,45 @@
+"""GBIF API-layer tests against recorded species/occurrence search fixtures."""
+
+from __future__ import annotations
+
+import httpx
+import pytest
+
+from tests.conftest import json_response, load_fixture, mock_client
+from world_reference_mcp.api.gbif import search_occurrences, search_species
+
+
+@pytest.mark.asyncio
+async def test_search_species_maps_fields() -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert request.url.path == "/v1/species/search"
+        assert request.url.params["rank"] == "SPECIES"
+        return json_response(load_fixture("gbif_species_search.json"))
+
+    async with mock_client("https://api.gbif.org/v1", handler) as client:
+        page = await search_species(
+            client, query="Vulpes vulpes", rank="species", offset=0, limit=2
+        )
+    assert page["offset"] == 0
+    (item, _) = page["items"]
+    assert item["gbif_taxon_key"] == "304190692"
+    assert item["scientific_name"] == "Vulpes vulpes"
+    assert item["rank"] == "SPECIES"
+    assert item["kingdom"] == "Animalia"
+
+
+@pytest.mark.asyncio
+async def test_search_occurrences_maps_fields() -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert request.url.path == "/v1/occurrence/search"
+        return json_response(load_fixture("gbif_occurrence_search.json"))
+
+    async with mock_client("https://api.gbif.org/v1", handler) as client:
+        page = await search_occurrences(
+            client, scientific_name="Vulpes vulpes", offset=0, limit=1
+        )
+    (item,) = page["items"]
+    assert item["occurrence_id"] == "5936472487"
+    assert item["gbif_taxon_key"] == "5219243"
+    assert item["country_code"] == "FI"
+    assert item["decimal_latitude"] == pytest.approx(60.319882)