webctx 0.2.1 → 0.2.3

package/internal/app/structured_source.go

@@ -0,0 +1,249 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+)
+
+const maxDirectSourceBytes = 1024 * 1024
+
+var (
+	jsonLDPattern   = regexp.MustCompile(`(?is)<script\b[^>]*type\s*=\s*["']application/ld\+json["'][^>]*>(.*?)</script\s*>`)
+	nextDataPattern = regexp.MustCompile(`(?is)<script\b[^>]*id\s*=\s*["']__NEXT_DATA__["'][^>]*>(.*?)</script\s*>`)
+)
+
+// These URLs are already structured source documents. Never scrape a raw
+// OpenAPI schema through HTML rendering when we can retrieve its actual bytes.
+func directSourceLanguage(rawURL string) string {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return ""
+	}
+	switch strings.ToLower(path.Ext(u.Path)) {
+	case ".json", ".jsonl":
+		return "json"
+	case ".yaml", ".yml":
+		return "yaml"
+	case ".toml":
+		return "toml"
+	case ".xml":
+		return "xml"
+	case ".graphql", ".gql":
+		return "graphql"
+	case ".txt":
+		return "text"
+	default:
+		return ""
+	}
+}
+
+// A bounded, redirect-checked direct read prevents accidental traversal from
+// an accepted public source to an internal address.
+func fetchPublicText(ctx context.Context, rawURL string, maxBytes int64) (string, error) {
+	if err := validateSourceURL(rawURL); err != nil {
+		return "", err
+	}
+	client := *http.DefaultClient
+	oldCheck := client.CheckRedirect
+	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		if err := validateSourceURL(req.URL.String()); err != nil {
+			return err
+		}
+		if oldCheck != nil {
+			return oldCheck(req, via)
+		}
+		if len(via) >= 10 {
+			return fmt.Errorf("too many redirects")
+		}
+		return nil
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("Accept", "application/json, text/markdown, text/plain, text/html, application/yaml, */*;q=0.5")
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return "", fmt.Errorf("source HTTP %d", resp.StatusCode)
+	}
+	if resp.ContentLength > maxBytes {
+		return "", fmt.Errorf("source is larger than the supported %d bytes", maxBytes)
+	}
+	body, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1))
+	if err != nil {
+		return "", err
+	}
+	if int64(len(body)) > maxBytes {
+		return "", fmt.Errorf("source exceeded the supported %d bytes", maxBytes)
+	}
+	return string(body), nil
+}
+
+func readDirectSource(rawURL string) (string, bool) {
+	language := directSourceLanguage(rawURL)
+	if language == "" {
+		return "", false
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
+	defer cancel()
+	body, err := fetchPublicText(ctx, rawURL, maxDirectSourceBytes)
+	if err != nil || strings.TrimSpace(body) == "" {
+		return "", false
+	}
+	if strings.Contains(strings.ToLower(body[:min(len(body), 256)]), "<html") {
+		return "", false
+	}
+	if language == "json" && !json.Valid([]byte(body)) { // don't label HTML/garbage as JSON
+		return "", false
+	}
+	return fmt.Sprintf("**URL:** %s\n\n```%s\n%s\n```", rawURL, language, strings.TrimSpace(body)), true
+}
+
+// Extract only public machine-readable metadata actually embedded in the
+// original HTML. Never execute arbitrary scripts or speculate about hidden APIs.
+func embeddedStructuredContext(rawURL, question string) string {
+	if question == "" || directSourceLanguage(rawURL) != "" {
+		return ""
+	}
+	if u, err := url.Parse(rawURL); err == nil && strings.EqualFold(u.Hostname(), "github.com") {
+		return "" // GitHub's native APIs are more faithful than page hydration data.
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)
+	defer cancel()
+	html, err := fetchPublicText(ctx, rawURL, maxDirectSourceBytes)
+	if err != nil || !strings.Contains(strings.ToLower(html), "<script") {
+		return ""
+	}
+	var results []string
+	for _, pattern := range []*regexp.Regexp{jsonLDPattern, nextDataPattern} {
+		for _, match := range pattern.FindAllStringSubmatch(html, 3) {
+			var raw any
+			if json.Unmarshal([]byte(strings.TrimSpace(match[1])), &raw) != nil {
+				continue
+			}
+			selection := selectedJSONFields(raw, question, 2500)
+			if selection != "" {
+				results = append(results, selection)
+			}
+		}
+	}
+	if len(results) == 0 {
+		return ""
+	}
+	return "### Embedded source metadata\n\nStructured JSON embedded in the source page (not interpreted as an API response):\n\n" + strings.Join(results, "\n\n")
+}
+
+// selectedJSONFields preserves exact JSON values and JSON Pointers rather
+// than returning syntactically broken chunks of a large schema or payload.
+func selectedJSONFields(raw any, question string, budget int) string {
+	type field struct {
+		path  string
+		value any
+		score int
+	}
+	terms := queryTerms(question)
+	fields := make([]field, 0, 32)
+	var walk func(any, string, int)
+	walk = func(value any, parent string, depth int) {
+		if depth > 9 || len(fields) >= 500 {
+			return
+		}
+		switch v := value.(type) {
+		case map[string]any:
+			for key, child := range v {
+				pointer := parent + "/" + strings.ReplaceAll(strings.ReplaceAll(key, "~", "~0"), "/", "~1")
+				score := termScore(key, terms)*4 + termScore(pointer, terms)
+				if score > 0 {
+					fields = append(fields, field{path: pointer, value: child, score: score})
+				}
+				walk(child, pointer, depth+1)
+			}
+		case []any:
+			for i, child := range v {
+				if i >= 20 { // never flatten unbounded payload collections
+					break
+				}
+				walk(child, fmt.Sprintf("%s/%d", parent, i), depth+1)
+			}
+		}
+	}
+	walk(raw, "", 0)
+	sort.SliceStable(fields, func(i, j int) bool {
+		if fields[i].score == fields[j].score {
+			return fields[i].path < fields[j].path
+		}
+		return fields[i].score > fields[j].score
+	})
+	var out []string
+	for _, f := range fields {
+		value, err := json.MarshalIndent(f.value, "", "  ")
+		if err != nil || len(value) > budget/2 || len(value) < 2 {
+			continue
+		}
+		item := fmt.Sprintf("**JSON Pointer:** `%s`\n\n```json\n%s\n```", f.path, value)
+		if len(strings.Join(out, "\n\n"))+len(item) > budget {
+			break
+		}
+		out = append(out, item)
+		if len(out) >= 5 {
+			break
+		}
+	}
+	if len(out) > 0 {
+		return strings.Join(out, "\n\n") + "\n\n*[Selected JSON fields; full document at source URL.]*"
+	}
+	pretty, err := json.MarshalIndent(raw, "", "  ")
+	if err != nil {
+		return ""
+	}
+	if len(pretty)+20 <= budget {
+		return "```json\n" + string(pretty) + "\n```"
+	}
+	// No relevant key matched: expose the shape without claiming completeness.
+	if obj, ok := raw.(map[string]any); ok {
+		keys := make([]string, 0, len(obj))
+		for key := range obj {
+			keys = append(keys, key)
+		}
+		sort.Strings(keys)
+		if len(keys) > 25 {
+			keys = keys[:25]
+		}
+		return "**Top-level JSON keys:** " + strings.Join(keys, ", ") + " (full document at source URL)"
+	}
+	return "*[Structured document is larger than the inline budget; inspect original source.]*"
+}
+
+func focusedRawJSON(markdown, question string, budget int) string {
+	begin := strings.Index(markdown, "```json\n")
+	if begin < 0 {
+		return ""
+	}
+	jsonStart := begin + len("```json\n")
+	end := strings.LastIndex(markdown, "\n```")
+	if end <= jsonStart {
+		return ""
+	}
+	var source any
+	if json.Unmarshal([]byte(markdown[jsonStart:end]), &source) != nil {
+		return ""
+	}
+	selected := selectedJSONFields(source, question, budget-250)
+	if selected == "" {
+		return ""
+	}
+	return strings.TrimSpace(markdown[:begin]) + "\n\n**Focused JSON evidence:**\n\n" + selected
+}

package/internal/app/structured_source_test.go

@@ -0,0 +1,70 @@
+package app
+
+import (
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func TestDirectOpenAPIJSONBypassesScraping(t *testing.T) {
+	old := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = old })
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		if req.URL.Host != "docs.example.com" || req.URL.Path != "/openapi.json" || req.Method != http.MethodGet {
+			t.Fatalf("unexpected direct source request: %s", req.URL)
+		}
+		return testHTTPResponse(req, http.StatusOK, `{"openapi":"3.1.0","paths":{"/v2/scrape":{}}}`, nil), nil
+	})}
+	got, err := ReadLink("https://docs.example.com/openapi.json")
+	if err != nil || !strings.Contains(got, `"openapi":"3.1.0"`) || !strings.Contains(got, "```json") {
+		t.Fatalf("did not fetch raw OpenAPI: %q, %v", got, err)
+	}
+}
+
+func TestDirectSourceRejectsPrivateRedirects(t *testing.T) {
+	old := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = old })
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		if req.URL.Host != "docs.example.com" {
+			t.Fatalf("followed unsafe redirect to %s", req.URL)
+		}
+		return testHTTPResponse(req, http.StatusFound, "", map[string]string{"Location": "http://127.0.0.1/private"}), nil
+	})}
+	if _, err := fetchPublicText(t.Context(), "https://docs.example.com/openapi.json", 100); err == nil {
+		t.Fatal("should reject public-to-private redirects")
+	}
+}
+
+func TestAlexandriaDiscoveryFallsBackWhenNotEnabled(t *testing.T) {
+	old := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = old })
+	var calls int
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		body, _ := io.ReadAll(req.Body)
+		calls++
+		if calls == 1 {
+			if !strings.Contains(string(body), `"domainTools":true`) {
+				t.Fatalf("Alexandria discovery not enabled: %s", body)
+			}
+			return testHTTPResponse(req, http.StatusForbidden, `{"error":"Alexandria unavailable"}`, nil), nil
+		}
+		if !strings.Contains(string(body), `"domainTools":false`) {
+			t.Fatalf("fallback did not disable domain matching: %s", body)
+		}
+		return testHTTPResponse(req, http.StatusOK, `{"success":true,"data":{"markdown":"# Good source\n\nAuthoritative page content."}}`, nil), nil
+	})}
+	result, err := firecrawlScrape("https://docs.example.com/reference", "fake-test-key", "")
+	if err != nil || calls != 2 || !strings.Contains(result.Markdown, "Good source") {
+		t.Fatalf("fallback broke regular scraping: %#v, %v, calls=%d", result, err, calls)
+	}
+}
+
+func TestAlexandriaToolDescriptionIsDiscoveryOnly(t *testing.T) {
+	got := formatAlexandriaTools([]any{
+		map[string]any{"provider": "particle", "capability": "podcasts/episodes/search", "description": "Find episodes"},
+	})
+	if !strings.Contains(got, "particle/podcasts/episodes/search") || !strings.Contains(got, "not executed") {
+		t.Fatalf("bad tool discovery metadata: %s", got)
+	}
+}

package/internal/app/tools.go

@@ -51,6 +51,11 @@ func Search(params SearchParams) (string, error) {
 	allExcludedDomains := append(append([]string{}, defaultExcludedDomains...), params.ExcludeDomains...)
 	truncatedKeyword := truncateWords(params.IncludeKeyword, 5)
 
+	// Free Alexandria tool discovery proceeds alongside normal web search.
+	// It never executes a paid tool or replaces source-grounded web results.
+	alexandriaDone := make(chan string, 1)
+	go func() { alexandriaDone <- discoverAlexandria(params.Query) }()
+
 	type namedSearch struct {
 		name string
 		fn   func(context.Context) (SearchResult, error)
@@ -118,6 +123,15 @@ func Search(params SearchParams) (string, error) {
 	}
 
 	parts := []string{fmt.Sprintf("Total Results: %d\n", len(ranked))}
+	// The ranked index remains available, but agents also get useful source
+	// context without having to issue several follow-up read-link calls.
+	if context := enrichSearch(params.Query, ranked); context != "" {
+		parts = append(parts, context, "")
+	}
+	if tools := <-alexandriaDone; tools != "" {
+		parts = append(parts, "", "## Related data providers", tools, "")
+	}
+	parts = append(parts, "## Search results", "")
 	for _, doc := range ranked {
 		title := decodeHTML(doc.Title)
 		overview := decodeHTML(doc.Overview)
@@ -131,6 +145,9 @@ func Search(params SearchParams) (string, error) {
 }
 
 func ReadLink(rawURL string) (string, error) {
+	if err := validateSourceURL(rawURL); err != nil {
+		return "", err
+	}
 	native := readGitHubNative(rawURL)
 	switch native.Outcome {
 	case GitHubNativeSuccess:
@@ -150,6 +167,10 @@ func ReadLink(rawURL string) (string, error) {
 		return "", native.Err
 	}
 
+	if direct, ok := readDirectSource(rawURL); ok {
+		return direct, nil
+	}
+
 	if ok, _ := checkMarkdownAvailable(rawURL); ok {
 		result, err := fetchMarkdownContent(rawURL)
 		if err == nil {
@@ -193,55 +214,38 @@ func scrapeLinkWithFirecrawl(rawURL string) (string, string, error) {
 	if apiKey == "" {
 		return "", "", missingCredentialsError("Error reading web page", []string{"FIRECRAWL_API_KEY"}, "for non-.md URLs")
 	}
-
-	requestBody := map[string]any{
-		"url":                 rawURL,
-		"formats":             []string{"markdown"},
-		"onlyMainContent":     true,
-		"skipTlsVerification": true,
-		"blockAds":            true,
-		"removeBase64Images":  true,
-		"maxAge":              600000,
-		"excludeTags":         []string{"script", "style", "meta", "noscript", "svg", "img", "nav", "footer", "header", "aside", ".advertisement", "#ad"},
-	}
-	if strings.HasSuffix(strings.ToLower(rawURL), ".pdf") {
-		requestBody["parsers"] = []string{"pdf"}
-	}
-
-	data, err := getFirecrawlQueue().enqueue(rawURL, func() (map[string]any, error) {
-		ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
-		defer cancel()
-		body, err := doJSONRequest(ctx, http.MethodPost, "https://api.firecrawl.dev/v2/scrape", map[string]string{
-			"Authorization": "Bearer " + apiKey,
-			"Content-Type":  "application/json",
-		}, requestBody)
-		if err != nil {
-			return nil, err
-		}
-		var parsed map[string]any
-		if err := json.Unmarshal(body, &parsed); err != nil {
-			return nil, err
+	// Firecrawl's default auto proxy already tries basic then enhanced when the
+	// transport fails. A second explicit enhanced attempt is only useful when a
+	// scrape responds successfully but returns a challenge/empty document.
+	first, err := firecrawlScrape(rawURL, apiKey, "")
+	if err == nil && usableScrapedMarkdown(first.Markdown) {
+		return first.Title, withSourceTools(first.Markdown, first.ToolSummary), nil
+	}
+	lastError := err
+	if err == nil {
+		lastError = fmt.Errorf("Firecrawl returned empty or blocked page content")
+		second, enhancedErr := firecrawlScrape(rawURL, apiKey, "enhanced")
+		if enhancedErr == nil && usableScrapedMarkdown(second.Markdown) {
+			return second.Title, withSourceTools(second.Markdown, second.ToolSummary), nil
 		}
-		if success, _ := parsed["success"].(bool); !success {
-			return nil, fmt.Errorf("Scraping failed for %s: %v", rawURL, parsed["error"])
+		if enhancedErr != nil {
+			lastError = enhancedErr
 		}
-		return parsed, nil
-	})
-	if err != nil {
-		return "", "", fmt.Errorf("Error reading web page: %v", err)
 	}
-
-	dataMap, _ := data["data"].(map[string]any)
-	metadata, _ := dataMap["metadata"].(map[string]any)
-	title, _ := metadata["title"].(string)
-	markdown, _ := dataMap["markdown"].(string)
-	if markdown == "" {
-		markdown = "No content extracted"
+	title, markdown, browserErr := firecrawlBrowserFallback(rawURL, apiKey)
+	if browserErr == nil && usableScrapedMarkdown(markdown) {
+		return title, markdown, nil
+	}
+	if browserErr == nil {
+		browserErr = fmt.Errorf("browser produced no usable content")
 	}
-	return title, markdown, nil
+	return "", "", fmt.Errorf("Error reading web page: scrape: %v; browser fallback: %w", lastError, browserErr)
 }
 
 func MapSite(rawURL string) (string, error) {
+	if err := validateSourceURL(rawURL); err != nil {
+		return "", err
+	}
 	apiKey := strings.TrimSpace(os.Getenv("FIRECRAWL_API_KEY"))
 	if apiKey == "" {
 		return "", missingCredentialsError("Error mapping website", []string{"FIRECRAWL_API_KEY"}, "")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webctx",
-  "version": "0.2.1",
+  "version": "0.2.3",
   "description": "Pure Go web search and browsing CLI using Brave, Tavily, Exa, and Firecrawl",
   "license": "MIT",
   "author": "amxv",
@@ -24,6 +24,8 @@
     "cmd",
     "internal",
     "go.mod",
+    "go.sum",
+    "pkg",
     "README.md",
     "docs",
     "LICENSE",

package/pkg/origo/server.go

@@ -0,0 +1,86 @@
+// Package origo provides the independently deployed MCP transport for the
+// shared WebCTX retrieval engine. Search is intentionally not exposed.
+package origo
+
+import (
+	"context"
+	"crypto/sha256"
+	"crypto/subtle"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/amxv/webctx/pkg/retrieval"
+	"github.com/modelcontextprotocol/go-sdk/mcp"
+)
+
+type linkInput struct {
+	URL string `json:"url" jsonschema:"Absolute HTTP(S) URL to read as source-grounded Markdown"`
+}
+
+type readLinkInput struct {
+	URL      string `json:"url" jsonschema:"Absolute HTTP(S) URL to read as source-grounded Markdown"`
+	Question string `json:"question,omitempty" jsonschema:"Optional question or task. When set, automatically follows relevant source links and returns focused, cited context."`
+}
+
+func makeServer() *mcp.Server {
+	s := mcp.NewServer(&mcp.Implementation{Name: "origo", Version: "0.1.0"}, nil)
+	mcp.AddTool(s, &mcp.Tool{
+		Name:        "read_link",
+		Title:       "Read link",
+		Description: "Read a URL as source-grounded Markdown. Optionally provide a question to automatically explore related docs and return relevant API contracts, code examples and precise source URLs. No search query required; no paid Alexandria tool execution.",
+		Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true},
+	}, func(_ context.Context, _ *mcp.CallToolRequest, input readLinkInput) (*mcp.CallToolResult, any, error) {
+		result, err := retrieval.ReadLinkFocused(input.URL, input.Question)
+		if err != nil {
+			return nil, nil, err
+		}
+		return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: result}}}, nil, nil
+	})
+	mcp.AddTool(s, &mcp.Tool{
+		Name:        "map_site",
+		Title:       "Map site",
+		Description: "Discover URLs on a site using Firecrawl's sitemap-based map API. Does not perform web search.",
+		Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true},
+	}, func(_ context.Context, _ *mcp.CallToolRequest, input linkInput) (*mcp.CallToolResult, any, error) {
+		result, err := retrieval.MapSite(input.URL)
+		if err != nil {
+			return nil, nil, err
+		}
+		return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: result}}}, nil, nil
+	})
+	return s
+}
+
+var transport = mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server {
+	return makeServer()
+}, &mcp.StreamableHTTPOptions{Stateless: true, JSONResponse: true})
+
+func Handler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Cache-Control", "no-store")
+		w.Header().Set("Referrer-Policy", "no-referrer")
+		w.Header().Set("X-Content-Type-Options", "nosniff")
+		w.Header().Set("X-Robots-Tag", "noindex")
+		secret := strings.TrimSpace(os.Getenv("ORIGO_API_KEY"))
+		if secret == "" {
+			http.Error(w, "MCP endpoint not configured", http.StatusServiceUnavailable)
+			return
+		}
+		keys := r.URL.Query()["key"]
+		if len(keys) != 1 || !validKey(secret, keys[0]) {
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return
+		}
+		transport.ServeHTTP(w, r)
+	})
+}
+
+func validKey(expected, supplied string) bool {
+	if supplied == "" {
+		return false
+	}
+	want := sha256.Sum256([]byte(expected))
+	got := sha256.Sum256([]byte(supplied))
+	return subtle.ConstantTimeCompare(want[:], got[:]) == 1
+}

package/pkg/origo/server_test.go

@@ -0,0 +1,78 @@
+package origo
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func request(t *testing.T, key, body string) *httptest.ResponseRecorder {
+	t.Helper()
+	path := "/mcp"
+	if key != "" {
+		path += "?key=" + key
+	}
+	req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(body))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "application/json, text/event-stream")
+	req.Header.Set("Mcp-Protocol-Version", "2025-11-25")
+	response := httptest.NewRecorder()
+	Handler().ServeHTTP(response, req)
+	return response
+}
+
+func TestServerRequiresConfiguredKey(t *testing.T) {
+	t.Setenv("ORIGO_API_KEY", "")
+	response := request(t, "somekey", `{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}`)
+	if response.Code != http.StatusServiceUnavailable {
+		t.Fatalf("without a configured key: expected 503, got %d", response.Code)
+	}
+}
+
+func TestServerAuthentication(t *testing.T) {
+	t.Setenv("ORIGO_API_KEY", "secret")
+	for _, key := range []string{"", "wrong"} {
+		response := request(t, key, `{}`)
+		if response.Code != http.StatusUnauthorized {
+			t.Fatalf("key %q: expected 401, got %d", key, response.Code)
+		}
+		if response.Header().Get("Cache-Control") != "no-store" {
+			t.Fatalf("no-store header missing")
+		}
+	}
+	response := request(t, "secret", `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"test","version":"1"}}}`)
+	if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), `"name":"origo"`) {
+		t.Fatalf("initialize failed: %d %s", response.Code, response.Body.String())
+	}
+}
+
+func TestExactlyTwoToolsAndNoSearch(t *testing.T) {
+	t.Setenv("ORIGO_API_KEY", "secret")
+	response := request(t, "secret", `{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}`)
+	if response.Code != http.StatusOK {
+		t.Fatalf("tools/list failed: %d %s", response.Code, response.Body.String())
+	}
+	text := response.Body.String()
+	if !strings.Contains(text, `"name":"read_link"`) || !strings.Contains(text, `"name":"map_site"`) || strings.Contains(text, `"name":"search"`) {
+		t.Fatalf("unexpected tools: %s", text)
+	}
+	if count := strings.Count(text, `"name":`); count != 2 {
+		t.Fatalf("expected exactly 2 tools, got %d: %s", count, text)
+	}
+	if count := strings.Count(text, `"readOnlyHint":true`); count != 2 {
+		t.Fatalf("expected both tools to declare read-only annotations, got %d: %s", count, text)
+	}
+}
+
+func TestQuestionIsOptionalOnReadLinkOnly(t *testing.T) {
+	t.Setenv("ORIGO_API_KEY", "secret")
+	response := request(t, "secret", `{"jsonrpc":"2.0","id":4,"method":"tools/list","params":{}}`)
+	if response.Code != http.StatusOK {
+		t.Fatalf("list tools: %s", response.Body.String())
+	}
+	text := response.Body.String()
+	if !strings.Contains(text, `"question"`) || !strings.Contains(text, `"required":["url"]`) {
+		t.Fatalf("question should be optional; URL remains required: %s", text)
+	}
+}

package/pkg/retrieval/retrieval.go

@@ -0,0 +1,17 @@
+// Package retrieval exposes WebCTX's shared retrieval engine to non-CLI
+// transports. The CLI remains backward-compatible while Origo MCP can call
+// these functions without importing Go internal packages from Vercel's
+// generated handler package.
+package retrieval
+
+import "github.com/amxv/webctx/internal/app"
+
+func ReadLink(url string) (string, error) { return app.ReadLink(url) }
+
+// ReadLinkFocused follows related documentation when the caller has a question.
+// A URL-only call retains the original complete-document behavior.
+func ReadLinkFocused(url, question string) (string, error) {
+	return app.ReadLinkFocused(url, question)
+}
+
+func MapSite(url string) (string, error) { return app.MapSite(url) }