webctx 0.2.1 → 0.2.3

package/internal/app/research_excerpt_test.go

@@ -0,0 +1,47 @@
+package app
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestFocusedExcerptPreservesWholeCurlExamples(t *testing.T) {
+	code := backtickFence + "bash\ncurl -X POST https://api.example.com/v2/scrape \\\n" +
+		" -H 'Content-Type: application/json' \\\n" +
+		" -d '{\"alexandria\":{\"capability\":\"find-tools\"}}'\n" + backtickFence
+	doc := "# Documentation\n\n" + strings.Repeat("Background prose unrelated to endpoints.\n", 200) +
+		"\n## Inspect tools over REST\n" + strings.Repeat("Introductory filler.\n", 70) +
+		"\n" + code + "\n" + strings.Repeat("Appendix filler.\n", 200)
+	got := focusExcerpt(doc, "How do I inspect Alexandria tools with a REST request example?", 1600)
+	if !strings.Contains(got, code) {
+		t.Fatalf("the complete REST example must survive extraction:\n%s", got)
+	}
+	if strings.Count(got, backtickFence) != 2 {
+		t.Fatalf("expected exactly one balanced code fence: %s", got)
+	}
+	if len(got) > 1800 {
+		t.Fatalf("excerpt exceeded budget: %d", len(got))
+	}
+}
+
+func TestOversizedCodeBlocksAreNeverPartiallyReturned(t *testing.T) {
+	huge := backtickFence + "json\n{\"payload\":\"" + strings.Repeat("a", 3000) + "\"}\n" + backtickFence
+	doc := "# API\n\n" + huge + "\n\n" + strings.Repeat("Further explanation.\n", 200)
+	got := atomicFocusedSection(doc, "What is the API payload?", 700)
+	if strings.Contains(got, "\"payload\"") || strings.Contains(got, backtickFence) {
+		t.Fatalf("oversized example should be omitted, not truncated:\n%s", got)
+	}
+	if !strings.Contains(got, "omitted") && !strings.Contains(got, "source URL") {
+		t.Fatalf("omission should be explicitly marked: %s", got)
+	}
+}
+
+func TestPlainCodeFenceOpeningNotTreatedAsClosing(t *testing.T) {
+	doc := "# API\n" + strings.Repeat("Preliminary text.\n", 80) + "\n" +
+		backtickFence + "\nPOST /v2/scrape\n{}\n" + backtickFence + "\n" +
+		strings.Repeat("End notes.\n", 150)
+	got := atomicFocusedSection(doc, "scrape request", 650)
+	if !strings.Contains(got, backtickFence+"\nPOST /v2/scrape\n{}\n"+backtickFence) {
+		t.Fatalf("plain opening fence was not handled atomically: %s", got)
+	}
+}

package/internal/app/research_openapi.go

@@ -0,0 +1,289 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"sort"
+	"strings"
+	"time"
+)
+
+// API contracts are public documentation, not calls to the documented API.
+// When a question requires exact REST payloads, inspecting the site's linked
+// OpenAPI spec is more faithful than extrapolating HTTP requests from SDK code.
+type openAPIContext struct {
+	URL     string
+	Content string
+}
+
+func asksForAPIContracts(question string) bool {
+	question = strings.ToLower(question)
+	return strings.Contains(question, "rest") || strings.Contains(question, "openapi") ||
+		(strings.Contains(question, "api") && (strings.Contains(question, "endpoint") || strings.Contains(question, "request") || strings.Contains(question, "schema") || strings.Contains(question, "contract"))) ||
+		(strings.Contains(question, "exact") && (strings.Contains(question, "curl") || strings.Contains(question, "request example")))
+}
+
+func discoverOpenAPIContract(rawURL, question string) openAPIContext {
+	if !asksForAPIContracts(question) || directSourceLanguage(rawURL) != "" {
+		return openAPIContext{}
+	}
+	u, err := url.Parse(rawURL)
+	if err != nil || validateSourceURL(rawURL) != nil || strings.EqualFold(u.Hostname(), "github.com") {
+		return openAPIContext{}
+	}
+	indexURL := u.Scheme + "://" + u.Host + "/llms.txt"
+	ctx, cancel := context.WithTimeout(context.Background(), 18*time.Second)
+	defer cancel()
+	index, err := fetchPublicText(ctx, indexURL, 300000)
+	if err != nil {
+		return openAPIContext{}
+	}
+	var specURLs []sourceCandidate
+	seen := make(map[string]bool)
+	for _, match := range mdLinkPattern.FindAllStringSubmatch(index, 300) {
+		dest, err := url.Parse(strings.TrimSpace(match[2]))
+		if err != nil {
+			continue
+		}
+		resolved := u.ResolveReference(dest)
+		name := strings.ToLower(resolved.Path)
+		if !strings.EqualFold(resolved.Hostname(), u.Hostname()) || resolved.Port() != u.Port() ||
+			!strings.HasSuffix(name, ".json") || !strings.Contains(name, "openapi") ||
+			validateSourceURL(resolved.String()) != nil || sensitiveURL(resolved) {
+			continue
+		}
+		sourceURL := canonicalSourceURL(resolved)
+		if seen[sourceURL] {
+			continue
+		}
+		seen[sourceURL] = true
+		score := 0
+		if strings.Contains(name, "v2-openapi") {
+			score += 10
+		}
+		if strings.HasPrefix(name, "/api-reference/") {
+			score += 5
+		}
+		if name == "/openapi.json" {
+			score += 5
+		}
+		score -= len(name) / 20 // Prefer the canonical path over translations.
+		specURLs = append(specURLs, sourceCandidate{url: sourceURL, score: score})
+	}
+	sort.SliceStable(specURLs, func(i, j int) bool { return specURLs[i].score > specURLs[j].score })
+	for i, item := range specURLs {
+		if i >= 2 { // Don't recursively fetch every language/version.
+			break
+		}
+		body, err := fetchPublicText(ctx, item.url, maxDirectSourceBytes)
+		if err != nil {
+			continue
+		}
+		if content := summarizeOpenAPIContract(body, question); content != "" {
+			return openAPIContext{URL: item.url, Content: content}
+		}
+	}
+	return openAPIContext{}
+}
+
+type openAPIOperation struct {
+	Method   string
+	Path     string
+	Summary  string
+	Score    int
+	Request  map[string]any
+	Response map[string]any
+}
+
+func summarizeOpenAPIContract(source, question string) string {
+	var spec struct {
+		OpenAPI string `json:"openapi"`
+		Swagger string `json:"swagger"`
+		Servers []struct {
+			URL string `json:"url"`
+		} `json:"servers"`
+		Paths      map[string]map[string]any `json:"paths"`
+		Components struct {
+			Schemas map[string]map[string]any `json:"schemas"`
+		} `json:"components"`
+	}
+	if json.Unmarshal([]byte(source), &spec) != nil || (spec.OpenAPI == "" && spec.Swagger == "") || len(spec.Paths) == 0 {
+		return ""
+	}
+	terms := queryTerms(question)
+	terms = append(terms, relatedAPITerms(question)...)
+	var operations []openAPIOperation
+	for path, methods := range spec.Paths {
+		// A broad question about Alexandria must not bring in unrelated API
+		// routes such as research papers or team activity merely because the
+		// path contains "search" or a summary happens to contain "API".
+		if strings.Contains(strings.ToLower(question), "alexandria") &&
+			strings.Contains(strings.Trim(path, "/"), "/") &&
+			!strings.Contains(strings.ToLower(path), "alexandria") {
+			continue
+		}
+		for method, rawDetail := range methods {
+			switch strings.ToUpper(method) {
+			case "GET", "POST", "PUT", "PATCH", "DELETE":
+			default:
+				continue
+			}
+			detail := mapValue(rawDetail)
+			summary := stringValue(detail["summary"])
+			req := mapValue(detail["requestBody"])
+			resp := mapValue(detail["responses"])
+			score := termScore(path+" "+summary, terms)*4 + termScore(stringValue(detail["description"]), terms)
+			if strings.Contains(strings.ToLower(question), "alexandria") &&
+				termScore(strings.Trim(path, "/"), []string{"search", "scrape", "find", "tools", "alexandria"}) == 0 {
+				continue
+			}
+			if examples := requestExamples(req); len(examples) > 0 {
+				for name, example := range examples {
+					score += termScore(name+" "+stringValue(example["summary"]), terms) * 3
+				}
+			}
+			if score > 0 {
+				operations = append(operations, openAPIOperation{
+					Method: strings.ToUpper(method), Path: path, Summary: summary, Score: score, Request: req, Response: resp,
+				})
+			}
+		}
+	}
+	sort.SliceStable(operations, func(i, j int) bool {
+		if operations[i].Score == operations[j].Score {
+			return operations[i].Path < operations[j].Path
+		}
+		return operations[i].Score > operations[j].Score
+	})
+	if len(operations) == 0 {
+		return ""
+	}
+	apiRoot := ""
+	for _, server := range spec.Servers {
+		if validateSourceURL(server.URL) == nil {
+			apiRoot = strings.TrimRight(server.URL, "/")
+			break
+		}
+	}
+	parts := []string{"OpenAPI-defined HTTP operations and example request bodies (not executed):"}
+	for i, operation := range operations {
+		if i >= 4 {
+			break
+		}
+		endpoint := operation.Path
+		if apiRoot != "" {
+			endpoint = apiRoot + operation.Path
+		}
+		parts = append(parts, "", fmt.Sprintf("### %s %s", operation.Method, endpoint))
+		if operation.Summary != "" {
+			parts = append(parts, operation.Summary)
+		}
+		if examples := requestExamples(operation.Request); len(examples) > 0 {
+			names := make([]string, 0, len(examples))
+			for name := range examples {
+				names = append(names, name)
+			}
+			sort.SliceStable(names, func(i, j int) bool {
+				a := termScore(names[i]+" "+stringValue(examples[names[i]]["summary"]), terms)
+				b := termScore(names[j]+" "+stringValue(examples[names[j]]["summary"]), terms)
+				if a == b {
+					return names[i] < names[j]
+				}
+				return a > b
+			})
+			emitted := 0
+			for _, name := range names {
+				value := examples[name]["value"]
+				body, err := json.MarshalIndent(value, "", "  ")
+				if err != nil || len(body) > 9000 || len(body) == 0 {
+					continue
+				}
+				if strings.Contains(strings.ToLower(question), "alexandria") &&
+					!strings.Contains(strings.ToLower(name+" "+string(body)), "alexandria") &&
+					!strings.Contains(strings.ToLower(name+" "+string(body)), "findtools") &&
+					!strings.Contains(strings.ToLower(name+" "+string(body)), "find-tools") {
+					continue
+				}
+				if emitted >= 2 {
+					break
+				}
+				emitted++
+				parts = append(parts, "", "**Published request example:** "+stringValue(examples[name]["summary"]))
+				if apiRoot != "" && !strings.Contains(endpoint, "{") {
+					// Keep the published JSON body verbatim; the API key must expand in the shell.
+					parts = append(parts, fmt.Sprintf("```bash\ncurl -X %s %q \\\n  -H \"Authorization: Bearer $API_KEY\" \\\n  -H \"Content-Type: application/json\" \\\n  --data-binary @- <<'JSON_REQUEST'\n%s\nJSON_REQUEST\n```", operation.Method, endpoint, string(body)))
+				} else {
+					parts = append(parts, "```json\n"+string(body)+"\n```")
+				}
+			}
+		}
+	}
+	for _, name := range []string{"FindToolsOptions", "AlexandriaCall", "FindToolsData"} {
+		if termScore(question, []string{"alexandria", "tools", "inspect", "schema", "contracts"}) == 0 {
+			break
+		}
+		contract, ok := spec.Components.Schemas[name]
+		if !ok {
+			continue
+		}
+		parts = append(parts, "", "### "+name)
+		if desc := stringValue(contract["description"]); desc != "" {
+			parts = append(parts, desc)
+		}
+		if props, ok := contract["properties"].(map[string]any); ok {
+			keys := make([]string, 0, len(props))
+			for key := range props {
+				keys = append(keys, key)
+			}
+			sort.Strings(keys)
+			for _, key := range keys {
+				item := mapValue(props[key])
+				typ := stringValue(item["type"])
+				if typ == "" {
+					typ = "schema"
+				}
+				description := stringValue(item["description"])
+				if len(description) > 240 {
+					description = description[:240] + "…"
+				}
+				parts = append(parts, fmt.Sprintf("- `%s` (%s): %s", key, typ, description))
+			}
+		}
+	}
+	return strings.Join(parts, "\n")
+}
+
+func mapValue(value any) map[string]any {
+	v, _ := value.(map[string]any)
+	return v
+}
+
+func requestExamples(request map[string]any) map[string]map[string]any {
+	content := mapValue(request["content"])
+	jsonContent := mapValue(content["application/json"])
+	examples := mapValue(jsonContent["examples"])
+	out := make(map[string]map[string]any, len(examples))
+	for key, example := range examples {
+		if item := mapValue(example); item != nil && item["value"] != nil {
+			out[key] = item
+		}
+	}
+	return out
+}
+
+func relatedAPITerms(question string) []string {
+	question = strings.ToLower(question)
+	var terms []string
+	if strings.Contains(question, "discover") || strings.Contains(question, "locat") {
+		terms = append(terms, "search", "find", "list")
+	}
+	if strings.Contains(question, "inspect") || strings.Contains(question, "contract") {
+		terms = append(terms, "find", "tools", "schema")
+	}
+	if strings.Contains(question, "execut") || strings.Contains(question, "run ") {
+		terms = append(terms, "scrape", "call", "execute")
+	}
+	return terms
+}

package/internal/app/research_openapi_test.go

@@ -0,0 +1,140 @@
+package app
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func exampleOpenAPISpec(t *testing.T) string {
+	t.Helper()
+	source := map[string]any{
+		"openapi": "3.0.0",
+		"servers": []any{map[string]any{"url": "https://api.example.com/v2"}},
+		"paths": map[string]any{
+			"/search": map[string]any{
+				"parameters": []any{map[string]any{"name": "unused"}}, // Legal non-operation path entry.
+				"post":       map[string]any{"summary": "Search the Alexandria catalogue"},
+			},
+			"/scrape": map[string]any{
+				"post": map[string]any{
+					"summary": "Execute a provider capability or find tool contracts",
+					"requestBody": map[string]any{
+						"content": map[string]any{
+							"application/json": map[string]any{
+								"examples": map[string]any{
+									"findTools": map[string]any{
+										"summary": "Find Tools (free)",
+										"value": map[string]any{
+											"alexandria": map[string]any{
+												"provider":   "firecrawl",
+												"capability": "find-tools",
+												"options": map[string]any{
+													"providers": []string{"particle"},
+													"expand":    []string{"options", "response"},
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		"components": map[string]any{"schemas": map[string]any{
+			"FindToolsOptions": map[string]any{
+				"description": "Browse providers and inspect tool contracts.",
+				"properties": map[string]any{
+					"expand":    map[string]any{"type": "array", "description": "Contract sections to return"},
+					"providers": map[string]any{"type": "array", "description": "Provider IDs"},
+				},
+			},
+		}},
+	}
+	bytes, err := json.Marshal(source)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(bytes)
+}
+
+func TestOpenAPIContractReturnsExactInspectionRESTExample(t *testing.T) {
+	result := summarizeOpenAPIContract(exampleOpenAPISpec(t),
+		"What are the Alexandria REST API endpoints for discovering, inspecting, and executing tools? Include exact request examples.")
+	for _, want := range []string{
+		"POST https://api.example.com/v2/search",
+		"POST https://api.example.com/v2/scrape",
+		"\"provider\": \"firecrawl\"",
+		"\"capability\": \"find-tools\"",
+		"\"expand\": [",
+		"FindToolsOptions",
+		"<<'JSON_REQUEST'",
+		"-H \"Authorization: Bearer $API_KEY\"",
+	} {
+		if !strings.Contains(result, want) {
+			t.Errorf("OpenAPI extraction lost %q:\n%s", want, result)
+		}
+	}
+	if count := strings.Count(result, "JSON_REQUEST\n"+strings.Repeat(string(rune(96)), 3)); count != 1 {
+		t.Fatalf("expected one complete REST example, got %d:\n%s", count, result)
+	}
+}
+
+func TestQuestionFocusedReadPullsOfficialOpenAPISpec(t *testing.T) {
+	oldClient := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = oldClient })
+	spec := exampleOpenAPISpec(t)
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		switch req.URL.Path {
+		case "/llms.txt":
+			return testHTTPResponse(req, http.StatusOK,
+				"# Docs\n\n## OpenAPI Specs\n\n- [v2-openapi](/api-reference/v2-openapi.json)\n", nil), nil
+		case "/api-reference/v2-openapi.json":
+			return testHTTPResponse(req, http.StatusOK, spec, nil), nil
+		default:
+			return testHTTPResponse(req, http.StatusNotFound, "", nil), nil
+		}
+	})}
+	primary := "# Alexandria\n\n## Discovery\n\ncurl https://api.example.com/v2/search\n" +
+		"\n## Execution\n\ncurl https://api.example.com/v2/scrape\n"
+	reader := func(url string) (string, error) { return primary, nil }
+	result, err := readLinkFocused("https://docs.example.com/features/alexandria",
+		"What are the Alexandria REST API endpoints for discovering, inspecting, and executing tools? Include exact request examples.",
+		reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{
+		"## REST API contract",
+		"**Source:** https://docs.example.com/api-reference/v2-openapi.json",
+		"\"capability\": \"find-tools\"",
+		"curl https://api.example.com/v2/search",
+		"curl https://api.example.com/v2/scrape",
+		"**Coverage:** 2 source document(s)",
+	} {
+		if !strings.Contains(result, want) {
+			t.Errorf("focused read missing %q:\n%s", want, result)
+		}
+	}
+}
+
+func TestOpenAPIDiscoveryNeverCrossesToUnrelatedHost(t *testing.T) {
+	oldClient := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = oldClient })
+	var calledOutsideIndex bool
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		if req.URL.Path != "/llms.txt" {
+			calledOutsideIndex = true
+		}
+		return testHTTPResponse(req, http.StatusOK,
+			"- [External OpenAPI](https://evil.example.com/openapi.json)\n"+
+				"- [Local OpenAPI](http://127.0.0.1/openapi.json)\n", nil), nil
+	})}
+	if found := discoverOpenAPIContract("https://docs.example.com/reference",
+		"Find exact REST API endpoint request examples"); found.Content != "" || calledOutsideIndex {
+		t.Fatalf("unsafe OpenAPI discovery: %+v, off-index=%v", found, calledOutsideIndex)
+	}
+}

package/internal/app/research_test.go

@@ -0,0 +1,150 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func TestReadLinkFocusedKeepsURLOnlyContract(t *testing.T) {
+	called := 0
+	reader := func(url string) (string, error) {
+		called++
+		if url != "https://docs.example.com/api" {
+			t.Fatalf("unexpected URL: %s", url)
+		}
+		return "# Entire document\n\nno focus needed", nil
+	}
+	got, err := readLinkFocused("https://docs.example.com/api", "", reader)
+	if err != nil || got != "# Entire document\n\nno focus needed" || called != 1 {
+		t.Fatalf("URL-only path changed: %q, %v, calls=%d", got, err, called)
+	}
+}
+
+func TestFocusedReadExpandsRelevantSameHostLinks(t *testing.T) {
+	old := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = old })
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		return testHTTPResponse(req, http.StatusNotFound, "", nil), nil
+	})}
+	urls := map[string]string{
+		"https://docs.example.com/guide":      "# Guide\n[Authentication](./auth)\n[Pagination](/pagination)\n[Offsite](https://evil.example/pagination)\n[Secret](./auth?token=secret)\n[Self](#pagination)",
+		"https://docs.example.com/auth":       "# Authentication\nSet the Authorization header with your token.",
+		"https://docs.example.com/pagination": "# Pagination\nPass next_cursor to read the next page.",
+	}
+	requested := make(map[string]int)
+	reader := func(url string) (string, error) {
+		requested[url]++
+		value, ok := urls[url]
+		if !ok {
+			return "", errors.New("missing URL")
+		}
+		return value, nil
+	}
+	got, err := readLinkFocused("https://docs.example.com/guide", "How do authentication and pagination work?", reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{"next_cursor", "Authorization header", "https://docs.example.com/auth", "https://docs.example.com/pagination", "**Coverage:** 3 source document(s)"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("missing %q in result:\n%s", want, got)
+		}
+	}
+	if len(requested) != 3 {
+		t.Fatalf("followed unsafe/irrelevant link: %+v", requested)
+	}
+}
+
+func TestFocusedExcerptPrioritizesRelevantHeadingAndCode(t *testing.T) {
+	large := "# Docs\n" + strings.Repeat("Large unrelated introduction.\n", 600) + "\n## Authenticated requests\nUse an Authorization header.\n```go\nheaders.Set(\"Authorization\", token)\n```\n\n## Other\n" + strings.Repeat("Unrelated text\n", 200)
+	got := focusExcerpt(large, "How to authenticate requests with Go?", 3000)
+	if !strings.Contains(got, "headers.Set") || strings.Contains(got, "Large unrelated introduction.Large") {
+		t.Fatalf("did not select API code example: %s", got)
+	}
+	if len(got) > 3500 {
+		t.Fatalf("excerpt unexpectedly large: %d", len(got))
+	}
+}
+
+func TestSearchResearchReturnsEvidenceAndSupportingSource(t *testing.T) {
+	results := []SearchDoc{
+		{URL: "https://docs.example.com/guide", Title: "Guide"},
+		{URL: "https://github.com/example/project", Title: "Source"},
+		{URL: "https://other.example.com/blog", Title: "Blog"},
+	}
+	reader := func(url string) (string, error) {
+		switch url {
+		case "https://docs.example.com/guide":
+			return "# Auth\nUse OAuth.\n[Pagination](/pagination)", nil
+		case "https://docs.example.com/pagination":
+			return "# Pagination\nnext_cursor is optional.", nil
+		case "https://github.com/example/project":
+			return "# Source\nOAuth settings are configurable.", nil
+		case "https://other.example.com/blog":
+			return "# Blog\nOAuth authorization examples.", nil
+		default:
+			return "", fmt.Errorf("unexpected URL: %s", url)
+		}
+	}
+	got := enrichSearchWithReader("OAuth pagination", results, reader)
+	if !strings.Contains(got, "Use OAuth.") || !strings.Contains(got, "next_cursor") || !strings.Contains(got, "Supporting reference") {
+		t.Fatalf("search lacked fetched source context: %s", got)
+	}
+}
+
+func TestFollowLinksRejectsOtherHostsSecretsAndAssets(t *testing.T) {
+	markdown := "[API auth](https://other.example.com/auth) [API auth](/auth?api_key=private) [API auth](/image.png) [API auth](http://127.0.0.1/auth) [API auth](/auth)"
+	got := relatedSourceLinks(markdown, "https://docs.example.com/guide", "api auth", 5)
+	if len(got) != 1 || got[0].url != "https://docs.example.com/auth" {
+		t.Fatalf("unsafe or irrelevant candidates: %+v", got)
+	}
+}
+
+func TestRelevanceMatchingDoesNotConfuseAPIWithScraping(t *testing.T) {
+	if termScore("advanced scraping guide", []string{"api"}) != 0 {
+		t.Fatal("API should not match the middle of the word scraping")
+	}
+	links := relatedSourceLinks("[Advanced scraping](/advanced-scraping-guide) [API endpoint reference](/api-reference/endpoint/scrape)", "https://docs.example.com/features/alexandria", "Alexandria API endpoints", 3)
+	if len(links) != 1 || !strings.Contains(links[0].url, "api-reference") {
+		t.Fatalf("irrelevant docs were followed: %+v", links)
+	}
+}
+
+func TestSearchPrefersTechnicalReferenceOverCommunityPages(t *testing.T) {
+	results := []SearchDoc{
+		{URL: "https://modelcontextprotocol.io/community/interest-groups/tool-annotations", Title: "Tool annotations interest group"},
+		{URL: "https://modelcontextprotocol.io/specification/2025-11-25/server/tools", Title: "Tools specification"},
+		{URL: "https://github.com/modelcontextprotocol/go-sdk", Title: "Go SDK for MCP"},
+		{URL: "https://pkg.go.dev/golang.org/x/tools/internal/mcp", Title: "Internal Go MCP"},
+	}
+	selected := chooseSearchSources("Go MCP SDK tool annotations", results, 2)
+	if len(selected) != 2 || selected[0].url != "https://github.com/modelcontextprotocol/go-sdk" || !strings.Contains(selected[1].url, "/specification/") {
+		t.Fatalf("selected irrelevant pages over official SDK/spec: %+v", selected)
+	}
+}
+
+func TestFollowLinksSkipGenericMCPLinksForSpecificQuestion(t *testing.T) {
+	content := "[Skills over MCP](/community/working-groups/skills-over-mcp) [Tool annotations](/specification/server/tools)"
+	links := relatedSourceLinks(content, "https://modelcontextprotocol.io/docs/tools", "Go MCP SDK tool annotations", 2)
+	if len(links) != 1 || !strings.Contains(links[0].url, "/specification/server/tools") {
+		t.Fatalf("followed generic cross-topic link: %+v", links)
+	}
+}
+
+func TestSelectJSONFieldsPreservesSyntaxAndPaths(t *testing.T) {
+	obj := map[string]any{"paths": map[string]any{"/v2/scrape": map[string]any{"post": map[string]any{"summary": "Scrape an API document"}}}, "metadata": strings.Repeat("x", 5000)}
+	got := selectedJSONFields(obj, "scrape", 1400)
+	if !strings.Contains(got, "/paths/~1v2~1scrape") || !strings.Contains(got, "```json") || !strings.Contains(got, "Scrape an API document") {
+		t.Fatalf("JSON selection missing source pointers: %s", got)
+	}
+}
+
+func TestLargeOpenAPISchemaUsesSourceJSONPointers(t *testing.T) {
+	source := `**URL:** https://docs.example.com/openapi.json` + "\n\n```json\n" + `{"paths":{"/v2/scrape":{"post":{"summary":"Scrape pages"}}},"large":"` + strings.Repeat("x", 12000) + `"}` + "\n```"
+	got := focusExcerpt(source, "scrape endpoint", 2500)
+	if !strings.Contains(got, "/paths/~1v2~1scrape") || !strings.Contains(got, "Scrape pages") || len(got) > 2700 {
+		t.Fatalf("large raw JSON should be selected by pointer: %s", got)
+	}
+}

package/internal/app/source_url.go

@@ -0,0 +1,25 @@
+package app
+
+import (
+	"fmt"
+	"net"
+	"net/url"
+	"strings"
+)
+
+// Reject non-public URL destinations before calling native readers or
+// hosted scraping providers. Credentials embedded in URLs are never accepted.
+func validateSourceURL(raw string) error {
+	u, err := url.Parse(raw)
+	if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil {
+		return fmt.Errorf("readable URL must be an absolute HTTP(S) URL without credentials")
+	}
+	host := strings.ToLower(strings.TrimSuffix(u.Hostname(), "."))
+	if host == "localhost" || strings.HasSuffix(host, ".localhost") || strings.HasSuffix(host, ".local") || strings.HasSuffix(host, ".internal") {
+		return fmt.Errorf("private/internal host is not a readable public URL")
+	}
+	if ip := net.ParseIP(host); ip != nil && (!ip.IsGlobalUnicast() || ip.IsPrivate() || ip.IsLoopback()) {
+		return fmt.Errorf("private or reserved IP address is not a readable public URL")
+	}
+	return nil
+}