webctx 0.2.2 → 0.2.4

package/internal/app/research_openapi.go

@@ -0,0 +1,289 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"sort"
+	"strings"
+	"time"
+)
+
+// API contracts are public documentation, not calls to the documented API.
+// When a question requires exact REST payloads, inspecting the site's linked
+// OpenAPI spec is more faithful than extrapolating HTTP requests from SDK code.
+type openAPIContext struct {
+	URL     string
+	Content string
+}
+
+func asksForAPIContracts(question string) bool {
+	question = strings.ToLower(question)
+	return strings.Contains(question, "rest") || strings.Contains(question, "openapi") ||
+		(strings.Contains(question, "api") && (strings.Contains(question, "endpoint") || strings.Contains(question, "request") || strings.Contains(question, "schema") || strings.Contains(question, "contract"))) ||
+		(strings.Contains(question, "exact") && (strings.Contains(question, "curl") || strings.Contains(question, "request example")))
+}
+
+func discoverOpenAPIContract(rawURL, question string) openAPIContext {
+	if !asksForAPIContracts(question) || directSourceLanguage(rawURL) != "" {
+		return openAPIContext{}
+	}
+	u, err := url.Parse(rawURL)
+	if err != nil || validateSourceURL(rawURL) != nil || strings.EqualFold(u.Hostname(), "github.com") {
+		return openAPIContext{}
+	}
+	indexURL := u.Scheme + "://" + u.Host + "/llms.txt"
+	ctx, cancel := context.WithTimeout(context.Background(), 18*time.Second)
+	defer cancel()
+	index, err := fetchPublicText(ctx, indexURL, 300000)
+	if err != nil {
+		return openAPIContext{}
+	}
+	var specURLs []sourceCandidate
+	seen := make(map[string]bool)
+	for _, match := range mdLinkPattern.FindAllStringSubmatch(index, 300) {
+		dest, err := url.Parse(strings.TrimSpace(match[2]))
+		if err != nil {
+			continue
+		}
+		resolved := u.ResolveReference(dest)
+		name := strings.ToLower(resolved.Path)
+		if !strings.EqualFold(resolved.Hostname(), u.Hostname()) || resolved.Port() != u.Port() ||
+			!strings.HasSuffix(name, ".json") || !strings.Contains(name, "openapi") ||
+			validateSourceURL(resolved.String()) != nil || sensitiveURL(resolved) {
+			continue
+		}
+		sourceURL := canonicalSourceURL(resolved)
+		if seen[sourceURL] {
+			continue
+		}
+		seen[sourceURL] = true
+		score := 0
+		if strings.Contains(name, "v2-openapi") {
+			score += 10
+		}
+		if strings.HasPrefix(name, "/api-reference/") {
+			score += 5
+		}
+		if name == "/openapi.json" {
+			score += 5
+		}
+		score -= len(name) / 20 // Prefer the canonical path over translations.
+		specURLs = append(specURLs, sourceCandidate{url: sourceURL, score: score})
+	}
+	sort.SliceStable(specURLs, func(i, j int) bool { return specURLs[i].score > specURLs[j].score })
+	for i, item := range specURLs {
+		if i >= 2 { // Don't recursively fetch every language/version.
+			break
+		}
+		body, err := fetchPublicText(ctx, item.url, maxDirectSourceBytes)
+		if err != nil {
+			continue
+		}
+		if content := summarizeOpenAPIContract(body, question); content != "" {
+			return openAPIContext{URL: item.url, Content: content}
+		}
+	}
+	return openAPIContext{}
+}
+
+type openAPIOperation struct {
+	Method   string
+	Path     string
+	Summary  string
+	Score    int
+	Request  map[string]any
+	Response map[string]any
+}
+
+func summarizeOpenAPIContract(source, question string) string {
+	var spec struct {
+		OpenAPI string `json:"openapi"`
+		Swagger string `json:"swagger"`
+		Servers []struct {
+			URL string `json:"url"`
+		} `json:"servers"`
+		Paths      map[string]map[string]any `json:"paths"`
+		Components struct {
+			Schemas map[string]map[string]any `json:"schemas"`
+		} `json:"components"`
+	}
+	if json.Unmarshal([]byte(source), &spec) != nil || (spec.OpenAPI == "" && spec.Swagger == "") || len(spec.Paths) == 0 {
+		return ""
+	}
+	terms := queryTerms(question)
+	terms = append(terms, relatedAPITerms(question)...)
+	var operations []openAPIOperation
+	for path, methods := range spec.Paths {
+		// A broad question about Alexandria must not bring in unrelated API
+		// routes such as research papers or team activity merely because the
+		// path contains "search" or a summary happens to contain "API".
+		if strings.Contains(strings.ToLower(question), "alexandria") &&
+			strings.Contains(strings.Trim(path, "/"), "/") &&
+			!strings.Contains(strings.ToLower(path), "alexandria") {
+			continue
+		}
+		for method, rawDetail := range methods {
+			switch strings.ToUpper(method) {
+			case "GET", "POST", "PUT", "PATCH", "DELETE":
+			default:
+				continue
+			}
+			detail := mapValue(rawDetail)
+			summary := stringValue(detail["summary"])
+			req := mapValue(detail["requestBody"])
+			resp := mapValue(detail["responses"])
+			score := termScore(path+" "+summary, terms)*4 + termScore(stringValue(detail["description"]), terms)
+			if strings.Contains(strings.ToLower(question), "alexandria") &&
+				termScore(strings.Trim(path, "/"), []string{"search", "scrape", "find", "tools", "alexandria"}) == 0 {
+				continue
+			}
+			if examples := requestExamples(req); len(examples) > 0 {
+				for name, example := range examples {
+					score += termScore(name+" "+stringValue(example["summary"]), terms) * 3
+				}
+			}
+			if score > 0 {
+				operations = append(operations, openAPIOperation{
+					Method: strings.ToUpper(method), Path: path, Summary: summary, Score: score, Request: req, Response: resp,
+				})
+			}
+		}
+	}
+	sort.SliceStable(operations, func(i, j int) bool {
+		if operations[i].Score == operations[j].Score {
+			return operations[i].Path < operations[j].Path
+		}
+		return operations[i].Score > operations[j].Score
+	})
+	if len(operations) == 0 {
+		return ""
+	}
+	apiRoot := ""
+	for _, server := range spec.Servers {
+		if validateSourceURL(server.URL) == nil {
+			apiRoot = strings.TrimRight(server.URL, "/")
+			break
+		}
+	}
+	parts := []string{"OpenAPI-defined HTTP operations and example request bodies (not executed):"}
+	for i, operation := range operations {
+		if i >= 4 {
+			break
+		}
+		endpoint := operation.Path
+		if apiRoot != "" {
+			endpoint = apiRoot + operation.Path
+		}
+		parts = append(parts, "", fmt.Sprintf("### %s %s", operation.Method, endpoint))
+		if operation.Summary != "" {
+			parts = append(parts, operation.Summary)
+		}
+		if examples := requestExamples(operation.Request); len(examples) > 0 {
+			names := make([]string, 0, len(examples))
+			for name := range examples {
+				names = append(names, name)
+			}
+			sort.SliceStable(names, func(i, j int) bool {
+				a := termScore(names[i]+" "+stringValue(examples[names[i]]["summary"]), terms)
+				b := termScore(names[j]+" "+stringValue(examples[names[j]]["summary"]), terms)
+				if a == b {
+					return names[i] < names[j]
+				}
+				return a > b
+			})
+			emitted := 0
+			for _, name := range names {
+				value := examples[name]["value"]
+				body, err := json.MarshalIndent(value, "", "  ")
+				if err != nil || len(body) > 9000 || len(body) == 0 {
+					continue
+				}
+				if strings.Contains(strings.ToLower(question), "alexandria") &&
+					!strings.Contains(strings.ToLower(name+" "+string(body)), "alexandria") &&
+					!strings.Contains(strings.ToLower(name+" "+string(body)), "findtools") &&
+					!strings.Contains(strings.ToLower(name+" "+string(body)), "find-tools") {
+					continue
+				}
+				if emitted >= 2 {
+					break
+				}
+				emitted++
+				parts = append(parts, "", "**Published request example:** "+stringValue(examples[name]["summary"]))
+				if apiRoot != "" && !strings.Contains(endpoint, "{") {
+					// Keep the published JSON body verbatim; the API key must expand in the shell.
+					parts = append(parts, fmt.Sprintf("```bash\ncurl -X %s %q \\\n  -H \"Authorization: Bearer $API_KEY\" \\\n  -H \"Content-Type: application/json\" \\\n  --data-binary @- <<'JSON_REQUEST'\n%s\nJSON_REQUEST\n```", operation.Method, endpoint, string(body)))
+				} else {
+					parts = append(parts, "```json\n"+string(body)+"\n```")
+				}
+			}
+		}
+	}
+	for _, name := range []string{"FindToolsOptions", "AlexandriaCall", "FindToolsData"} {
+		if termScore(question, []string{"alexandria", "tools", "inspect", "schema", "contracts"}) == 0 {
+			break
+		}
+		contract, ok := spec.Components.Schemas[name]
+		if !ok {
+			continue
+		}
+		parts = append(parts, "", "### "+name)
+		if desc := stringValue(contract["description"]); desc != "" {
+			parts = append(parts, desc)
+		}
+		if props, ok := contract["properties"].(map[string]any); ok {
+			keys := make([]string, 0, len(props))
+			for key := range props {
+				keys = append(keys, key)
+			}
+			sort.Strings(keys)
+			for _, key := range keys {
+				item := mapValue(props[key])
+				typ := stringValue(item["type"])
+				if typ == "" {
+					typ = "schema"
+				}
+				description := stringValue(item["description"])
+				if len(description) > 240 {
+					description = description[:240] + "…"
+				}
+				parts = append(parts, fmt.Sprintf("- `%s` (%s): %s", key, typ, description))
+			}
+		}
+	}
+	return strings.Join(parts, "\n")
+}
+
+func mapValue(value any) map[string]any {
+	v, _ := value.(map[string]any)
+	return v
+}
+
+func requestExamples(request map[string]any) map[string]map[string]any {
+	content := mapValue(request["content"])
+	jsonContent := mapValue(content["application/json"])
+	examples := mapValue(jsonContent["examples"])
+	out := make(map[string]map[string]any, len(examples))
+	for key, example := range examples {
+		if item := mapValue(example); item != nil && item["value"] != nil {
+			out[key] = item
+		}
+	}
+	return out
+}
+
+func relatedAPITerms(question string) []string {
+	question = strings.ToLower(question)
+	var terms []string
+	if strings.Contains(question, "discover") || strings.Contains(question, "locat") {
+		terms = append(terms, "search", "find", "list")
+	}
+	if strings.Contains(question, "inspect") || strings.Contains(question, "contract") {
+		terms = append(terms, "find", "tools", "schema")
+	}
+	if strings.Contains(question, "execut") || strings.Contains(question, "run ") {
+		terms = append(terms, "scrape", "call", "execute")
+	}
+	return terms
+}

package/internal/app/research_openapi_test.go

@@ -0,0 +1,140 @@
+package app
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func exampleOpenAPISpec(t *testing.T) string {
+	t.Helper()
+	source := map[string]any{
+		"openapi": "3.0.0",
+		"servers": []any{map[string]any{"url": "https://api.example.com/v2"}},
+		"paths": map[string]any{
+			"/search": map[string]any{
+				"parameters": []any{map[string]any{"name": "unused"}}, // Legal non-operation path entry.
+				"post":       map[string]any{"summary": "Search the Alexandria catalogue"},
+			},
+			"/scrape": map[string]any{
+				"post": map[string]any{
+					"summary": "Execute a provider capability or find tool contracts",
+					"requestBody": map[string]any{
+						"content": map[string]any{
+							"application/json": map[string]any{
+								"examples": map[string]any{
+									"findTools": map[string]any{
+										"summary": "Find Tools (free)",
+										"value": map[string]any{
+											"alexandria": map[string]any{
+												"provider":   "firecrawl",
+												"capability": "find-tools",
+												"options": map[string]any{
+													"providers": []string{"particle"},
+													"expand":    []string{"options", "response"},
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		"components": map[string]any{"schemas": map[string]any{
+			"FindToolsOptions": map[string]any{
+				"description": "Browse providers and inspect tool contracts.",
+				"properties": map[string]any{
+					"expand":    map[string]any{"type": "array", "description": "Contract sections to return"},
+					"providers": map[string]any{"type": "array", "description": "Provider IDs"},
+				},
+			},
+		}},
+	}
+	bytes, err := json.Marshal(source)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(bytes)
+}
+
+func TestOpenAPIContractReturnsExactInspectionRESTExample(t *testing.T) {
+	result := summarizeOpenAPIContract(exampleOpenAPISpec(t),
+		"What are the Alexandria REST API endpoints for discovering, inspecting, and executing tools? Include exact request examples.")
+	for _, want := range []string{
+		"POST https://api.example.com/v2/search",
+		"POST https://api.example.com/v2/scrape",
+		"\"provider\": \"firecrawl\"",
+		"\"capability\": \"find-tools\"",
+		"\"expand\": [",
+		"FindToolsOptions",
+		"<<'JSON_REQUEST'",
+		"-H \"Authorization: Bearer $API_KEY\"",
+	} {
+		if !strings.Contains(result, want) {
+			t.Errorf("OpenAPI extraction lost %q:\n%s", want, result)
+		}
+	}
+	if count := strings.Count(result, "JSON_REQUEST\n"+strings.Repeat(string(rune(96)), 3)); count != 1 {
+		t.Fatalf("expected one complete REST example, got %d:\n%s", count, result)
+	}
+}
+
+func TestQuestionFocusedReadPullsOfficialOpenAPISpec(t *testing.T) {
+	oldClient := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = oldClient })
+	spec := exampleOpenAPISpec(t)
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		switch req.URL.Path {
+		case "/llms.txt":
+			return testHTTPResponse(req, http.StatusOK,
+				"# Docs\n\n## OpenAPI Specs\n\n- [v2-openapi](/api-reference/v2-openapi.json)\n", nil), nil
+		case "/api-reference/v2-openapi.json":
+			return testHTTPResponse(req, http.StatusOK, spec, nil), nil
+		default:
+			return testHTTPResponse(req, http.StatusNotFound, "", nil), nil
+		}
+	})}
+	primary := "# Alexandria\n\n## Discovery\n\ncurl https://api.example.com/v2/search\n" +
+		"\n## Execution\n\ncurl https://api.example.com/v2/scrape\n"
+	reader := func(url string) (string, error) { return primary, nil }
+	result, err := readLinkFocused("https://docs.example.com/features/alexandria",
+		"What are the Alexandria REST API endpoints for discovering, inspecting, and executing tools? Include exact request examples.",
+		reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{
+		"## REST API contract",
+		"**Source:** https://docs.example.com/api-reference/v2-openapi.json",
+		"\"capability\": \"find-tools\"",
+		"curl https://api.example.com/v2/search",
+		"curl https://api.example.com/v2/scrape",
+		"**Coverage:** 2 source document(s)",
+	} {
+		if !strings.Contains(result, want) {
+			t.Errorf("focused read missing %q:\n%s", want, result)
+		}
+	}
+}
+
+func TestOpenAPIDiscoveryNeverCrossesToUnrelatedHost(t *testing.T) {
+	oldClient := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = oldClient })
+	var calledOutsideIndex bool
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		if req.URL.Path != "/llms.txt" {
+			calledOutsideIndex = true
+		}
+		return testHTTPResponse(req, http.StatusOK,
+			"- [External OpenAPI](https://evil.example.com/openapi.json)\n"+
+				"- [Local OpenAPI](http://127.0.0.1/openapi.json)\n", nil), nil
+	})}
+	if found := discoverOpenAPIContract("https://docs.example.com/reference",
+		"Find exact REST API endpoint request examples"); found.Content != "" || calledOutsideIndex {
+		t.Fatalf("unsafe OpenAPI discovery: %+v, off-index=%v", found, calledOutsideIndex)
+	}
+}

package/internal/app/research_test.go

@@ -0,0 +1,154 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+)
+
+func TestReadLinkFocusedKeepsURLOnlyContract(t *testing.T) {
+	called := 0
+	reader := func(url string) (string, error) {
+		called++
+		if url != "https://docs.example.com/api" {
+			t.Fatalf("unexpected URL: %s", url)
+		}
+		return "# Entire document\n\nno focus needed", nil
+	}
+	got, err := readLinkFocused("https://docs.example.com/api", "", reader)
+	if err != nil || got != "# Entire document\n\nno focus needed" || called != 1 {
+		t.Fatalf("URL-only path changed: %q, %v, calls=%d", got, err, called)
+	}
+}
+
+func TestFocusedReadExpandsRelevantSameHostLinks(t *testing.T) {
+	old := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = old })
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		return testHTTPResponse(req, http.StatusNotFound, "", nil), nil
+	})}
+	urls := map[string]string{
+		"https://docs.example.com/guide":      "# Guide\n[Authentication](./auth)\n[Pagination](/pagination)\n[Offsite](https://evil.example/pagination)\n[Secret](./auth?token=secret)\n[Self](#pagination)",
+		"https://docs.example.com/auth":       "# Authentication\nSet the Authorization header with your token.",
+		"https://docs.example.com/pagination": "# Pagination\nPass next_cursor to read the next page.",
+	}
+	requested := make(map[string]int)
+	var requestedMu sync.Mutex
+	reader := func(url string) (string, error) {
+		requestedMu.Lock()
+		requested[url]++
+		requestedMu.Unlock()
+		value, ok := urls[url]
+		if !ok {
+			return "", errors.New("missing URL")
+		}
+		return value, nil
+	}
+	got, err := readLinkFocused("https://docs.example.com/guide", "How do authentication and pagination work?", reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{"next_cursor", "Authorization header", "https://docs.example.com/auth", "https://docs.example.com/pagination", "**Coverage:** 3 source document(s)"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("missing %q in result:\n%s", want, got)
+		}
+	}
+	if len(requested) != 3 {
+		t.Fatalf("followed unsafe/irrelevant link: %+v", requested)
+	}
+}
+
+func TestFocusedExcerptPrioritizesRelevantHeadingAndCode(t *testing.T) {
+	large := "# Docs\n" + strings.Repeat("Large unrelated introduction.\n", 600) + "\n## Authenticated requests\nUse an Authorization header.\n```go\nheaders.Set(\"Authorization\", token)\n```\n\n## Other\n" + strings.Repeat("Unrelated text\n", 200)
+	got := focusExcerpt(large, "How to authenticate requests with Go?", 3000)
+	if !strings.Contains(got, "headers.Set") || strings.Contains(got, "Large unrelated introduction.Large") {
+		t.Fatalf("did not select API code example: %s", got)
+	}
+	if len(got) > 3500 {
+		t.Fatalf("excerpt unexpectedly large: %d", len(got))
+	}
+}
+
+func TestSearchResearchReturnsEvidenceAndSupportingSource(t *testing.T) {
+	results := []SearchDoc{
+		{URL: "https://docs.example.com/guide", Title: "Guide"},
+		{URL: "https://github.com/example/project", Title: "Source"},
+		{URL: "https://other.example.com/blog", Title: "Blog"},
+	}
+	reader := func(url string) (string, error) {
+		switch url {
+		case "https://docs.example.com/guide":
+			return "# Auth\nUse OAuth.\n[Pagination](/pagination)", nil
+		case "https://docs.example.com/pagination":
+			return "# Pagination\nnext_cursor is optional.", nil
+		case "https://github.com/example/project":
+			return "# Source\nOAuth settings are configurable.", nil
+		case "https://other.example.com/blog":
+			return "# Blog\nOAuth authorization examples.", nil
+		default:
+			return "", fmt.Errorf("unexpected URL: %s", url)
+		}
+	}
+	got := enrichSearchWithReader("OAuth pagination", results, reader)
+	if !strings.Contains(got, "Use OAuth.") || !strings.Contains(got, "next_cursor") || !strings.Contains(got, "Supporting reference") {
+		t.Fatalf("search lacked fetched source context: %s", got)
+	}
+}
+
+func TestFollowLinksRejectsOtherHostsSecretsAndAssets(t *testing.T) {
+	markdown := "[API auth](https://other.example.com/auth) [API auth](/auth?api_key=private) [API auth](/image.png) [API auth](http://127.0.0.1/auth) [API auth](/auth)"
+	got := relatedSourceLinks(markdown, "https://docs.example.com/guide", "api auth", 5)
+	if len(got) != 1 || got[0].url != "https://docs.example.com/auth" {
+		t.Fatalf("unsafe or irrelevant candidates: %+v", got)
+	}
+}
+
+func TestRelevanceMatchingDoesNotConfuseAPIWithScraping(t *testing.T) {
+	if termScore("advanced scraping guide", []string{"api"}) != 0 {
+		t.Fatal("API should not match the middle of the word scraping")
+	}
+	links := relatedSourceLinks("[Advanced scraping](/advanced-scraping-guide) [API endpoint reference](/api-reference/endpoint/scrape)", "https://docs.example.com/features/alexandria", "Alexandria API endpoints", 3)
+	if len(links) != 1 || !strings.Contains(links[0].url, "api-reference") {
+		t.Fatalf("irrelevant docs were followed: %+v", links)
+	}
+}
+
+func TestSearchPrefersTechnicalReferenceOverCommunityPages(t *testing.T) {
+	results := []SearchDoc{
+		{URL: "https://modelcontextprotocol.io/community/interest-groups/tool-annotations", Title: "Tool annotations interest group"},
+		{URL: "https://modelcontextprotocol.io/specification/2025-11-25/server/tools", Title: "Tools specification"},
+		{URL: "https://github.com/modelcontextprotocol/go-sdk", Title: "Go SDK for MCP"},
+		{URL: "https://pkg.go.dev/golang.org/x/tools/internal/mcp", Title: "Internal Go MCP"},
+	}
+	selected := chooseSearchSources("Go MCP SDK tool annotations", results, 2)
+	if len(selected) != 2 || selected[0].url != "https://github.com/modelcontextprotocol/go-sdk" || !strings.Contains(selected[1].url, "/specification/") {
+		t.Fatalf("selected irrelevant pages over official SDK/spec: %+v", selected)
+	}
+}
+
+func TestFollowLinksSkipGenericMCPLinksForSpecificQuestion(t *testing.T) {
+	content := "[Skills over MCP](/community/working-groups/skills-over-mcp) [Tool annotations](/specification/server/tools)"
+	links := relatedSourceLinks(content, "https://modelcontextprotocol.io/docs/tools", "Go MCP SDK tool annotations", 2)
+	if len(links) != 1 || !strings.Contains(links[0].url, "/specification/server/tools") {
+		t.Fatalf("followed generic cross-topic link: %+v", links)
+	}
+}
+
+func TestSelectJSONFieldsPreservesSyntaxAndPaths(t *testing.T) {
+	obj := map[string]any{"paths": map[string]any{"/v2/scrape": map[string]any{"post": map[string]any{"summary": "Scrape an API document"}}}, "metadata": strings.Repeat("x", 5000)}
+	got := selectedJSONFields(obj, "scrape", 1400)
+	if !strings.Contains(got, "/paths/~1v2~1scrape") || !strings.Contains(got, "```json") || !strings.Contains(got, "Scrape an API document") {
+		t.Fatalf("JSON selection missing source pointers: %s", got)
+	}
+}
+
+func TestLargeOpenAPISchemaUsesSourceJSONPointers(t *testing.T) {
+	source := `**URL:** https://docs.example.com/openapi.json` + "\n\n```json\n" + `{"paths":{"/v2/scrape":{"post":{"summary":"Scrape pages"}}},"large":"` + strings.Repeat("x", 12000) + `"}` + "\n```"
+	got := focusExcerpt(source, "scrape endpoint", 2500)
+	if !strings.Contains(got, "/paths/~1v2~1scrape") || !strings.Contains(got, "Scrape pages") || len(got) > 2700 {
+		t.Fatalf("large raw JSON should be selected by pointer: %s", got)
+	}
+}

package/internal/app/retrieval_quality.go

@@ -0,0 +1,106 @@
+package app
+
+import (
+	"fmt"
+	"log"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+)
+
+// CAPTCHA pages can contain thousands of bytes of translated prompts. Page
+// length is not evidence of useful content, so challenge detection must not
+// be gated on the size of the extracted Markdown.
+func challengeReason(markdown string) string {
+	text := strings.ToLower(strings.TrimSpace(markdown))
+	if text == "" {
+		return "empty source"
+	}
+	prefix := text[:min(len(text), 4000)]
+	for _, marker := range []string{
+		"just a moment...",
+		"checking your browser",
+		"access denied | cloudflare",
+		"attention required! | cloudflare",
+		"enable javascript and cookies",
+		"no content extracted",
+		"please complete the security check",
+		"verify you are human",
+		"verifying you are human",
+		"verify that you are human",
+		"please verify you are a human",
+		"we need to verify your browser",
+		"challenge-platform",
+	} {
+		if strings.Contains(prefix, marker) {
+			return "browser verification challenge"
+		}
+	}
+	if strings.Contains(prefix, "hcaptcha") || strings.Contains(prefix, "h-captcha") {
+		for _, secondary := range []string{
+			"select all", "please verify", "i am human",
+			"privacy - terms", "please complete", "security check",
+			"captcha challenge", "verify your", "please solve",
+		} {
+			if strings.Contains(text, secondary) {
+				return "hCaptcha challenge"
+			}
+		}
+	}
+	if (strings.Count(text, "captcha") > 2 || strings.Count(text, "verify you are human") > 0) &&
+		(strings.Contains(text, "try again") || strings.Contains(text, "challenge") || strings.Contains(text, "i am human")) {
+		return "CAPTCHA challenge"
+	}
+	return ""
+}
+
+func usableSourceContent(markdown string) bool {
+	text := strings.TrimSpace(markdown)
+	if len(text) < 10 || challengeReason(text) != "" {
+		return false
+	}
+	// Guard against a 200 status containing an HTML block/error page.
+	prefix := strings.ToLower(text[:min(len(text), 350)])
+	return !strings.Contains(prefix, "<!doctype html") &&
+		!strings.Contains(prefix, "<html")
+}
+
+// WEBCTX_DEBUG=1 enables stage traces. Only host + URL path are included;
+// query parameters and credentials are intentionally not logged.
+func traceRetrieval(rawURL, stage, outcome string, start time.Time, err error) {
+	if os.Getenv("WEBCTX_DEBUG") != "1" {
+		return
+	}
+	source := "invalid-url"
+	if u, parseErr := url.Parse(rawURL); parseErr == nil && u.Hostname() != "" {
+		source = u.Hostname() + u.EscapedPath()
+	}
+	detail := ""
+	if err != nil {
+		detail = fmt.Sprintf(" error=%q", sanitizedRetrievalError(err))
+	}
+	log.Printf("webctx retrieval source=%q stage=%q outcome=%q duration_ms=%d%s",
+		source, stage, outcome, time.Since(start).Milliseconds(), detail)
+}
+
+func sanitizedRetrievalError(err error) string {
+	if err == nil {
+		return ""
+	}
+	message := strings.ToLower(err.Error())
+	switch {
+	case strings.Contains(message, "403"), strings.Contains(message, "forbidden"):
+		return "forbidden"
+	case strings.Contains(message, "429"), strings.Contains(message, "rate limit"):
+		return "rate-limited"
+	case strings.Contains(message, "timeout"), strings.Contains(message, "deadline"):
+		return "timeout"
+	case strings.Contains(message, "captcha"), strings.Contains(message, "challenge"):
+		return "challenge"
+	case strings.Contains(message, "404"), strings.Contains(message, "not found"):
+		return "not-found"
+	default:
+		return "source-unavailable"
+	}
+}