webctx 0.2.3 → 0.2.4

package/internal/app/firecrawl_fallback.go

@@ -18,20 +18,7 @@ type scrapedPage struct {
 }
 
 func usableScrapedMarkdown(markdown string) bool {
-	text := strings.ToLower(strings.TrimSpace(markdown))
-	if len(text) < 10 {
-		return false
-	}
-	for _, marker := range []string{
-		"just a moment...", "checking your browser", "verify you are human",
-		"enable javascript and cookies", "access denied | cloudflare",
-		"attention required! | cloudflare", "no content extracted",
-	} {
-		if strings.Contains(text, marker) && len(text) < 2000 {
-			return false
-		}
-	}
-	return true
+	return usableSourceContent(markdown)
 }
 
 func firecrawlScrape(rawURL, apiKey, proxy string) (scrapedPage, error) {
@@ -165,6 +152,9 @@ func firecrawlBrowserFallback(rawURL, apiKey string) (string, string, error) {
 	if parsed, err := strconv.Unquote(strings.TrimSpace(text)); err == nil {
 		text = parsed
 	}
+	if reason := challengeReason(text); reason != "" {
+		return "", "", fmt.Errorf("browser fallback returned %s rather than documentation", reason)
+	}
 	return "Browser extraction", text, nil
 }
 

package/internal/app/firecrawl_fallback_test.go

@@ -1,6 +1,7 @@
 package app
 
 import (
+	"encoding/json"
 	"io"
 	"net/http"
 	"strings"
@@ -18,6 +19,43 @@ func TestSourceURLSafety(t *testing.T) {
 	}
 }
 
+func TestLongBrowserCAPTCHAFallbackFailsClearly(t *testing.T) {
+	original := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = original })
+	challenge := "hCaptcha Please verify you are human. " +
+		strings.Repeat("French Deutsch Español English 日本語 中文 ", 150)
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		switch req.Method + " " + req.URL.Path {
+		case "POST /v2/scrape":
+			content, _ := json.Marshal(map[string]any{
+				"success": true,
+				"data":    map[string]any{"markdown": challenge},
+			})
+			return testHTTPResponse(req, http.StatusOK, string(content), nil), nil
+		case "POST /v2/interact":
+			return testHTTPResponse(req, http.StatusOK, `{"success":true,"id":"test-captcha"}`, nil), nil
+		case "POST /v2/interact/test-captcha/execute":
+			body, _ := json.Marshal(map[string]any{
+				"success": true, "stdout": challenge, "exitCode": 0,
+			})
+			return testHTTPResponse(req, http.StatusOK, string(body), nil), nil
+		case "DELETE /v2/interact/test-captcha":
+			return testHTTPResponse(req, http.StatusOK, `{"success":true}`, nil), nil
+		default:
+			t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
+			return nil, nil
+		}
+	})}
+	t.Setenv("FIRECRAWL_API_KEY", "unit-test")
+	_, returned, err := scrapeLinkWithFirecrawl("https://docs.example.com/challenged")
+	if err == nil || returned != "" || !strings.Contains(err.Error(), "challenge") {
+		t.Fatalf("long browser challenge leaked into result or lacked clear failure: %q %v", returned, err)
+	}
+	if strings.Contains(err.Error(), "French Deutsch") {
+		t.Fatalf("challenge payload should not be logged verbatim in errors: %s", err)
+	}
+}
+
 func TestScrapeEscalatesToEnhancedOnBlockedBody(t *testing.T) {
 	originalClient := http.DefaultClient
 	t.Cleanup(func() { http.DefaultClient = originalClient })

package/internal/app/native_markdown.go

@@ -0,0 +1,76 @@
+package app
+
+import (
+	"context"
+	"net/url"
+	"path"
+	"strings"
+	"time"
+)
+
+const maxNativeMarkdownBytes = 1024 * 1024
+
+// Documentation sites frequently expose canonical HTML and an authored
+// Markdown representation. Use bounded GET probes instead of HEAD:
+// Stripe sends Content-Length: 0 on a successful Markdown HEAD response,
+// and Cloudflare uses /index.md for directory URLs.
+func markdownCandidateURLs(rawURL string) []string {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return nil
+	}
+	p := u.Path
+	if strings.HasSuffix(strings.ToLower(p), ".md") {
+		return []string{rawURL}
+	}
+	var candidates []string
+	if strings.HasSuffix(p, "/") || p == "" {
+		candidates = append(candidates, strings.TrimSuffix(p, "/")+"/index.md")
+	} else if path.Ext(p) == "" {
+		candidates = append(candidates, p+".md", p+"/index.md")
+	} else {
+		return nil
+	}
+	seen := map[string]bool{}
+	var out []string
+	for _, candidate := range candidates {
+		next := *u
+		next.Path = candidate
+		next.RawPath = ""
+		next.Fragment = ""
+		next.RawFragment = ""
+		value := next.String()
+		if !seen[value] && validateSourceURL(value) == nil {
+			out = append(out, value)
+			seen[value] = true
+		}
+	}
+	return out
+}
+
+func usableNativeMarkdown(content string) bool {
+	text := strings.TrimSpace(content)
+	if len(text) < 18 || strings.IndexByte(text, 0) >= 0 || challengeReason(text) != "" {
+		return false
+	}
+	prefix := strings.ToLower(text[:min(len(text), 700)])
+	if strings.Contains(prefix, "<!doctype html") || strings.Contains(prefix, "<html") ||
+		strings.Contains(prefix, "<head") || strings.Contains(prefix, "<body") {
+		return false
+	}
+	return true
+}
+
+func readNativeMarkdown(rawURL string) (*MarkdownResult, error) {
+	for _, candidate := range markdownCandidateURLs(rawURL) {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		body, err := fetchPublicText(ctx, candidate, maxNativeMarkdownBytes)
+		cancel()
+		if err != nil || !usableNativeMarkdown(body) {
+			continue
+		}
+		title := firstHeadingOrFallback(body, path.Base(rawURL))
+		return &MarkdownResult{URL: rawURL, SourceURL: candidate, Title: title, Markdown: body}, nil
+	}
+	return nil, nil
+}

package/internal/app/native_markdown_test.go

@@ -0,0 +1,78 @@
+package app
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func TestCanonicalStripeGuidePrefersMarkdownWithZeroHeadLength(t *testing.T) {
+	previous := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = previous })
+	calls := []string{}
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		calls = append(calls, req.Method+" "+req.URL.Path)
+		if req.Method == http.MethodHead {
+			t.Fatal("native Markdown detection must not require HEAD metadata")
+		}
+		if req.Method == http.MethodGet && req.URL.Path == "/payments/checkout.md" {
+			r := testHTTPResponse(req, http.StatusOK, "# Build a payments page\n\nStripe-hosted checkout is supported.", map[string]string{"Content-Type": "text/markdown"})
+			r.ContentLength = 0 // Publishers can return zero or unknown length metadata.
+			return r, nil
+		}
+		t.Fatalf("unexpected extra request: %s %s", req.Method, req.URL)
+		return nil, nil
+	})}
+	out, err := ReadLink("https://docs.stripe.com/payments/checkout")
+	if err != nil || !strings.Contains(out, "**Markdown source:** https://docs.stripe.com/payments/checkout.md") {
+		t.Fatalf("native source not selected: %v %s", err, out)
+	}
+	if len(calls) != 1 {
+		t.Fatalf("unexpected calls: %+v", calls)
+	}
+}
+
+func TestCanonicalCloudflareDirectoryReadsIndexMarkdown(t *testing.T) {
+	previous := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = previous })
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		if req.Method != http.MethodGet || req.URL.Path != "/durable-objects/index.md" {
+			t.Fatalf("unexpected URL: %s %s", req.Method, req.URL)
+		}
+		return testHTTPResponse(req, http.StatusOK,
+			"# Cloudflare Durable Objects\n\n[Get started](https://developers.cloudflare.com/durable-objects/get-started/)",
+			map[string]string{"Content-Type": "text/markdown"}), nil
+	})}
+	out, err := ReadLink("https://developers.cloudflare.com/durable-objects/")
+	if err != nil || !strings.Contains(out, "**Markdown source:** https://developers.cloudflare.com/durable-objects/index.md") {
+		t.Fatalf("directory-index native Markdown failed: %v %s", err, out)
+	}
+}
+
+func TestCAPTCHADisguisedAsMarkdownIsRejected(t *testing.T) {
+	previous := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = previous })
+	var scrapeCalled bool
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		if req.URL.Path == "/guide.md" {
+			return testHTTPResponse(req, http.StatusOK, "# hCaptcha\nPlease verify you are human. "+
+				strings.Repeat("English French Deutsch Español Chinese 日本語 ", 200), map[string]string{"Content-Type": "text/markdown"}), nil
+		}
+		if req.URL.Path == "/guide/index.md" {
+			return testHTTPResponse(req, http.StatusNotFound, "", nil), nil
+		}
+		if req.Method == http.MethodPost && req.URL.Path == "/v2/scrape" {
+			scrapeCalled = true
+			return testHTTPResponse(req, http.StatusOK,
+				`{"success":true,"data":{"markdown":"# Legitimate guide\n\nFull instructions for a public API."}}`,
+				nil), nil
+		}
+		t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
+		return nil, nil
+	})}
+	t.Setenv("FIRECRAWL_API_KEY", "test")
+	out, err := ReadLink("https://docs.example.com/guide")
+	if err != nil || !scrapeCalled || strings.Contains(out, "Deutsch Español") || !strings.Contains(out, "Full instructions") {
+		t.Fatalf("CAPTCHA disguised as Markdown leaked or fallback failed: %v %s", err, out)
+	}
+}

package/internal/app/research.go

@@ -20,10 +20,19 @@ const (
 	focusedPrimaryBudget = 22000
 	focusedRelatedBudget = 6500
 	searchSourceBudget   = 4000
-	maxRelatedSources    = 2
+	maxRelatedSources    = 5
+	maxFollowupSources   = 2
 	maxSearchSources     = 3
 )
 
+func relatedExcerptBudget(rawURL string) int {
+	lower := strings.ToLower(rawURL)
+	if strings.Contains(lower, "/get-started/") || strings.Contains(lower, "/quickstart") {
+		return 18000 // Preserve end-to-end runnable setup guides.
+	}
+	return focusedRelatedBudget
+}
+
 var (
 	mdLinkPattern     = regexp.MustCompile(`\[([^\]\n]{2,120})\]\(([^\s)]+)(?:\s+"[^"]*")?\)`)
 	wordPattern       = regexp.MustCompile(`[a-zA-Z][a-zA-Z0-9_-]{2,}`)
@@ -79,25 +88,17 @@ func readLinkFocused(rawURL, question string, reader func(string) (string, error
 		return "", err
 	}
 
-	links := relatedSourceLinks(primary, rawURL, question, maxRelatedSources)
-	if len(links) < maxRelatedSources {
-		// Many doc systems publish llms.txt as their navigation index. It is a
-		// cheap way to find exact API reference pages not linked from this page.
-		for _, candidate := range linksFromDocsIndex(rawURL, question, maxRelatedSources) {
-			if len(links) == maxRelatedSources {
-				break
-			}
-			duplicate := candidate.url == rawURL
-			for _, chosen := range links {
-				duplicate = duplicate || chosen.url == candidate.url
-			}
-			if !duplicate {
-				links = append(links, candidate)
-			}
-		}
-	}
+	// Combine on-page links and authoritative Markdown index links instead of
+	// filling the entire budget from whichever links happen to appear first.
+	// In particular, an implementation question should prefer get-started,
+	// SQLite, webhook, and API references over incidental alarm/concept pages.
+	links := researchCandidates(rawURL, primary, question, maxRelatedSources)
 	structuredDone := make(chan string, 1)
-	go func() { structuredDone <- embeddedStructuredContext(rawURL, question) }()
+	if strings.Contains(primary, "**Markdown source:**") {
+		structuredDone <- ""
+	} else {
+		go func() { structuredDone <- embeddedStructuredContext(rawURL, question) }()
+	}
 	more := readSources(links, reader)
 	parts := []string{
 		"# Source-grounded context",
@@ -107,27 +108,91 @@ func readLinkFocused(rawURL, question string, reader func(string) (string, error
 		"## Primary source",
 		"**Source:** " + rawURL,
 		"",
-		focusExcerpt(primary, question, focusedPrimaryBudget),
 	}
+	primaryExcerpt := focusExcerpt(primary, question, focusedPrimaryBudget)
+	parts = append(parts, primaryExcerpt)
 	successful := 1
+	relatedSuccessful := 0
+	allRetrieved := primary
+	outputEvidence := primaryExcerpt
+	seenURLs := map[string]bool{rawURL: true}
 	for _, source := range more {
 		if source.err != nil {
 			continue
 		}
 		successful++
-		parts = append(parts, "", "## Related source", "**Source:** "+source.url, "", focusExcerpt(source.content, question, focusedRelatedBudget))
+		relatedSuccessful++
+		seenURLs[source.url] = true
+		allRetrieved += "\n" + source.content
+		excerpt := focusExcerpt(source.content, question, relatedExcerptBudget(source.url))
+		outputEvidence += "\n" + excerpt
+		parts = append(parts, "", "## Related source", "**Source:** "+source.url, "", excerpt)
+	}
+	// One additional bounded expansion if required code/API evidence remains
+	// missing. This follows references from the implementation pages we actually
+	// found, not arbitrary link hops across the entire site.
+	missing := missingResearchEvidence(question, allRetrieved)
+	if len(missing) > 0 {
+		var next []sourceCandidate
+		for _, source := range more {
+			if source.err != nil {
+				continue
+			}
+			if strings.Contains(strings.ToLower(source.content), "article has multiple variants") {
+				for _, candidate := range relatedSourceLinks(source.content, source.url, question, 8) {
+					if !seenURLs[candidate.url] {
+						seenURLs[candidate.url] = true
+						candidate.score += 50 // A published page variant is the actual documentation, not an index.
+						if strings.Contains(strings.ToLower(question), "hosted") &&
+							strings.Contains(strings.ToLower(candidate.url+" "+candidate.label), "hosted") {
+							candidate.score += 20
+						}
+						next = append(next, candidate)
+					}
+				}
+			}
+			for _, candidate := range relatedSourceLinks(source.content, source.url, strings.Join(missing, " "), 8) {
+				if !seenURLs[candidate.url] {
+					seenURLs[candidate.url] = true
+					candidate.score += implementationLinkBonus(candidate, question)
+					next = append(next, candidate)
+				}
+			}
+		}
+		sort.SliceStable(next, func(i, j int) bool { return next[i].score > next[j].score })
+		if len(next) > maxFollowupSources {
+			next = next[:maxFollowupSources]
+		}
+		followups := readSources(next, reader)
+		more = append(more, followups...)
+		for _, source := range followups {
+			if source.err != nil {
+				continue
+			}
+			successful++
+			relatedSuccessful++
+			allRetrieved += "\n" + source.content
+			excerpt := focusExcerpt(source.content, question, relatedExcerptBudget(source.url))
+			outputEvidence += "\n" + excerpt
+			parts = append(parts, "", "## Supporting implementation reference", "**Source:** "+source.url, "", excerpt)
+		}
 	}
 	if openAPIDone != nil {
 		if contract := <-openAPIDone; contract.Content != "" {
 			successful++
+			outputEvidence += "\n" + contract.Content
 			parts = append(parts, "", "## REST API contract", "**Source:** "+contract.URL, "", contract.Content)
 		}
 	}
 	if structured := <-structuredDone; structured != "" {
+		outputEvidence += "\n" + structured
 		parts = append(parts, "", structured)
 	}
+	if unmet := missingResearchEvidence(question, outputEvidence); len(unmet) > 0 {
+		parts = append(parts, "", "**Requested details not fully evidenced in returned excerpts:** "+strings.Join(unmet, ", ")+". Follow the cited source pages rather than assuming these details.")
+	}
 	parts = append(parts, "", fmt.Sprintf("**Coverage:** %d source document(s) inspected. Extracts are source-grounded, not an independently verified synthesis. REST commands assembled from an OpenAPI operation and its published example are labeled as such.", successful))
-	if failed := len(more) - (successful - 1); failed > 0 {
+	if failed := len(more) - relatedSuccessful; failed > 0 {
 		parts = append(parts, fmt.Sprintf("%d related page(s) could not be retrieved; the primary source is preserved.", failed))
 	}
 	return strings.Join(parts, "\n"), nil
@@ -184,24 +249,48 @@ func enrichSearchWithReader(query string, ranked []SearchDoc, reader func(string
 }
 
 func linksFromDocsIndex(rawURL, question string, limit int) []sourceCandidate {
+	return linksFromDocsIndexWithPrimary(rawURL, "", question, limit)
+}
+
+func linksFromDocsIndexWithPrimary(rawURL, primary, question string, limit int) []sourceCandidate {
 	u, err := url.Parse(rawURL)
 	if err != nil || u.Hostname() == "github.com" || directSourceLanguage(rawURL) != "" {
 		return nil
 	}
-	if !strings.HasPrefix(u.Hostname(), "docs.") && !strings.Contains(u.Path, "/docs/") && !strings.Contains(u.Path, "/reference/") {
+	if !strings.HasPrefix(u.Hostname(), "docs.") && !strings.Contains(u.Path, "/docs/") &&
+		!strings.Contains(u.Path, "/reference/") && !strings.Contains(primary, "llms.txt") {
 		return nil
 	}
-	indexURL := u.Scheme + "://" + u.Host + "/llms.txt"
-	if rawURL == indexURL {
-		return nil
+	indexURLs := documentIndexURLs(u, primary)
+	var candidates []sourceCandidate
+	seen := map[string]bool{}
+	for _, indexURL := range indexURLs {
+		if rawURL == indexURL {
+			continue
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		index, err := fetchPublicText(ctx, indexURL, 300000)
+		cancel()
+		if err != nil || !strings.Contains(index, "](") {
+			continue
+		}
+		for _, item := range relatedSourceLinks(index, indexURL, question, 300) {
+			item.score += implementationLinkBonus(item, question)
+			item.score += sourceScopeBonus(item, rawURL, question)
+			if !seen[item.url] {
+				seen[item.url] = true
+				candidates = append(candidates, item)
+			}
+		}
+		if len(candidates) >= limit {
+			break
+		}
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	index, err := fetchPublicText(ctx, indexURL, 300000)
-	if err != nil || !strings.Contains(index, "](") {
-		return nil
+	sort.SliceStable(candidates, func(i, j int) bool { return candidates[i].score > candidates[j].score })
+	if len(candidates) > limit {
+		candidates = candidates[:limit]
 	}
-	return relatedSourceLinks(index, indexURL, question, limit)
+	return candidates
 }
 
 func readSources(candidates []sourceCandidate, reader func(string) (string, error)) []sourceRead {
@@ -279,7 +368,7 @@ func relatedSourceLinks(markdown, baseURL, question string, limit int) []sourceC
 	terms := queryTerms(question)
 	seen := map[string]bool{canonicalSourceURL(base): true}
 	var links []sourceCandidate
-	for i, match := range mdLinkPattern.FindAllStringSubmatch(markdown, 300) {
+	for i, match := range mdLinkPattern.FindAllStringSubmatch(markdown, 1000) {
 		destination := strings.TrimSpace(match[2])
 		if strings.Contains(destination, "\\") { // malformed Markdown-escaped URLs
 			continue

package/internal/app/research_navigation.go

@@ -0,0 +1,237 @@
+package app
+
+import (
+	"net/url"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+var documentationIndexPattern = regexp.MustCompile("https?://[^\\s<>\"')\\]]+/llms\\.txt")
+
+func documentIndexURLs(base *url.URL, primary string) []string {
+	var urls []string
+	seen := map[string]bool{}
+	for _, raw := range documentationIndexPattern.FindAllString(primary, 5) {
+		target, err := url.Parse(raw)
+		if err != nil || validateSourceURL(raw) != nil ||
+			!strings.EqualFold(base.Hostname(), target.Hostname()) || target.Port() != base.Port() {
+			continue
+		}
+		if !seen[raw] {
+			urls = append(urls, raw)
+			seen[raw] = true
+		}
+	}
+	fallback := base.Scheme + "://" + base.Host + "/llms.txt"
+	if !seen[fallback] {
+		urls = append(urls, fallback)
+	}
+	return urls
+}
+
+func implementationQuestion(question string) bool {
+	text := strings.ToLower(question)
+	for _, signal := range []string{
+		"implement", "integrat", "runnable", "configuration",
+		"code example", "complete", "how to", "set up", "using a", "creating",
+	} {
+		if strings.Contains(text, signal) {
+			return true
+		}
+	}
+	return false
+}
+
+func implementationLinkBonus(candidate sourceCandidate, question string) int {
+	lower := strings.ToLower(candidate.url + " " + candidate.label)
+	q := strings.ToLower(question)
+	score := 0
+	if implementationQuestion(question) {
+		switch {
+		case strings.Contains(lower, "/get-started/"), strings.Contains(lower, "/quickstart"),
+			strings.Contains(lower, "getting started"):
+			score += 64 // Publisher's end-to-end guide, not isolated snippets.
+		case strings.Contains(lower, "tutorial"), strings.Contains(lower, "integration"),
+			strings.Contains(lower, "how-to"):
+			score += 16
+		case strings.Contains(lower, "example"):
+			score += 6
+		}
+	}
+	if strings.Contains(q, "sql") || strings.Contains(q, "sqlite") {
+		if strings.Contains(lower, "sqlite") || strings.Contains(lower, "storage") {
+			score += 26
+		} else if strings.Contains(lower, "/examples/") {
+			score -= 16
+		}
+	}
+	if strings.Contains(q, "webhook") || strings.Contains(q, "fulfill") || strings.Contains(q, "fulfil") {
+		if strings.Contains(lower, "webhook") || strings.Contains(lower, "fulfill") || strings.Contains(lower, "fulfil") {
+			score += 30
+		}
+	}
+	if strings.Contains(q, "checkout") && (strings.Contains(lower, "checkout") || strings.Contains(lower, "session")) {
+		score += 6
+	}
+	if strings.Contains(q, "signature") && strings.Contains(lower, "signature") {
+		score += 24
+	}
+	if strings.Contains(q, "curl") || strings.Contains(q, "rest") || strings.Contains(q, "endpoint") {
+		if strings.Contains(lower, "/api/") || strings.Contains(lower, "api-reference") {
+			score += 10
+		}
+	}
+	if strings.Contains(q, "session") && strings.Contains(q, "creat") &&
+		(strings.Contains(lower, "/sessions/create") || strings.Contains(lower, "create a checkout session")) {
+		score += 38
+	}
+	if strings.Contains(q, "checkout") && strings.Contains(lower, "/checkout/quickstart") {
+		score += 20
+	}
+	if strings.Contains(q, "fulfill") || strings.Contains(q, "fulfil") {
+		if strings.Contains(lower, "/checkout/fulfillment") {
+			score += 24
+		}
+	}
+	if strings.Contains(lower, "/alarms/") && !strings.Contains(q, "alarm") {
+		score -= 24
+	}
+	if strings.Contains(lower, "/concepts/") && implementationQuestion(question) {
+		score -= 12
+	}
+	if strings.Contains(lower, "legacy") && (strings.Contains(q, "new ") || strings.Contains(q, "current")) && !strings.Contains(q, "legacy") {
+		score -= 25
+	}
+	return score
+}
+
+// Search indices on a large documentation host often contain links for many
+// unrelated products. Prefer links in the original section or with a product
+// named in the question. This prevents "webhooks for Climate Orders" from
+// outranking generic Checkout webhooks solely because both contain "webhook".
+func sourceScopeBonus(candidate sourceCandidate, rawURL, question string) int {
+	base, err1 := url.Parse(rawURL)
+	target, err2 := url.Parse(candidate.url)
+	if err1 != nil || err2 != nil {
+		return 0
+	}
+	q := strings.ToLower(question)
+	sourceSegments := strings.Split(strings.Trim(base.Path, "/"), "/")
+	targetSegments := strings.Split(strings.Trim(target.Path, "/"), "/")
+	if len(targetSegments) == 0 {
+		return 0
+	}
+	score := 0
+	if len(sourceSegments) > 0 && targetSegments[0] == sourceSegments[0] {
+		score += 8
+	}
+	if len(targetSegments) > 0 && targetSegments[0] != "" &&
+		!strings.Contains(q, strings.TrimSuffix(targetSegments[0], "s")) &&
+		!strings.Contains(q, targetSegments[0]) &&
+		targetSegments[0] != "api" && targetSegments[0] != "webhooks" &&
+		targetSegments[0] != "events" &&
+		(len(sourceSegments) == 0 || targetSegments[0] != sourceSegments[0]) {
+		score -= 24
+	}
+	// A general Worker overview is less relevant than Durable Objects'
+	// focused setup instructions even if both mention "Worker".
+	if strings.Contains(q, "durable object") && strings.Contains(target.Path, "/workers/") &&
+		!strings.Contains(target.Path, "/durable-objects/") {
+		score -= 14
+	}
+	return score
+}
+
+func normalizedDocumentationURL(raw string) string {
+	u, err := url.Parse(raw)
+	if err != nil {
+		return raw
+	}
+	p := strings.TrimSuffix(u.Path, "/index.md")
+	p = strings.TrimSuffix(p, ".md")
+	p = strings.TrimSuffix(p, "/")
+	u.Path = p
+	u.Fragment = ""
+	u.RawFragment = ""
+	u.RawPath = ""
+	return u.String()
+}
+
+func researchCandidates(rawURL, primary, question string, limit int) []sourceCandidate {
+	baseCandidates := relatedSourceLinks(primary, rawURL, question, limit*4)
+	indexCandidates := linksFromDocsIndexWithPrimary(rawURL, primary, question, limit*5)
+	seen := map[string]bool{normalizedDocumentationURL(rawURL): true}
+	var candidates []sourceCandidate
+	for _, c := range append(baseCandidates, indexCandidates...) {
+		key := normalizedDocumentationURL(c.url)
+		if seen[key] {
+			continue
+		}
+		seen[key] = true
+		c.score += implementationLinkBonus(c, question)
+		c.score += sourceScopeBonus(c, rawURL, question)
+		candidates = append(candidates, c)
+	}
+	sort.SliceStable(candidates, func(i, j int) bool {
+		if candidates[i].score == candidates[j].score {
+			return candidates[i].order < candidates[j].order
+		}
+		return candidates[i].score > candidates[j].score
+	})
+	if len(candidates) > limit {
+		return candidates[:limit]
+	}
+	return candidates
+}
+
+type evidenceNeed struct {
+	name         string
+	alternatives []string
+}
+
+// Check for explicit source evidence; do not claim that a plausible guess
+// supplies missing request fields or code.
+func evidenceNeeds(question string) []evidenceNeed {
+	q := strings.ToLower(question)
+	var needs []evidenceNeed
+	for _, literal := range []string{"line_items", "success_url", "cancel_url", "sql.exec", "wrangler", "sqlite"} {
+		if strings.Contains(q, literal) {
+			needs = append(needs, evidenceNeed{literal, []string{literal}})
+		}
+	}
+	if strings.Contains(q, "webhook") {
+		needs = append(needs, evidenceNeed{"webhook events", []string{"webhook", "checkout.session.completed"}})
+		if strings.Contains(q, "signature") || strings.Contains(q, "secure") {
+			needs = append(needs, evidenceNeed{"webhook signature verification", []string{"stripe-signature", "constructevent", "signature verification", "signatureverification"}})
+		}
+	}
+	if strings.Contains(q, "sql table") || strings.Contains(q, "creating a sql") {
+		needs = append(needs, evidenceNeed{"SQL table creation", []string{"create table"}})
+	}
+	if strings.Contains(q, "accesses the binding") || strings.Contains(q, "access the binding") {
+		needs = append(needs, evidenceNeed{"Worker binding access", []string{"getbyname", "env.my_durable_object", "env.durable_object"}})
+	}
+	if strings.Contains(q, "checkout") && (strings.Contains(q, "fulfill") || strings.Contains(q, "fulfil")) {
+		needs = append(needs, evidenceNeed{"Checkout fulfillment event", []string{"checkout.session.completed"}})
+	}
+	return needs
+}
+
+func missingResearchEvidence(question, text string) []string {
+	lower := strings.ToLower(text)
+	var missing []string
+	for _, need := range evidenceNeeds(question) {
+		found := false
+		for _, candidate := range need.alternatives {
+			if strings.Contains(lower, candidate) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			missing = append(missing, need.name)
+		}
+	}
+	return missing
+}

package/internal/app/research_navigation_test.go

@@ -0,0 +1,72 @@
+package app
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func TestStripeFocusedLinksPreferCheckoutAndWebhookDocs(t *testing.T) {
+	old := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = old })
+	index := "# Documentation\n\n" +
+		"- [Climate order webhooks](https://docs.stripe.com/climate/orders/webhooks.md)\n" +
+		"- [Create a Checkout Session](https://docs.stripe.com/api/checkout/sessions/create.md)\n" +
+		"- [Build a Stripe-hosted checkout page](https://docs.stripe.com/checkout/quickstart.md)\n" +
+		"- [Set up and deploy a webhook](https://docs.stripe.com/webhooks/quickstart.md)\n" +
+		"- [Fulfill orders](https://docs.stripe.com/checkout/fulfillment.md)\n" +
+		"- [Verify webhook signatures](https://docs.stripe.com/webhooks.md#verify-signature)\n"
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		if req.URL.Path == "/llms.txt" {
+			return testHTTPResponse(req, http.StatusOK, index, nil), nil
+		}
+		return testHTTPResponse(req, http.StatusNotFound, "", nil), nil
+	})}
+	question := "Implement Stripe-hosted Checkout: create a Checkout Session REST request and securely fulfill orders using webhooks, including signature verification"
+	primary := "# Stripe Checkout\n\n[Start building your checkout integration](https://docs.stripe.com/checkout/quickstart.md)"
+	got := researchCandidates("https://docs.stripe.com/payments/checkout", primary, question, 5)
+	var links []string
+	for _, c := range got {
+		links = append(links, c.url)
+	}
+	output := strings.Join(links, "\n")
+	for _, want := range []string{
+		"/checkout/quickstart.md",
+		"/api/checkout/sessions/create.md",
+		"/webhooks/quickstart.md",
+		"/checkout/fulfillment.md",
+	} {
+		if !strings.Contains(output, want) {
+			t.Errorf("expected %q among selected source links: %s", want, output)
+		}
+	}
+	if strings.Contains(output, "/climate/") {
+		t.Errorf("irrelevant Climate-specific webhook outranked Checkout sources: %s", output)
+	}
+}
+
+func TestCloudflareNewSQLiteQuestionPrefersGettingStarted(t *testing.T) {
+	old := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = old })
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		return testHTTPResponse(req, http.StatusForbidden, "", nil), nil
+	})}
+	primary := "# Cloudflare Durable Objects\n" +
+		"[Durable Object alarms](https://developers.cloudflare.com/durable-objects/api/alarms/)\n" +
+		"[SQLite storage](https://developers.cloudflare.com/durable-objects/api/sqlite-storage-api/)\n" +
+		"[Get started](https://developers.cloudflare.com/durable-objects/get-started/)\n" +
+		"> Documentation Index https://developers.cloudflare.com/durable-objects/llms.txt\n"
+	question := "For a new Cloudflare Worker using a SQLite-backed Durable Object, show complete wrangler configuration, Worker binding, and SQL table code."
+	got := researchCandidates("https://developers.cloudflare.com/durable-objects/", primary, question, 2)
+	if len(got) != 2 {
+		t.Fatalf("unexpected number of selected references: %+v", got)
+	}
+	var links []string
+	for _, c := range got {
+		links = append(links, c.url)
+	}
+	output := strings.Join(links, "\n")
+	if !strings.Contains(output, "/get-started/") || strings.Contains(output, "/alarms/") {
+		t.Fatalf("unrelated alarm material displaced getting-started guide: %s", output)
+	}
+}

package/internal/app/research_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
+	"sync"
 	"testing"
 )
 
@@ -35,8 +36,11 @@ func TestFocusedReadExpandsRelevantSameHostLinks(t *testing.T) {
 		"https://docs.example.com/pagination": "# Pagination\nPass next_cursor to read the next page.",
 	}
 	requested := make(map[string]int)
+	var requestedMu sync.Mutex
 	reader := func(url string) (string, error) {
+		requestedMu.Lock()
 		requested[url]++
+		requestedMu.Unlock()
 		value, ok := urls[url]
 		if !ok {
 			return "", errors.New("missing URL")

package/internal/app/retrieval_quality.go

@@ -0,0 +1,106 @@
+package app
+
+import (
+	"fmt"
+	"log"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+)
+
+// CAPTCHA pages can contain thousands of bytes of translated prompts. Page
+// length is not evidence of useful content, so challenge detection must not
+// be gated on the size of the extracted Markdown.
+func challengeReason(markdown string) string {
+	text := strings.ToLower(strings.TrimSpace(markdown))
+	if text == "" {
+		return "empty source"
+	}
+	prefix := text[:min(len(text), 4000)]
+	for _, marker := range []string{
+		"just a moment...",
+		"checking your browser",
+		"access denied | cloudflare",
+		"attention required! | cloudflare",
+		"enable javascript and cookies",
+		"no content extracted",
+		"please complete the security check",
+		"verify you are human",
+		"verifying you are human",
+		"verify that you are human",
+		"please verify you are a human",
+		"we need to verify your browser",
+		"challenge-platform",
+	} {
+		if strings.Contains(prefix, marker) {
+			return "browser verification challenge"
+		}
+	}
+	if strings.Contains(prefix, "hcaptcha") || strings.Contains(prefix, "h-captcha") {
+		for _, secondary := range []string{
+			"select all", "please verify", "i am human",
+			"privacy - terms", "please complete", "security check",
+			"captcha challenge", "verify your", "please solve",
+		} {
+			if strings.Contains(text, secondary) {
+				return "hCaptcha challenge"
+			}
+		}
+	}
+	if (strings.Count(text, "captcha") > 2 || strings.Count(text, "verify you are human") > 0) &&
+		(strings.Contains(text, "try again") || strings.Contains(text, "challenge") || strings.Contains(text, "i am human")) {
+		return "CAPTCHA challenge"
+	}
+	return ""
+}
+
+func usableSourceContent(markdown string) bool {
+	text := strings.TrimSpace(markdown)
+	if len(text) < 10 || challengeReason(text) != "" {
+		return false
+	}
+	// Guard against a 200 status containing an HTML block/error page.
+	prefix := strings.ToLower(text[:min(len(text), 350)])
+	return !strings.Contains(prefix, "<!doctype html") &&
+		!strings.Contains(prefix, "<html")
+}
+
+// WEBCTX_DEBUG=1 enables stage traces. Only host + URL path are included;
+// query parameters and credentials are intentionally not logged.
+func traceRetrieval(rawURL, stage, outcome string, start time.Time, err error) {
+	if os.Getenv("WEBCTX_DEBUG") != "1" {
+		return
+	}
+	source := "invalid-url"
+	if u, parseErr := url.Parse(rawURL); parseErr == nil && u.Hostname() != "" {
+		source = u.Hostname() + u.EscapedPath()
+	}
+	detail := ""
+	if err != nil {
+		detail = fmt.Sprintf(" error=%q", sanitizedRetrievalError(err))
+	}
+	log.Printf("webctx retrieval source=%q stage=%q outcome=%q duration_ms=%d%s",
+		source, stage, outcome, time.Since(start).Milliseconds(), detail)
+}
+
+func sanitizedRetrievalError(err error) string {
+	if err == nil {
+		return ""
+	}
+	message := strings.ToLower(err.Error())
+	switch {
+	case strings.Contains(message, "403"), strings.Contains(message, "forbidden"):
+		return "forbidden"
+	case strings.Contains(message, "429"), strings.Contains(message, "rate limit"):
+		return "rate-limited"
+	case strings.Contains(message, "timeout"), strings.Contains(message, "deadline"):
+		return "timeout"
+	case strings.Contains(message, "captcha"), strings.Contains(message, "challenge"):
+		return "challenge"
+	case strings.Contains(message, "404"), strings.Contains(message, "not found"):
+		return "not-found"
+	default:
+		return "source-unavailable"
+	}
+}

package/internal/app/retrieval_quality_test.go

@@ -0,0 +1,63 @@
+package app
+
+import (
+	"bytes"
+	"errors"
+	"log"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestLongCAPTCHAPayloadCannotPassAsDocumentation(t *testing.T) {
+	longChallenge := "hCaptcha - Please verify you are human. " +
+		strings.Repeat("Deutsch Français 日本語 中文 Hindi Español English ", 1000)
+	if usableScrapedMarkdown(longChallenge) || usableNativeMarkdown(longChallenge) {
+		t.Fatal("large CAPTCHA payload must be rejected even when >2KB")
+	}
+	if challengeReason(longChallenge) != "browser verification challenge" {
+		t.Fatalf("missing browser challenge reason: %q", challengeReason(longChallenge))
+	}
+}
+
+func TestNormalDocumentationAboutCAPTCHAsIsNotAutomaticallyBlocked(t *testing.T) {
+	doc := "# hCaptcha integration guide\n\nUse the hCaptcha SDK to add a verification widget to your website.\n" +
+		strings.Repeat("Install the client library and validate tokens on your server.\n", 50)
+	if !usableScrapedMarkdown(doc) {
+		t.Fatal("ordinary documentation about captcha should not be rejected")
+	}
+}
+
+func TestExplicitEvidenceGapReportsMissingFields(t *testing.T) {
+	q := "Create a Checkout Session with line_items, mode, success_url and cancel_url and explain webhook signature verification"
+	excerpts := "Use line_items and success_url in a session. Verify webhooks using Stripe-Signature."
+	gaps := missingResearchEvidence(q, excerpts)
+	if !strings.Contains(strings.Join(gaps, ","), "cancel_url") {
+		t.Fatalf("missing required request field was not reported: %+v", gaps)
+	}
+}
+
+func TestEmbeddedJSONExampleDoesNotBypassExcerptBudget(t *testing.T) {
+	doc := "**URL:** https://example.com/guide\n\n# Guide\n" +
+		strings.Repeat("This guide describes a public API and its fields.\n", 1000) +
+		"\n" + backtickFence + "json\n" + `{"example":{"id":"ok"}}` + "\n" + backtickFence
+	out := focusExcerpt(doc, "public API fields", 3500)
+	if len(out) > 4000 {
+		t.Fatalf("returned source bypassed excerpt budget: size=%d", len(out))
+	}
+}
+
+func TestRetrievalDebugTraceDoesNotLogCredentials(t *testing.T) {
+	original := log.Writer()
+	t.Cleanup(func() { log.SetOutput(original) })
+	var output bytes.Buffer
+	log.SetOutput(&output)
+	t.Setenv("WEBCTX_DEBUG", "1")
+	traceRetrieval("https://docs.example.com/guide?api_key=secret-value&token=sensitive",
+		"browser", "blocked", time.Now(), errors.New("unauthorized token sensitive"))
+	result := output.String()
+	if !strings.Contains(result, `stage="browser"`) || !strings.Contains(result, `error="source-unavailable"`) ||
+		strings.Contains(result, "secret-value") || strings.Contains(result, "sensitive") {
+		t.Fatalf("debug trace leaked query parameters or credentials: %s", result)
+	}
+}

package/internal/app/scrape.go

@@ -2,9 +2,6 @@ package app
 
 import (
 	"context"
-	"fmt"
-	"io"
-	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -15,9 +12,10 @@ import (
 )
 
 type MarkdownResult struct {
-	URL      string
-	Title    string
-	Markdown string
+	URL       string
+	SourceURL string
+	Title     string
+	Markdown  string
 }
 
 const keychainServiceName = "webctx"
@@ -29,65 +27,6 @@ var (
 	keychainLookup    = lookupKeychainSecret
 )
 
-func checkMarkdownAvailable(raw string) (bool, error) {
-	mdURL := raw
-	if !strings.HasSuffix(strings.ToLower(mdURL), ".md") {
-		mdURL += ".md"
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	req, err := http.NewRequestWithContext(ctx, http.MethodHead, mdURL, nil)
-	if err != nil {
-		return false, err
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return false, err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return false, nil
-	}
-	contentType := strings.ToLower(resp.Header.Get("Content-Type"))
-	contentLength := resp.ContentLength
-	return (strings.Contains(contentType, "markdown") || strings.Contains(contentType, "text/plain")) && contentLength > 50, nil
-}
-
-func fetchMarkdownContent(raw string) (*MarkdownResult, error) {
-	mdURL := raw
-	if !strings.HasSuffix(strings.ToLower(mdURL), ".md") {
-		mdURL += ".md"
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	content, status, err := fetchText(ctx, mdURL)
-	if err != nil {
-		return nil, err
-	}
-	if status < 200 || status >= 300 {
-		return nil, fmt.Errorf("Failed to fetch markdown: %d", status)
-	}
-	title := firstHeadingOrFallback(content, filepath.Base(raw))
-	return &MarkdownResult{URL: raw, Title: title, Markdown: content}, nil
-}
-
-func fetchText(ctx context.Context, rawURL string) (string, int, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
-	if err != nil {
-		return "", 0, err
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", 0, err
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return "", resp.StatusCode, err
-	}
-	return string(body), resp.StatusCode, nil
-}
-
 func firstHeadingOrFallback(markdown, fallback string) string {
 	for _, line := range strings.Split(markdown, "\n") {
 		trimmed := strings.TrimSpace(line)

package/internal/app/structured_source.go

@@ -228,10 +228,20 @@ func selectedJSONFields(raw any, question string, budget int) string {
 }
 
 func focusedRawJSON(markdown, question string, budget int) string {
+	// Raw JSON documents have one JSON fence just after the URL. Looking for
+	// JSON examples in a long Markdown guide previously allowed the entire
+	// preceding guide to escape the excerpt budget.
+	if !strings.HasPrefix(strings.TrimSpace(markdown), "**URL:**") {
+		return ""
+	}
 	begin := strings.Index(markdown, "```json\n")
 	if begin < 0 {
 		return ""
 	}
+	prefix := strings.TrimSpace(markdown[:begin])
+	if len(prefix) > 400 || strings.Contains(prefix, "\n#") {
+		return ""
+	}
 	jsonStart := begin + len("```json\n")
 	end := strings.LastIndex(markdown, "\n```")
 	if end <= jsonStart {

package/internal/app/tools.go

@@ -171,12 +171,12 @@ func ReadLink(rawURL string) (string, error) {
 		return direct, nil
 	}
 
-	if ok, _ := checkMarkdownAvailable(rawURL); ok {
-		result, err := fetchMarkdownContent(rawURL)
-		if err == nil {
-			return formatReadLink(result.Title, result.URL, result.Markdown), nil
-		}
+	nativeStarted := time.Now()
+	if result, err := readNativeMarkdown(rawURL); err == nil && result != nil {
+		traceRetrieval(rawURL, "native-markdown", "success", nativeStarted, nil)
+		return formatReadLink(result.Title, result.URL, "**Markdown source:** "+result.SourceURL+"\n\n"+result.Markdown), nil
 	}
+	traceRetrieval(rawURL, "native-markdown", "miss", nativeStarted, nil)
 
 	title, markdown, err := scrapeLinkWithFirecrawl(rawURL)
 	if err != nil {
@@ -217,28 +217,37 @@ func scrapeLinkWithFirecrawl(rawURL string) (string, string, error) {
 	// Firecrawl's default auto proxy already tries basic then enhanced when the
 	// transport fails. A second explicit enhanced attempt is only useful when a
 	// scrape responds successfully but returns a challenge/empty document.
+	started := time.Now()
 	first, err := firecrawlScrape(rawURL, apiKey, "")
 	if err == nil && usableScrapedMarkdown(first.Markdown) {
+		traceRetrieval(rawURL, "firecrawl-auto", "success", started, nil)
 		return first.Title, withSourceTools(first.Markdown, first.ToolSummary), nil
 	}
 	lastError := err
+	traceRetrieval(rawURL, "firecrawl-auto", "escalate", started, err)
 	if err == nil {
-		lastError = fmt.Errorf("Firecrawl returned empty or blocked page content")
+		lastError = fmt.Errorf("Firecrawl returned empty or blocked page content: %s", challengeReason(first.Markdown))
+		started = time.Now()
 		second, enhancedErr := firecrawlScrape(rawURL, apiKey, "enhanced")
 		if enhancedErr == nil && usableScrapedMarkdown(second.Markdown) {
+			traceRetrieval(rawURL, "firecrawl-enhanced", "success", started, nil)
 			return second.Title, withSourceTools(second.Markdown, second.ToolSummary), nil
 		}
+		traceRetrieval(rawURL, "firecrawl-enhanced", "escalate", started, enhancedErr)
 		if enhancedErr != nil {
 			lastError = enhancedErr
 		}
 	}
+	started = time.Now()
 	title, markdown, browserErr := firecrawlBrowserFallback(rawURL, apiKey)
 	if browserErr == nil && usableScrapedMarkdown(markdown) {
+		traceRetrieval(rawURL, "browser", "success", started, nil)
 		return title, markdown, nil
 	}
 	if browserErr == nil {
-		browserErr = fmt.Errorf("browser produced no usable content")
+		browserErr = fmt.Errorf("browser produced no usable content: %s", challengeReason(markdown))
 	}
+	traceRetrieval(rawURL, "browser", "blocked", started, browserErr)
 	return "", "", fmt.Errorf("Error reading web page: scrape: %v; browser fallback: %w", lastError, browserErr)
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webctx",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "description": "Pure Go web search and browsing CLI using Brave, Tavily, Exa, and Firecrawl",
   "license": "MIT",
   "author": "amxv",