webctx 0.2.3 → 0.2.5

package/internal/app/research_navigation.go

@@ -0,0 +1,237 @@
+package app
+
+import (
+	"net/url"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+var documentationIndexPattern = regexp.MustCompile("https?://[^\\s<>\"')\\]]+/llms\\.txt")
+
+func documentIndexURLs(base *url.URL, primary string) []string {
+	var urls []string
+	seen := map[string]bool{}
+	for _, raw := range documentationIndexPattern.FindAllString(primary, 5) {
+		target, err := url.Parse(raw)
+		if err != nil || validateSourceURL(raw) != nil ||
+			!strings.EqualFold(base.Hostname(), target.Hostname()) || target.Port() != base.Port() {
+			continue
+		}
+		if !seen[raw] {
+			urls = append(urls, raw)
+			seen[raw] = true
+		}
+	}
+	fallback := base.Scheme + "://" + base.Host + "/llms.txt"
+	if !seen[fallback] {
+		urls = append(urls, fallback)
+	}
+	return urls
+}
+
+func implementationQuestion(question string) bool {
+	text := strings.ToLower(question)
+	for _, signal := range []string{
+		"implement", "integrat", "runnable", "configuration",
+		"code example", "complete", "how to", "set up", "using a", "creating",
+	} {
+		if strings.Contains(text, signal) {
+			return true
+		}
+	}
+	return false
+}
+
+func implementationLinkBonus(candidate sourceCandidate, question string) int {
+	lower := strings.ToLower(candidate.url + " " + candidate.label)
+	q := strings.ToLower(question)
+	score := 0
+	if implementationQuestion(question) {
+		switch {
+		case strings.Contains(lower, "/get-started/"), strings.Contains(lower, "/quickstart"),
+			strings.Contains(lower, "getting started"):
+			score += 64 // Publisher's end-to-end guide, not isolated snippets.
+		case strings.Contains(lower, "tutorial"), strings.Contains(lower, "integration"),
+			strings.Contains(lower, "how-to"):
+			score += 16
+		case strings.Contains(lower, "example"):
+			score += 6
+		}
+	}
+	if strings.Contains(q, "sql") || strings.Contains(q, "sqlite") {
+		if strings.Contains(lower, "sqlite") || strings.Contains(lower, "storage") {
+			score += 26
+		} else if strings.Contains(lower, "/examples/") {
+			score -= 16
+		}
+	}
+	if strings.Contains(q, "webhook") || strings.Contains(q, "fulfill") || strings.Contains(q, "fulfil") {
+		if strings.Contains(lower, "webhook") || strings.Contains(lower, "fulfill") || strings.Contains(lower, "fulfil") {
+			score += 30
+		}
+	}
+	if strings.Contains(q, "checkout") && (strings.Contains(lower, "checkout") || strings.Contains(lower, "session")) {
+		score += 6
+	}
+	if strings.Contains(q, "signature") && strings.Contains(lower, "signature") {
+		score += 24
+	}
+	if strings.Contains(q, "curl") || strings.Contains(q, "rest") || strings.Contains(q, "endpoint") {
+		if strings.Contains(lower, "/api/") || strings.Contains(lower, "api-reference") {
+			score += 10
+		}
+	}
+	if strings.Contains(q, "session") && strings.Contains(q, "creat") &&
+		(strings.Contains(lower, "/sessions/create") || strings.Contains(lower, "create a checkout session")) {
+		score += 38
+	}
+	if strings.Contains(q, "checkout") && strings.Contains(lower, "/checkout/quickstart") {
+		score += 20
+	}
+	if strings.Contains(q, "fulfill") || strings.Contains(q, "fulfil") {
+		if strings.Contains(lower, "/checkout/fulfillment") {
+			score += 24
+		}
+	}
+	if strings.Contains(lower, "/alarms/") && !strings.Contains(q, "alarm") {
+		score -= 24
+	}
+	if strings.Contains(lower, "/concepts/") && implementationQuestion(question) {
+		score -= 12
+	}
+	if strings.Contains(lower, "legacy") && (strings.Contains(q, "new ") || strings.Contains(q, "current")) && !strings.Contains(q, "legacy") {
+		score -= 25
+	}
+	return score
+}
+
+// Search indices on a large documentation host often contain links for many
+// unrelated products. Prefer links in the original section or with a product
+// named in the question. This prevents "webhooks for Climate Orders" from
+// outranking generic Checkout webhooks solely because both contain "webhook".
+func sourceScopeBonus(candidate sourceCandidate, rawURL, question string) int {
+	base, err1 := url.Parse(rawURL)
+	target, err2 := url.Parse(candidate.url)
+	if err1 != nil || err2 != nil {
+		return 0
+	}
+	q := strings.ToLower(question)
+	sourceSegments := strings.Split(strings.Trim(base.Path, "/"), "/")
+	targetSegments := strings.Split(strings.Trim(target.Path, "/"), "/")
+	if len(targetSegments) == 0 {
+		return 0
+	}
+	score := 0
+	if len(sourceSegments) > 0 && targetSegments[0] == sourceSegments[0] {
+		score += 8
+	}
+	if len(targetSegments) > 0 && targetSegments[0] != "" &&
+		!strings.Contains(q, strings.TrimSuffix(targetSegments[0], "s")) &&
+		!strings.Contains(q, targetSegments[0]) &&
+		targetSegments[0] != "api" && targetSegments[0] != "webhooks" &&
+		targetSegments[0] != "events" &&
+		(len(sourceSegments) == 0 || targetSegments[0] != sourceSegments[0]) {
+		score -= 24
+	}
+	// A general Worker overview is less relevant than Durable Objects'
+	// focused setup instructions even if both mention "Worker".
+	if strings.Contains(q, "durable object") && strings.Contains(target.Path, "/workers/") &&
+		!strings.Contains(target.Path, "/durable-objects/") {
+		score -= 14
+	}
+	return score
+}
+
+func normalizedDocumentationURL(raw string) string {
+	u, err := url.Parse(raw)
+	if err != nil {
+		return raw
+	}
+	p := strings.TrimSuffix(u.Path, "/index.md")
+	p = strings.TrimSuffix(p, ".md")
+	p = strings.TrimSuffix(p, "/")
+	u.Path = p
+	u.Fragment = ""
+	u.RawFragment = ""
+	u.RawPath = ""
+	return u.String()
+}
+
+func researchCandidates(rawURL, primary, question string, limit int) []sourceCandidate {
+	baseCandidates := relatedSourceLinks(primary, rawURL, question, limit*4)
+	indexCandidates := linksFromDocsIndexWithPrimary(rawURL, primary, question, limit*5)
+	seen := map[string]bool{normalizedDocumentationURL(rawURL): true}
+	var candidates []sourceCandidate
+	for _, c := range append(baseCandidates, indexCandidates...) {
+		key := normalizedDocumentationURL(c.url)
+		if seen[key] {
+			continue
+		}
+		seen[key] = true
+		c.score += implementationLinkBonus(c, question)
+		c.score += sourceScopeBonus(c, rawURL, question)
+		candidates = append(candidates, c)
+	}
+	sort.SliceStable(candidates, func(i, j int) bool {
+		if candidates[i].score == candidates[j].score {
+			return candidates[i].order < candidates[j].order
+		}
+		return candidates[i].score > candidates[j].score
+	})
+	if len(candidates) > limit {
+		return candidates[:limit]
+	}
+	return candidates
+}
+
+type evidenceNeed struct {
+	name         string
+	alternatives []string
+}
+
+// Check for explicit source evidence; do not claim that a plausible guess
+// supplies missing request fields or code.
+func evidenceNeeds(question string) []evidenceNeed {
+	q := strings.ToLower(question)
+	var needs []evidenceNeed
+	for _, literal := range []string{"line_items", "success_url", "cancel_url", "sql.exec", "wrangler", "sqlite"} {
+		if strings.Contains(q, literal) {
+			needs = append(needs, evidenceNeed{literal, []string{literal}})
+		}
+	}
+	if strings.Contains(q, "webhook") {
+		needs = append(needs, evidenceNeed{"webhook events", []string{"webhook", "checkout.session.completed"}})
+		if strings.Contains(q, "signature") || strings.Contains(q, "secure") {
+			needs = append(needs, evidenceNeed{"webhook signature verification", []string{"stripe-signature", "constructevent", "signature verification", "signatureverification"}})
+		}
+	}
+	if strings.Contains(q, "sql table") || strings.Contains(q, "creating a sql") {
+		needs = append(needs, evidenceNeed{"SQL table creation", []string{"create table"}})
+	}
+	if strings.Contains(q, "accesses the binding") || strings.Contains(q, "access the binding") {
+		needs = append(needs, evidenceNeed{"Worker binding access", []string{"getbyname", "env.my_durable_object", "env.durable_object"}})
+	}
+	if strings.Contains(q, "checkout") && (strings.Contains(q, "fulfill") || strings.Contains(q, "fulfil")) {
+		needs = append(needs, evidenceNeed{"Checkout fulfillment event", []string{"checkout.session.completed"}})
+	}
+	return needs
+}
+
+func missingResearchEvidence(question, text string) []string {
+	lower := strings.ToLower(text)
+	var missing []string
+	for _, need := range evidenceNeeds(question) {
+		found := false
+		for _, candidate := range need.alternatives {
+			if strings.Contains(lower, candidate) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			missing = append(missing, need.name)
+		}
+	}
+	return missing
+}

package/internal/app/research_navigation_test.go

@@ -0,0 +1,72 @@
+package app
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func TestStripeFocusedLinksPreferCheckoutAndWebhookDocs(t *testing.T) {
+	old := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = old })
+	index := "# Documentation\n\n" +
+		"- [Climate order webhooks](https://docs.stripe.com/climate/orders/webhooks.md)\n" +
+		"- [Create a Checkout Session](https://docs.stripe.com/api/checkout/sessions/create.md)\n" +
+		"- [Build a Stripe-hosted checkout page](https://docs.stripe.com/checkout/quickstart.md)\n" +
+		"- [Set up and deploy a webhook](https://docs.stripe.com/webhooks/quickstart.md)\n" +
+		"- [Fulfill orders](https://docs.stripe.com/checkout/fulfillment.md)\n" +
+		"- [Verify webhook signatures](https://docs.stripe.com/webhooks.md#verify-signature)\n"
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		if req.URL.Path == "/llms.txt" {
+			return testHTTPResponse(req, http.StatusOK, index, nil), nil
+		}
+		return testHTTPResponse(req, http.StatusNotFound, "", nil), nil
+	})}
+	question := "Implement Stripe-hosted Checkout: create a Checkout Session REST request and securely fulfill orders using webhooks, including signature verification"
+	primary := "# Stripe Checkout\n\n[Start building your checkout integration](https://docs.stripe.com/checkout/quickstart.md)"
+	got := researchCandidates("https://docs.stripe.com/payments/checkout", primary, question, 5)
+	var links []string
+	for _, c := range got {
+		links = append(links, c.url)
+	}
+	output := strings.Join(links, "\n")
+	for _, want := range []string{
+		"/checkout/quickstart.md",
+		"/api/checkout/sessions/create.md",
+		"/webhooks/quickstart.md",
+		"/checkout/fulfillment.md",
+	} {
+		if !strings.Contains(output, want) {
+			t.Errorf("expected %q among selected source links: %s", want, output)
+		}
+	}
+	if strings.Contains(output, "/climate/") {
+		t.Errorf("irrelevant Climate-specific webhook outranked Checkout sources: %s", output)
+	}
+}
+
+func TestCloudflareNewSQLiteQuestionPrefersGettingStarted(t *testing.T) {
+	old := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = old })
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		return testHTTPResponse(req, http.StatusForbidden, "", nil), nil
+	})}
+	primary := "# Cloudflare Durable Objects\n" +
+		"[Durable Object alarms](https://developers.cloudflare.com/durable-objects/api/alarms/)\n" +
+		"[SQLite storage](https://developers.cloudflare.com/durable-objects/api/sqlite-storage-api/)\n" +
+		"[Get started](https://developers.cloudflare.com/durable-objects/get-started/)\n" +
+		"> Documentation Index https://developers.cloudflare.com/durable-objects/llms.txt\n"
+	question := "For a new Cloudflare Worker using a SQLite-backed Durable Object, show complete wrangler configuration, Worker binding, and SQL table code."
+	got := researchCandidates("https://developers.cloudflare.com/durable-objects/", primary, question, 2)
+	if len(got) != 2 {
+		t.Fatalf("unexpected number of selected references: %+v", got)
+	}
+	var links []string
+	for _, c := range got {
+		links = append(links, c.url)
+	}
+	output := strings.Join(links, "\n")
+	if !strings.Contains(output, "/get-started/") || strings.Contains(output, "/alarms/") {
+		t.Fatalf("unrelated alarm material displaced getting-started guide: %s", output)
+	}
+}

package/internal/app/research_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
+	"sync"
 	"testing"
 )
 
@@ -35,8 +36,11 @@ func TestFocusedReadExpandsRelevantSameHostLinks(t *testing.T) {
 		"https://docs.example.com/pagination": "# Pagination\nPass next_cursor to read the next page.",
 	}
 	requested := make(map[string]int)
+	var requestedMu sync.Mutex
 	reader := func(url string) (string, error) {
+		requestedMu.Lock()
 		requested[url]++
+		requestedMu.Unlock()
 		value, ok := urls[url]
 		if !ok {
 			return "", errors.New("missing URL")

package/internal/app/retrieval_quality.go

@@ -0,0 +1,106 @@
+package app
+
+import (
+	"fmt"
+	"log"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+)
+
+// CAPTCHA pages can contain thousands of bytes of translated prompts. Page
+// length is not evidence of useful content, so challenge detection must not
+// be gated on the size of the extracted Markdown.
+func challengeReason(markdown string) string {
+	text := strings.ToLower(strings.TrimSpace(markdown))
+	if text == "" {
+		return "empty source"
+	}
+	prefix := text[:min(len(text), 4000)]
+	for _, marker := range []string{
+		"just a moment...",
+		"checking your browser",
+		"access denied | cloudflare",
+		"attention required! | cloudflare",
+		"enable javascript and cookies",
+		"no content extracted",
+		"please complete the security check",
+		"verify you are human",
+		"verifying you are human",
+		"verify that you are human",
+		"please verify you are a human",
+		"we need to verify your browser",
+		"challenge-platform",
+	} {
+		if strings.Contains(prefix, marker) {
+			return "browser verification challenge"
+		}
+	}
+	if strings.Contains(prefix, "hcaptcha") || strings.Contains(prefix, "h-captcha") {
+		for _, secondary := range []string{
+			"select all", "please verify", "i am human",
+			"privacy - terms", "please complete", "security check",
+			"captcha challenge", "verify your", "please solve",
+		} {
+			if strings.Contains(text, secondary) {
+				return "hCaptcha challenge"
+			}
+		}
+	}
+	if (strings.Count(text, "captcha") > 2 || strings.Count(text, "verify you are human") > 0) &&
+		(strings.Contains(text, "try again") || strings.Contains(text, "challenge") || strings.Contains(text, "i am human")) {
+		return "CAPTCHA challenge"
+	}
+	return ""
+}
+
+func usableSourceContent(markdown string) bool {
+	text := strings.TrimSpace(markdown)
+	if len(text) < 10 || challengeReason(text) != "" {
+		return false
+	}
+	// Guard against a 200 status containing an HTML block/error page.
+	prefix := strings.ToLower(text[:min(len(text), 350)])
+	return !strings.Contains(prefix, "<!doctype html") &&
+		!strings.Contains(prefix, "<html")
+}
+
+// WEBCTX_DEBUG=1 enables stage traces. Only host + URL path are included;
+// query parameters and credentials are intentionally not logged.
+func traceRetrieval(rawURL, stage, outcome string, start time.Time, err error) {
+	if os.Getenv("WEBCTX_DEBUG") != "1" {
+		return
+	}
+	source := "invalid-url"
+	if u, parseErr := url.Parse(rawURL); parseErr == nil && u.Hostname() != "" {
+		source = u.Hostname() + u.EscapedPath()
+	}
+	detail := ""
+	if err != nil {
+		detail = fmt.Sprintf(" error=%q", sanitizedRetrievalError(err))
+	}
+	log.Printf("webctx retrieval source=%q stage=%q outcome=%q duration_ms=%d%s",
+		source, stage, outcome, time.Since(start).Milliseconds(), detail)
+}
+
+func sanitizedRetrievalError(err error) string {
+	if err == nil {
+		return ""
+	}
+	message := strings.ToLower(err.Error())
+	switch {
+	case strings.Contains(message, "403"), strings.Contains(message, "forbidden"):
+		return "forbidden"
+	case strings.Contains(message, "429"), strings.Contains(message, "rate limit"):
+		return "rate-limited"
+	case strings.Contains(message, "timeout"), strings.Contains(message, "deadline"):
+		return "timeout"
+	case strings.Contains(message, "captcha"), strings.Contains(message, "challenge"):
+		return "challenge"
+	case strings.Contains(message, "404"), strings.Contains(message, "not found"):
+		return "not-found"
+	default:
+		return "source-unavailable"
+	}
+}

package/internal/app/retrieval_quality_test.go

@@ -0,0 +1,63 @@
+package app
+
+import (
+	"bytes"
+	"errors"
+	"log"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestLongCAPTCHAPayloadCannotPassAsDocumentation(t *testing.T) {
+	longChallenge := "hCaptcha - Please verify you are human. " +
+		strings.Repeat("Deutsch Français 日本語 中文 Hindi Español English ", 1000)
+	if usableScrapedMarkdown(longChallenge) || usableNativeMarkdown(longChallenge) {
+		t.Fatal("large CAPTCHA payload must be rejected even when >2KB")
+	}
+	if challengeReason(longChallenge) != "browser verification challenge" {
+		t.Fatalf("missing browser challenge reason: %q", challengeReason(longChallenge))
+	}
+}
+
+func TestNormalDocumentationAboutCAPTCHAsIsNotAutomaticallyBlocked(t *testing.T) {
+	doc := "# hCaptcha integration guide\n\nUse the hCaptcha SDK to add a verification widget to your website.\n" +
+		strings.Repeat("Install the client library and validate tokens on your server.\n", 50)
+	if !usableScrapedMarkdown(doc) {
+		t.Fatal("ordinary documentation about captcha should not be rejected")
+	}
+}
+
+func TestExplicitEvidenceGapReportsMissingFields(t *testing.T) {
+	q := "Create a Checkout Session with line_items, mode, success_url and cancel_url and explain webhook signature verification"
+	excerpts := "Use line_items and success_url in a session. Verify webhooks using Stripe-Signature."
+	gaps := missingResearchEvidence(q, excerpts)
+	if !strings.Contains(strings.Join(gaps, ","), "cancel_url") {
+		t.Fatalf("missing required request field was not reported: %+v", gaps)
+	}
+}
+
+func TestEmbeddedJSONExampleDoesNotBypassExcerptBudget(t *testing.T) {
+	doc := "**URL:** https://example.com/guide\n\n# Guide\n" +
+		strings.Repeat("This guide describes a public API and its fields.\n", 1000) +
+		"\n" + backtickFence + "json\n" + `{"example":{"id":"ok"}}` + "\n" + backtickFence
+	out := focusExcerpt(doc, "public API fields", 3500)
+	if len(out) > 4000 {
+		t.Fatalf("returned source bypassed excerpt budget: size=%d", len(out))
+	}
+}
+
+func TestRetrievalDebugTraceDoesNotLogCredentials(t *testing.T) {
+	original := log.Writer()
+	t.Cleanup(func() { log.SetOutput(original) })
+	var output bytes.Buffer
+	log.SetOutput(&output)
+	t.Setenv("WEBCTX_DEBUG", "1")
+	traceRetrieval("https://docs.example.com/guide?api_key=secret-value&token=sensitive",
+		"browser", "blocked", time.Now(), errors.New("unauthorized token sensitive"))
+	result := output.String()
+	if !strings.Contains(result, `stage="browser"`) || !strings.Contains(result, `error="source-unavailable"`) ||
+		strings.Contains(result, "secret-value") || strings.Contains(result, "sensitive") {
+		t.Fatalf("debug trace leaked query parameters or credentials: %s", result)
+	}
+}

package/internal/app/scrape.go

@@ -2,9 +2,6 @@ package app
 
 import (
 	"context"
-	"fmt"
-	"io"
-	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -15,9 +12,10 @@ import (
 )
 
 type MarkdownResult struct {
-	URL      string
-	Title    string
-	Markdown string
+	URL       string
+	SourceURL string
+	Title     string
+	Markdown  string
 }
 
 const keychainServiceName = "webctx"
@@ -29,65 +27,6 @@ var (
 	keychainLookup    = lookupKeychainSecret
 )
 
-func checkMarkdownAvailable(raw string) (bool, error) {
-	mdURL := raw
-	if !strings.HasSuffix(strings.ToLower(mdURL), ".md") {
-		mdURL += ".md"
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	req, err := http.NewRequestWithContext(ctx, http.MethodHead, mdURL, nil)
-	if err != nil {
-		return false, err
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return false, err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return false, nil
-	}
-	contentType := strings.ToLower(resp.Header.Get("Content-Type"))
-	contentLength := resp.ContentLength
-	return (strings.Contains(contentType, "markdown") || strings.Contains(contentType, "text/plain")) && contentLength > 50, nil
-}
-
-func fetchMarkdownContent(raw string) (*MarkdownResult, error) {
-	mdURL := raw
-	if !strings.HasSuffix(strings.ToLower(mdURL), ".md") {
-		mdURL += ".md"
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	content, status, err := fetchText(ctx, mdURL)
-	if err != nil {
-		return nil, err
-	}
-	if status < 200 || status >= 300 {
-		return nil, fmt.Errorf("Failed to fetch markdown: %d", status)
-	}
-	title := firstHeadingOrFallback(content, filepath.Base(raw))
-	return &MarkdownResult{URL: raw, Title: title, Markdown: content}, nil
-}
-
-func fetchText(ctx context.Context, rawURL string) (string, int, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
-	if err != nil {
-		return "", 0, err
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", 0, err
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return "", resp.StatusCode, err
-	}
-	return string(body), resp.StatusCode, nil
-}
-
 func firstHeadingOrFallback(markdown, fallback string) string {
 	for _, line := range strings.Split(markdown, "\n") {
 		trimmed := strings.TrimSpace(line)

package/internal/app/structured_source.go

@@ -228,10 +228,20 @@ func selectedJSONFields(raw any, question string, budget int) string {
 }
 
 func focusedRawJSON(markdown, question string, budget int) string {
+	// Raw JSON documents have one JSON fence just after the URL. Looking for
+	// JSON examples in a long Markdown guide previously allowed the entire
+	// preceding guide to escape the excerpt budget.
+	if !strings.HasPrefix(strings.TrimSpace(markdown), "**URL:**") {
+		return ""
+	}
 	begin := strings.Index(markdown, "```json\n")
 	if begin < 0 {
 		return ""
 	}
+	prefix := strings.TrimSpace(markdown[:begin])
+	if len(prefix) > 400 || strings.Contains(prefix, "\n#") {
+		return ""
+	}
 	jsonStart := begin + len("```json\n")
 	end := strings.LastIndex(markdown, "\n```")
 	if end <= jsonStart {

package/internal/app/tools.go

@@ -171,12 +171,12 @@ func ReadLink(rawURL string) (string, error) {
 		return direct, nil
 	}
 
-	if ok, _ := checkMarkdownAvailable(rawURL); ok {
-		result, err := fetchMarkdownContent(rawURL)
-		if err == nil {
-			return formatReadLink(result.Title, result.URL, result.Markdown), nil
-		}
+	nativeStarted := time.Now()
+	if result, err := readNativeMarkdown(rawURL); err == nil && result != nil {
+		traceRetrieval(rawURL, "native-markdown", "success", nativeStarted, nil)
+		return formatReadLink(result.Title, result.URL, "**Markdown source:** "+result.SourceURL+"\n\n"+result.Markdown), nil
 	}
+	traceRetrieval(rawURL, "native-markdown", "miss", nativeStarted, nil)
 
 	title, markdown, err := scrapeLinkWithFirecrawl(rawURL)
 	if err != nil {
@@ -217,28 +217,37 @@ func scrapeLinkWithFirecrawl(rawURL string) (string, string, error) {
 	// Firecrawl's default auto proxy already tries basic then enhanced when the
 	// transport fails. A second explicit enhanced attempt is only useful when a
 	// scrape responds successfully but returns a challenge/empty document.
+	started := time.Now()
 	first, err := firecrawlScrape(rawURL, apiKey, "")
 	if err == nil && usableScrapedMarkdown(first.Markdown) {
+		traceRetrieval(rawURL, "firecrawl-auto", "success", started, nil)
 		return first.Title, withSourceTools(first.Markdown, first.ToolSummary), nil
 	}
 	lastError := err
+	traceRetrieval(rawURL, "firecrawl-auto", "escalate", started, err)
 	if err == nil {
-		lastError = fmt.Errorf("Firecrawl returned empty or blocked page content")
+		lastError = fmt.Errorf("Firecrawl returned empty or blocked page content: %s", challengeReason(first.Markdown))
+		started = time.Now()
 		second, enhancedErr := firecrawlScrape(rawURL, apiKey, "enhanced")
 		if enhancedErr == nil && usableScrapedMarkdown(second.Markdown) {
+			traceRetrieval(rawURL, "firecrawl-enhanced", "success", started, nil)
 			return second.Title, withSourceTools(second.Markdown, second.ToolSummary), nil
 		}
+		traceRetrieval(rawURL, "firecrawl-enhanced", "escalate", started, enhancedErr)
 		if enhancedErr != nil {
 			lastError = enhancedErr
 		}
 	}
+	started = time.Now()
 	title, markdown, browserErr := firecrawlBrowserFallback(rawURL, apiKey)
 	if browserErr == nil && usableScrapedMarkdown(markdown) {
+		traceRetrieval(rawURL, "browser", "success", started, nil)
 		return title, markdown, nil
 	}
 	if browserErr == nil {
-		browserErr = fmt.Errorf("browser produced no usable content")
+		browserErr = fmt.Errorf("browser produced no usable content: %s", challengeReason(markdown))
 	}
+	traceRetrieval(rawURL, "browser", "blocked", started, browserErr)
 	return "", "", fmt.Errorf("Error reading web page: scrape: %v; browser fallback: %w", lastError, browserErr)
 }
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "webctx",
-  "version": "0.2.3",
-  "description": "Pure Go web search and browsing CLI using Brave, Tavily, Exa, and Firecrawl",
+  "version": "0.2.5",
+  "description": "Go CLI for agent-friendly web research, structured-data discovery, and source-native querying",
   "license": "MIT",
   "author": "amxv",
   "repository": {