webctx 0.2.3 → 0.2.5

package/internal/app/data_types.go

@@ -0,0 +1,40 @@
+package app
+
+// ResearchData discovers data sources and their available operations. Search
+// and catalogue browsing are both free Firecrawl Alexandria operations.
+type ResearchDataInput struct {
+	Query      string   `json:"query,omitempty" jsonschema:"Optional natural-language description of the information needed"`
+	Mode       string   `json:"mode,omitempty" jsonschema:"ranked semantic discovery or catalogue browsing with filters and pagination; defaults to ranked for a query"`
+	URLs       []string `json:"urls,omitempty" jsonschema:"Filter by websites associated with a data source"`
+	Sources    []string `json:"sources,omitempty" jsonschema:"Filter by source IDs returned by research"`
+	Categories []string `json:"categories,omitempty" jsonschema:"Filter by catalogue category IDs"`
+	Groups     []string `json:"groups,omitempty" jsonschema:"Filter by catalogue group IDs"`
+	Operations []string `json:"operations,omitempty" jsonschema:"Filter by exact operation IDs"`
+	View       string   `json:"view,omitempty" jsonschema:"What to browse: sources, groups, or operations; default operations for a query, otherwise sources"`
+	Include    []string `json:"include,omitempty" jsonschema:"Additional contract details: inputs, output, examples. Default is a concise overview"`
+	Limit      int      `json:"limit,omitempty" jsonschema:"Maximum results in this page, 1-100; default 10"`
+	Offset     int      `json:"offset,omitempty" jsonschema:"Catalogue pagination offset, zero-based"`
+	Raw        bool     `json:"raw,omitempty" jsonschema:"Also include the complete original Firecrawl response"`
+}
+
+// InspectDataInput returns the exact contract for a selected operation, or
+// browses all operations of a source when passed just a source ID.
+type InspectDataInput struct {
+	ID      string   `json:"id" jsonschema:"Source or operation ID returned by research, e.g. particle/podcasts/episodes/search"`
+	Include []string `json:"include,omitempty" jsonschema:"Contract sections: inputs, output, examples; all by default"`
+	Raw     bool     `json:"raw,omitempty" jsonschema:"Include the complete original Firecrawl response"`
+}
+
+type DataCall struct {
+	ID     string         `json:"id" jsonschema:"Exact operation ID returned by research or inspect"`
+	Inputs map[string]any `json:"inputs,omitempty" jsonschema:"Provider-specific input fields copied from the inspected contract"`
+}
+
+// ExecuteDataInput supports up to ten calls in a single provider request.
+// It validates each call against the free catalogue before using credits.
+type ExecuteDataInput struct {
+	Calls      []DataCall `json:"calls" jsonschema:"One to ten operation calls; inspect each operation to discover exact input field names"`
+	MaxCredits int        `json:"max_credits,omitempty" jsonschema:"Maximum predicted credits for this request; defaults to WEBCTX_ALEXANDRIA_MAX_CREDITS or 100"`
+	RequestID  string     `json:"request_id,omitempty" jsonschema:"Optional stable idempotency identifier for retrying a paid request; reuse unchanged if retrying"`
+	Raw        bool       `json:"raw,omitempty" jsonschema:"Include the complete original provider response"`
+}

package/internal/app/firecrawl_fallback.go

@@ -18,20 +18,7 @@ type scrapedPage struct {
 }
 
 func usableScrapedMarkdown(markdown string) bool {
-	text := strings.ToLower(strings.TrimSpace(markdown))
-	if len(text) < 10 {
-		return false
-	}
-	for _, marker := range []string{
-		"just a moment...", "checking your browser", "verify you are human",
-		"enable javascript and cookies", "access denied | cloudflare",
-		"attention required! | cloudflare", "no content extracted",
-	} {
-		if strings.Contains(text, marker) && len(text) < 2000 {
-			return false
-		}
-	}
-	return true
+	return usableSourceContent(markdown)
 }
 
 func firecrawlScrape(rawURL, apiKey, proxy string) (scrapedPage, error) {
@@ -165,6 +152,9 @@ func firecrawlBrowserFallback(rawURL, apiKey string) (string, string, error) {
 	if parsed, err := strconv.Unquote(strings.TrimSpace(text)); err == nil {
 		text = parsed
 	}
+	if reason := challengeReason(text); reason != "" {
+		return "", "", fmt.Errorf("browser fallback returned %s rather than documentation", reason)
+	}
 	return "Browser extraction", text, nil
 }
 

package/internal/app/firecrawl_fallback_test.go

@@ -1,6 +1,7 @@
 package app
 
 import (
+	"encoding/json"
 	"io"
 	"net/http"
 	"strings"
@@ -18,6 +19,43 @@ func TestSourceURLSafety(t *testing.T) {
 	}
 }
 
+func TestLongBrowserCAPTCHAFallbackFailsClearly(t *testing.T) {
+	original := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = original })
+	challenge := "hCaptcha Please verify you are human. " +
+		strings.Repeat("French Deutsch Español English 日本語 中文 ", 150)
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		switch req.Method + " " + req.URL.Path {
+		case "POST /v2/scrape":
+			content, _ := json.Marshal(map[string]any{
+				"success": true,
+				"data":    map[string]any{"markdown": challenge},
+			})
+			return testHTTPResponse(req, http.StatusOK, string(content), nil), nil
+		case "POST /v2/interact":
+			return testHTTPResponse(req, http.StatusOK, `{"success":true,"id":"test-captcha"}`, nil), nil
+		case "POST /v2/interact/test-captcha/execute":
+			body, _ := json.Marshal(map[string]any{
+				"success": true, "stdout": challenge, "exitCode": 0,
+			})
+			return testHTTPResponse(req, http.StatusOK, string(body), nil), nil
+		case "DELETE /v2/interact/test-captcha":
+			return testHTTPResponse(req, http.StatusOK, `{"success":true}`, nil), nil
+		default:
+			t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
+			return nil, nil
+		}
+	})}
+	t.Setenv("FIRECRAWL_API_KEY", "unit-test")
+	_, returned, err := scrapeLinkWithFirecrawl("https://docs.example.com/challenged")
+	if err == nil || returned != "" || !strings.Contains(err.Error(), "challenge") {
+		t.Fatalf("long browser challenge leaked into result or lacked clear failure: %q %v", returned, err)
+	}
+	if strings.Contains(err.Error(), "French Deutsch") {
+		t.Fatalf("challenge payload should not be logged verbatim in errors: %s", err)
+	}
+}
+
 func TestScrapeEscalatesToEnhancedOnBlockedBody(t *testing.T) {
 	originalClient := http.DefaultClient
 	t.Cleanup(func() { http.DefaultClient = originalClient })

package/internal/app/native_markdown.go

@@ -0,0 +1,76 @@
+package app
+
+import (
+	"context"
+	"net/url"
+	"path"
+	"strings"
+	"time"
+)
+
+const maxNativeMarkdownBytes = 1024 * 1024
+
+// Documentation sites frequently expose canonical HTML and an authored
+// Markdown representation. Use bounded GET probes instead of HEAD:
+// Stripe sends Content-Length: 0 on a successful Markdown HEAD response,
+// and Cloudflare uses /index.md for directory URLs.
+func markdownCandidateURLs(rawURL string) []string {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return nil
+	}
+	p := u.Path
+	if strings.HasSuffix(strings.ToLower(p), ".md") {
+		return []string{rawURL}
+	}
+	var candidates []string
+	if strings.HasSuffix(p, "/") || p == "" {
+		candidates = append(candidates, strings.TrimSuffix(p, "/")+"/index.md")
+	} else if path.Ext(p) == "" {
+		candidates = append(candidates, p+".md", p+"/index.md")
+	} else {
+		return nil
+	}
+	seen := map[string]bool{}
+	var out []string
+	for _, candidate := range candidates {
+		next := *u
+		next.Path = candidate
+		next.RawPath = ""
+		next.Fragment = ""
+		next.RawFragment = ""
+		value := next.String()
+		if !seen[value] && validateSourceURL(value) == nil {
+			out = append(out, value)
+			seen[value] = true
+		}
+	}
+	return out
+}
+
+func usableNativeMarkdown(content string) bool {
+	text := strings.TrimSpace(content)
+	if len(text) < 18 || strings.IndexByte(text, 0) >= 0 || challengeReason(text) != "" {
+		return false
+	}
+	prefix := strings.ToLower(text[:min(len(text), 700)])
+	if strings.Contains(prefix, "<!doctype html") || strings.Contains(prefix, "<html") ||
+		strings.Contains(prefix, "<head") || strings.Contains(prefix, "<body") {
+		return false
+	}
+	return true
+}
+
+func readNativeMarkdown(rawURL string) (*MarkdownResult, error) {
+	for _, candidate := range markdownCandidateURLs(rawURL) {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		body, err := fetchPublicText(ctx, candidate, maxNativeMarkdownBytes)
+		cancel()
+		if err != nil || !usableNativeMarkdown(body) {
+			continue
+		}
+		title := firstHeadingOrFallback(body, path.Base(rawURL))
+		return &MarkdownResult{URL: rawURL, SourceURL: candidate, Title: title, Markdown: body}, nil
+	}
+	return nil, nil
+}

package/internal/app/native_markdown_test.go

@@ -0,0 +1,78 @@
+package app
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func TestCanonicalStripeGuidePrefersMarkdownWithZeroHeadLength(t *testing.T) {
+	previous := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = previous })
+	calls := []string{}
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		calls = append(calls, req.Method+" "+req.URL.Path)
+		if req.Method == http.MethodHead {
+			t.Fatal("native Markdown detection must not require HEAD metadata")
+		}
+		if req.Method == http.MethodGet && req.URL.Path == "/payments/checkout.md" {
+			r := testHTTPResponse(req, http.StatusOK, "# Build a payments page\n\nStripe-hosted checkout is supported.", map[string]string{"Content-Type": "text/markdown"})
+			r.ContentLength = 0 // Publishers can return zero or unknown length metadata.
+			return r, nil
+		}
+		t.Fatalf("unexpected extra request: %s %s", req.Method, req.URL)
+		return nil, nil
+	})}
+	out, err := ReadLink("https://docs.stripe.com/payments/checkout")
+	if err != nil || !strings.Contains(out, "**Markdown source:** https://docs.stripe.com/payments/checkout.md") {
+		t.Fatalf("native source not selected: %v %s", err, out)
+	}
+	if len(calls) != 1 {
+		t.Fatalf("unexpected calls: %+v", calls)
+	}
+}
+
+func TestCanonicalCloudflareDirectoryReadsIndexMarkdown(t *testing.T) {
+	previous := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = previous })
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		if req.Method != http.MethodGet || req.URL.Path != "/durable-objects/index.md" {
+			t.Fatalf("unexpected URL: %s %s", req.Method, req.URL)
+		}
+		return testHTTPResponse(req, http.StatusOK,
+			"# Cloudflare Durable Objects\n\n[Get started](https://developers.cloudflare.com/durable-objects/get-started/)",
+			map[string]string{"Content-Type": "text/markdown"}), nil
+	})}
+	out, err := ReadLink("https://developers.cloudflare.com/durable-objects/")
+	if err != nil || !strings.Contains(out, "**Markdown source:** https://developers.cloudflare.com/durable-objects/index.md") {
+		t.Fatalf("directory-index native Markdown failed: %v %s", err, out)
+	}
+}
+
+func TestCAPTCHADisguisedAsMarkdownIsRejected(t *testing.T) {
+	previous := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = previous })
+	var scrapeCalled bool
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		if req.URL.Path == "/guide.md" {
+			return testHTTPResponse(req, http.StatusOK, "# hCaptcha\nPlease verify you are human. "+
+				strings.Repeat("English French Deutsch Español Chinese 日本語 ", 200), map[string]string{"Content-Type": "text/markdown"}), nil
+		}
+		if req.URL.Path == "/guide/index.md" {
+			return testHTTPResponse(req, http.StatusNotFound, "", nil), nil
+		}
+		if req.Method == http.MethodPost && req.URL.Path == "/v2/scrape" {
+			scrapeCalled = true
+			return testHTTPResponse(req, http.StatusOK,
+				`{"success":true,"data":{"markdown":"# Legitimate guide\n\nFull instructions for a public API."}}`,
+				nil), nil
+		}
+		t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
+		return nil, nil
+	})}
+	t.Setenv("FIRECRAWL_API_KEY", "test")
+	out, err := ReadLink("https://docs.example.com/guide")
+	if err != nil || !scrapeCalled || strings.Contains(out, "Deutsch Español") || !strings.Contains(out, "Full instructions") {
+		t.Fatalf("CAPTCHA disguised as Markdown leaked or fallback failed: %v %s", err, out)
+	}
+}

package/internal/app/research.go

@@ -20,10 +20,19 @@ const (
 	focusedPrimaryBudget = 22000
 	focusedRelatedBudget = 6500
 	searchSourceBudget   = 4000
-	maxRelatedSources    = 2
+	maxRelatedSources    = 5
+	maxFollowupSources   = 2
 	maxSearchSources     = 3
 )
 
+func relatedExcerptBudget(rawURL string) int {
+	lower := strings.ToLower(rawURL)
+	if strings.Contains(lower, "/get-started/") || strings.Contains(lower, "/quickstart") {
+		return 18000 // Preserve end-to-end runnable setup guides.
+	}
+	return focusedRelatedBudget
+}
+
 var (
 	mdLinkPattern     = regexp.MustCompile(`\[([^\]\n]{2,120})\]\(([^\s)]+)(?:\s+"[^"]*")?\)`)
 	wordPattern       = regexp.MustCompile(`[a-zA-Z][a-zA-Z0-9_-]{2,}`)
@@ -79,25 +88,17 @@ func readLinkFocused(rawURL, question string, reader func(string) (string, error
 		return "", err
 	}
 
-	links := relatedSourceLinks(primary, rawURL, question, maxRelatedSources)
-	if len(links) < maxRelatedSources {
-		// Many doc systems publish llms.txt as their navigation index. It is a
-		// cheap way to find exact API reference pages not linked from this page.
-		for _, candidate := range linksFromDocsIndex(rawURL, question, maxRelatedSources) {
-			if len(links) == maxRelatedSources {
-				break
-			}
-			duplicate := candidate.url == rawURL
-			for _, chosen := range links {
-				duplicate = duplicate || chosen.url == candidate.url
-			}
-			if !duplicate {
-				links = append(links, candidate)
-			}
-		}
-	}
+	// Combine on-page links and authoritative Markdown index links instead of
+	// filling the entire budget from whichever links happen to appear first.
+	// In particular, an implementation question should prefer get-started,
+	// SQLite, webhook, and API references over incidental alarm/concept pages.
+	links := researchCandidates(rawURL, primary, question, maxRelatedSources)
 	structuredDone := make(chan string, 1)
-	go func() { structuredDone <- embeddedStructuredContext(rawURL, question) }()
+	if strings.Contains(primary, "**Markdown source:**") {
+		structuredDone <- ""
+	} else {
+		go func() { structuredDone <- embeddedStructuredContext(rawURL, question) }()
+	}
 	more := readSources(links, reader)
 	parts := []string{
 		"# Source-grounded context",
@@ -107,27 +108,91 @@ func readLinkFocused(rawURL, question string, reader func(string) (string, error
 		"## Primary source",
 		"**Source:** " + rawURL,
 		"",
-		focusExcerpt(primary, question, focusedPrimaryBudget),
 	}
+	primaryExcerpt := focusExcerpt(primary, question, focusedPrimaryBudget)
+	parts = append(parts, primaryExcerpt)
 	successful := 1
+	relatedSuccessful := 0
+	allRetrieved := primary
+	outputEvidence := primaryExcerpt
+	seenURLs := map[string]bool{rawURL: true}
 	for _, source := range more {
 		if source.err != nil {
 			continue
 		}
 		successful++
-		parts = append(parts, "", "## Related source", "**Source:** "+source.url, "", focusExcerpt(source.content, question, focusedRelatedBudget))
+		relatedSuccessful++
+		seenURLs[source.url] = true
+		allRetrieved += "\n" + source.content
+		excerpt := focusExcerpt(source.content, question, relatedExcerptBudget(source.url))
+		outputEvidence += "\n" + excerpt
+		parts = append(parts, "", "## Related source", "**Source:** "+source.url, "", excerpt)
+	}
+	// One additional bounded expansion if required code/API evidence remains
+	// missing. This follows references from the implementation pages we actually
+	// found, not arbitrary link hops across the entire site.
+	missing := missingResearchEvidence(question, allRetrieved)
+	if len(missing) > 0 {
+		var next []sourceCandidate
+		for _, source := range more {
+			if source.err != nil {
+				continue
+			}
+			if strings.Contains(strings.ToLower(source.content), "article has multiple variants") {
+				for _, candidate := range relatedSourceLinks(source.content, source.url, question, 8) {
+					if !seenURLs[candidate.url] {
+						seenURLs[candidate.url] = true
+						candidate.score += 50 // A published page variant is the actual documentation, not an index.
+						if strings.Contains(strings.ToLower(question), "hosted") &&
+							strings.Contains(strings.ToLower(candidate.url+" "+candidate.label), "hosted") {
+							candidate.score += 20
+						}
+						next = append(next, candidate)
+					}
+				}
+			}
+			for _, candidate := range relatedSourceLinks(source.content, source.url, strings.Join(missing, " "), 8) {
+				if !seenURLs[candidate.url] {
+					seenURLs[candidate.url] = true
+					candidate.score += implementationLinkBonus(candidate, question)
+					next = append(next, candidate)
+				}
+			}
+		}
+		sort.SliceStable(next, func(i, j int) bool { return next[i].score > next[j].score })
+		if len(next) > maxFollowupSources {
+			next = next[:maxFollowupSources]
+		}
+		followups := readSources(next, reader)
+		more = append(more, followups...)
+		for _, source := range followups {
+			if source.err != nil {
+				continue
+			}
+			successful++
+			relatedSuccessful++
+			allRetrieved += "\n" + source.content
+			excerpt := focusExcerpt(source.content, question, relatedExcerptBudget(source.url))
+			outputEvidence += "\n" + excerpt
+			parts = append(parts, "", "## Supporting implementation reference", "**Source:** "+source.url, "", excerpt)
+		}
 	}
 	if openAPIDone != nil {
 		if contract := <-openAPIDone; contract.Content != "" {
 			successful++
+			outputEvidence += "\n" + contract.Content
 			parts = append(parts, "", "## REST API contract", "**Source:** "+contract.URL, "", contract.Content)
 		}
 	}
 	if structured := <-structuredDone; structured != "" {
+		outputEvidence += "\n" + structured
 		parts = append(parts, "", structured)
 	}
+	if unmet := missingResearchEvidence(question, outputEvidence); len(unmet) > 0 {
+		parts = append(parts, "", "**Requested details not fully evidenced in returned excerpts:** "+strings.Join(unmet, ", ")+". Follow the cited source pages rather than assuming these details.")
+	}
 	parts = append(parts, "", fmt.Sprintf("**Coverage:** %d source document(s) inspected. Extracts are source-grounded, not an independently verified synthesis. REST commands assembled from an OpenAPI operation and its published example are labeled as such.", successful))
-	if failed := len(more) - (successful - 1); failed > 0 {
+	if failed := len(more) - relatedSuccessful; failed > 0 {
 		parts = append(parts, fmt.Sprintf("%d related page(s) could not be retrieved; the primary source is preserved.", failed))
 	}
 	return strings.Join(parts, "\n"), nil
@@ -184,24 +249,48 @@ func enrichSearchWithReader(query string, ranked []SearchDoc, reader func(string
 }
 
 func linksFromDocsIndex(rawURL, question string, limit int) []sourceCandidate {
+	return linksFromDocsIndexWithPrimary(rawURL, "", question, limit)
+}
+
+func linksFromDocsIndexWithPrimary(rawURL, primary, question string, limit int) []sourceCandidate {
 	u, err := url.Parse(rawURL)
 	if err != nil || u.Hostname() == "github.com" || directSourceLanguage(rawURL) != "" {
 		return nil
 	}
-	if !strings.HasPrefix(u.Hostname(), "docs.") && !strings.Contains(u.Path, "/docs/") && !strings.Contains(u.Path, "/reference/") {
+	if !strings.HasPrefix(u.Hostname(), "docs.") && !strings.Contains(u.Path, "/docs/") &&
+		!strings.Contains(u.Path, "/reference/") && !strings.Contains(primary, "llms.txt") {
 		return nil
 	}
-	indexURL := u.Scheme + "://" + u.Host + "/llms.txt"
-	if rawURL == indexURL {
-		return nil
+	indexURLs := documentIndexURLs(u, primary)
+	var candidates []sourceCandidate
+	seen := map[string]bool{}
+	for _, indexURL := range indexURLs {
+		if rawURL == indexURL {
+			continue
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		index, err := fetchPublicText(ctx, indexURL, 300000)
+		cancel()
+		if err != nil || !strings.Contains(index, "](") {
+			continue
+		}
+		for _, item := range relatedSourceLinks(index, indexURL, question, 300) {
+			item.score += implementationLinkBonus(item, question)
+			item.score += sourceScopeBonus(item, rawURL, question)
+			if !seen[item.url] {
+				seen[item.url] = true
+				candidates = append(candidates, item)
+			}
+		}
+		if len(candidates) >= limit {
+			break
+		}
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	index, err := fetchPublicText(ctx, indexURL, 300000)
-	if err != nil || !strings.Contains(index, "](") {
-		return nil
+	sort.SliceStable(candidates, func(i, j int) bool { return candidates[i].score > candidates[j].score })
+	if len(candidates) > limit {
+		candidates = candidates[:limit]
 	}
-	return relatedSourceLinks(index, indexURL, question, limit)
+	return candidates
 }
 
 func readSources(candidates []sourceCandidate, reader func(string) (string, error)) []sourceRead {
@@ -279,7 +368,7 @@ func relatedSourceLinks(markdown, baseURL, question string, limit int) []sourceC
 	terms := queryTerms(question)
 	seen := map[string]bool{canonicalSourceURL(base): true}
 	var links []sourceCandidate
-	for i, match := range mdLinkPattern.FindAllStringSubmatch(markdown, 300) {
+	for i, match := range mdLinkPattern.FindAllStringSubmatch(markdown, 1000) {
 		destination := strings.TrimSpace(match[2])
 		if strings.Contains(destination, "\\") { // malformed Markdown-escaped URLs
 			continue