webctx 0.2.2 → 0.2.4

package/internal/app/research_excerpt.go

@@ -0,0 +1,149 @@
+package app
+
+import (
+	"sort"
+	"strings"
+)
+
+const backtickFence = "\x60\x60\x60"
+
+type evidenceBlock struct {
+	text  string
+	code  bool
+	score int
+	order int
+}
+
+// A code fence is indivisible evidence: do not silently truncate a command,
+// request body or SDK example and then invent a closing fence.
+func splitEvidenceBlocks(markdown, question string) []evidenceBlock {
+	var blocks []evidenceBlock
+	var content strings.Builder
+	terms := queryTerms(question)
+	inCode := false
+	fence := ""
+	flush := func(code bool) {
+		text := strings.TrimSpace(content.String())
+		content.Reset()
+		if text == "" {
+			return
+		}
+		score := termScore(text, terms)
+		if code {
+			score *= 3
+			if strings.Contains(strings.ToLower(question), "rest") &&
+				(strings.Contains(text, "curl") || strings.Contains(text, "POST ") || strings.Contains(text, "GET ")) {
+				score += 4
+			}
+		}
+		blocks = append(blocks, evidenceBlock{text: text, code: code, score: score, order: len(blocks)})
+	}
+	for _, line := range strings.Split(markdown, "\n") {
+		trimmed := strings.TrimSpace(line)
+		opening := !inCode && (strings.HasPrefix(trimmed, backtickFence) || strings.HasPrefix(trimmed, "~~~"))
+		if opening {
+			flush(false)
+			inCode = true
+			fence = trimmed[:3]
+		}
+		content.WriteString(line)
+		content.WriteByte('\n')
+		if inCode && !opening && strings.TrimSpace(strings.TrimPrefix(trimmed, fence)) == "" &&
+			strings.HasPrefix(trimmed, fence) {
+			flush(true)
+			inCode = false
+			fence = ""
+		}
+	}
+	// Unclosed code blocks are not valid examples; treat them as source prose.
+	flush(false)
+	return blocks
+}
+
+// Select complete, high-relevance examples and explanatory prose from long
+// sections, preserving original order and the hard context budget.
+func atomicFocusedSection(markdown, question string, budget int) string {
+	markdown = strings.TrimSpace(markdown)
+	if len(markdown) <= budget {
+		return markdown
+	}
+	if budget < 180 {
+		return "*[Section exceeds excerpt budget; follow the source URL.]*"
+	}
+	blocks := splitEvidenceBlocks(markdown, question)
+	const omitted = "\n\n*[Other source text omitted. Follow the original URL for full details.]*"
+	remaining := budget - len(omitted) - 8
+	if remaining < 80 {
+		return "*[Section exceeds excerpt budget; follow the source URL.]*"
+	}
+	selected := make(map[int]string)
+	var codes []evidenceBlock
+	var prose []evidenceBlock
+	for _, block := range blocks {
+		if block.code {
+			codes = append(codes, block)
+		} else {
+			prose = append(prose, block)
+		}
+	}
+	sort.SliceStable(codes, func(i, j int) bool {
+		if codes[i].score == codes[j].score {
+			return codes[i].order < codes[j].order
+		}
+		return codes[i].score > codes[j].score
+	})
+	// Reserve a little space for the section explanation.
+	codeAllowance := remaining * 85 / 100
+	for _, block := range codes {
+		size := len(block.text) + 2
+		if size <= codeAllowance {
+			selected[block.order] = block.text
+			codeAllowance -= size
+			remaining -= size
+		}
+	}
+	sort.SliceStable(prose, func(i, j int) bool {
+		if prose[i].score == prose[j].score {
+			return prose[i].order < prose[j].order
+		}
+		return prose[i].score > prose[j].score
+	})
+	for _, block := range prose {
+		if remaining < 100 {
+			break
+		}
+		if len(block.text)+2 <= remaining {
+			selected[block.order] = block.text
+			remaining -= len(block.text) + 2
+		} else if part := completeProseLines(block.text, remaining-4); part != "" {
+			selected[block.order] = part
+			remaining -= len(part) + 2
+		}
+	}
+	var output []string
+	for _, block := range blocks {
+		if text := selected[block.order]; text != "" {
+			output = append(output, text)
+		}
+	}
+	if len(output) == 0 {
+		return "*[Code examples exceed the excerpt budget; inspect source URL for the full examples.]*"
+	}
+	return strings.Join(output, "\n\n") + omitted
+}
+
+func completeProseLines(text string, maxBytes int) string {
+	if maxBytes <= 0 {
+		return ""
+	}
+	var lines []string
+	size := 0
+	for _, line := range strings.Split(text, "\n") {
+		if size+len(line)+1 > maxBytes {
+			break
+		}
+		lines = append(lines, line)
+		size += len(line) + 1
+	}
+	return strings.TrimSpace(strings.Join(lines, "\n"))
+}

package/internal/app/research_excerpt_test.go

@@ -0,0 +1,47 @@
+package app
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestFocusedExcerptPreservesWholeCurlExamples(t *testing.T) {
+	code := backtickFence + "bash\ncurl -X POST https://api.example.com/v2/scrape \\\n" +
+		" -H 'Content-Type: application/json' \\\n" +
+		" -d '{\"alexandria\":{\"capability\":\"find-tools\"}}'\n" + backtickFence
+	doc := "# Documentation\n\n" + strings.Repeat("Background prose unrelated to endpoints.\n", 200) +
+		"\n## Inspect tools over REST\n" + strings.Repeat("Introductory filler.\n", 70) +
+		"\n" + code + "\n" + strings.Repeat("Appendix filler.\n", 200)
+	got := focusExcerpt(doc, "How do I inspect Alexandria tools with a REST request example?", 1600)
+	if !strings.Contains(got, code) {
+		t.Fatalf("the complete REST example must survive extraction:\n%s", got)
+	}
+	if strings.Count(got, backtickFence) != 2 {
+		t.Fatalf("expected exactly one balanced code fence: %s", got)
+	}
+	if len(got) > 1800 {
+		t.Fatalf("excerpt exceeded budget: %d", len(got))
+	}
+}
+
+func TestOversizedCodeBlocksAreNeverPartiallyReturned(t *testing.T) {
+	huge := backtickFence + "json\n{\"payload\":\"" + strings.Repeat("a", 3000) + "\"}\n" + backtickFence
+	doc := "# API\n\n" + huge + "\n\n" + strings.Repeat("Further explanation.\n", 200)
+	got := atomicFocusedSection(doc, "What is the API payload?", 700)
+	if strings.Contains(got, "\"payload\"") || strings.Contains(got, backtickFence) {
+		t.Fatalf("oversized example should be omitted, not truncated:\n%s", got)
+	}
+	if !strings.Contains(got, "omitted") && !strings.Contains(got, "source URL") {
+		t.Fatalf("omission should be explicitly marked: %s", got)
+	}
+}
+
+func TestPlainCodeFenceOpeningNotTreatedAsClosing(t *testing.T) {
+	doc := "# API\n" + strings.Repeat("Preliminary text.\n", 80) + "\n" +
+		backtickFence + "\nPOST /v2/scrape\n{}\n" + backtickFence + "\n" +
+		strings.Repeat("End notes.\n", 150)
+	got := atomicFocusedSection(doc, "scrape request", 650)
+	if !strings.Contains(got, backtickFence+"\nPOST /v2/scrape\n{}\n"+backtickFence) {
+		t.Fatalf("plain opening fence was not handled atomically: %s", got)
+	}
+}

package/internal/app/research_navigation.go

@@ -0,0 +1,237 @@
+package app
+
+import (
+	"net/url"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+var documentationIndexPattern = regexp.MustCompile("https?://[^\\s<>\"')\\]]+/llms\\.txt")
+
+func documentIndexURLs(base *url.URL, primary string) []string {
+	var urls []string
+	seen := map[string]bool{}
+	for _, raw := range documentationIndexPattern.FindAllString(primary, 5) {
+		target, err := url.Parse(raw)
+		if err != nil || validateSourceURL(raw) != nil ||
+			!strings.EqualFold(base.Hostname(), target.Hostname()) || target.Port() != base.Port() {
+			continue
+		}
+		if !seen[raw] {
+			urls = append(urls, raw)
+			seen[raw] = true
+		}
+	}
+	fallback := base.Scheme + "://" + base.Host + "/llms.txt"
+	if !seen[fallback] {
+		urls = append(urls, fallback)
+	}
+	return urls
+}
+
+func implementationQuestion(question string) bool {
+	text := strings.ToLower(question)
+	for _, signal := range []string{
+		"implement", "integrat", "runnable", "configuration",
+		"code example", "complete", "how to", "set up", "using a", "creating",
+	} {
+		if strings.Contains(text, signal) {
+			return true
+		}
+	}
+	return false
+}
+
+func implementationLinkBonus(candidate sourceCandidate, question string) int {
+	lower := strings.ToLower(candidate.url + " " + candidate.label)
+	q := strings.ToLower(question)
+	score := 0
+	if implementationQuestion(question) {
+		switch {
+		case strings.Contains(lower, "/get-started/"), strings.Contains(lower, "/quickstart"),
+			strings.Contains(lower, "getting started"):
+			score += 64 // Publisher's end-to-end guide, not isolated snippets.
+		case strings.Contains(lower, "tutorial"), strings.Contains(lower, "integration"),
+			strings.Contains(lower, "how-to"):
+			score += 16
+		case strings.Contains(lower, "example"):
+			score += 6
+		}
+	}
+	if strings.Contains(q, "sql") || strings.Contains(q, "sqlite") {
+		if strings.Contains(lower, "sqlite") || strings.Contains(lower, "storage") {
+			score += 26
+		} else if strings.Contains(lower, "/examples/") {
+			score -= 16
+		}
+	}
+	if strings.Contains(q, "webhook") || strings.Contains(q, "fulfill") || strings.Contains(q, "fulfil") {
+		if strings.Contains(lower, "webhook") || strings.Contains(lower, "fulfill") || strings.Contains(lower, "fulfil") {
+			score += 30
+		}
+	}
+	if strings.Contains(q, "checkout") && (strings.Contains(lower, "checkout") || strings.Contains(lower, "session")) {
+		score += 6
+	}
+	if strings.Contains(q, "signature") && strings.Contains(lower, "signature") {
+		score += 24
+	}
+	if strings.Contains(q, "curl") || strings.Contains(q, "rest") || strings.Contains(q, "endpoint") {
+		if strings.Contains(lower, "/api/") || strings.Contains(lower, "api-reference") {
+			score += 10
+		}
+	}
+	if strings.Contains(q, "session") && strings.Contains(q, "creat") &&
+		(strings.Contains(lower, "/sessions/create") || strings.Contains(lower, "create a checkout session")) {
+		score += 38
+	}
+	if strings.Contains(q, "checkout") && strings.Contains(lower, "/checkout/quickstart") {
+		score += 20
+	}
+	if strings.Contains(q, "fulfill") || strings.Contains(q, "fulfil") {
+		if strings.Contains(lower, "/checkout/fulfillment") {
+			score += 24
+		}
+	}
+	if strings.Contains(lower, "/alarms/") && !strings.Contains(q, "alarm") {
+		score -= 24
+	}
+	if strings.Contains(lower, "/concepts/") && implementationQuestion(question) {
+		score -= 12
+	}
+	if strings.Contains(lower, "legacy") && (strings.Contains(q, "new ") || strings.Contains(q, "current")) && !strings.Contains(q, "legacy") {
+		score -= 25
+	}
+	return score
+}
+
+// Search indices on a large documentation host often contain links for many
+// unrelated products. Prefer links in the original section or with a product
+// named in the question. This prevents "webhooks for Climate Orders" from
+// outranking generic Checkout webhooks solely because both contain "webhook".
+func sourceScopeBonus(candidate sourceCandidate, rawURL, question string) int {
+	base, err1 := url.Parse(rawURL)
+	target, err2 := url.Parse(candidate.url)
+	if err1 != nil || err2 != nil {
+		return 0
+	}
+	q := strings.ToLower(question)
+	sourceSegments := strings.Split(strings.Trim(base.Path, "/"), "/")
+	targetSegments := strings.Split(strings.Trim(target.Path, "/"), "/")
+	if len(targetSegments) == 0 {
+		return 0
+	}
+	score := 0
+	if len(sourceSegments) > 0 && targetSegments[0] == sourceSegments[0] {
+		score += 8
+	}
+	if len(targetSegments) > 0 && targetSegments[0] != "" &&
+		!strings.Contains(q, strings.TrimSuffix(targetSegments[0], "s")) &&
+		!strings.Contains(q, targetSegments[0]) &&
+		targetSegments[0] != "api" && targetSegments[0] != "webhooks" &&
+		targetSegments[0] != "events" &&
+		(len(sourceSegments) == 0 || targetSegments[0] != sourceSegments[0]) {
+		score -= 24
+	}
+	// A general Worker overview is less relevant than Durable Objects'
+	// focused setup instructions even if both mention "Worker".
+	if strings.Contains(q, "durable object") && strings.Contains(target.Path, "/workers/") &&
+		!strings.Contains(target.Path, "/durable-objects/") {
+		score -= 14
+	}
+	return score
+}
+
+func normalizedDocumentationURL(raw string) string {
+	u, err := url.Parse(raw)
+	if err != nil {
+		return raw
+	}
+	p := strings.TrimSuffix(u.Path, "/index.md")
+	p = strings.TrimSuffix(p, ".md")
+	p = strings.TrimSuffix(p, "/")
+	u.Path = p
+	u.Fragment = ""
+	u.RawFragment = ""
+	u.RawPath = ""
+	return u.String()
+}
+
+func researchCandidates(rawURL, primary, question string, limit int) []sourceCandidate {
+	baseCandidates := relatedSourceLinks(primary, rawURL, question, limit*4)
+	indexCandidates := linksFromDocsIndexWithPrimary(rawURL, primary, question, limit*5)
+	seen := map[string]bool{normalizedDocumentationURL(rawURL): true}
+	var candidates []sourceCandidate
+	for _, c := range append(baseCandidates, indexCandidates...) {
+		key := normalizedDocumentationURL(c.url)
+		if seen[key] {
+			continue
+		}
+		seen[key] = true
+		c.score += implementationLinkBonus(c, question)
+		c.score += sourceScopeBonus(c, rawURL, question)
+		candidates = append(candidates, c)
+	}
+	sort.SliceStable(candidates, func(i, j int) bool {
+		if candidates[i].score == candidates[j].score {
+			return candidates[i].order < candidates[j].order
+		}
+		return candidates[i].score > candidates[j].score
+	})
+	if len(candidates) > limit {
+		return candidates[:limit]
+	}
+	return candidates
+}
+
+type evidenceNeed struct {
+	name         string
+	alternatives []string
+}
+
+// Check for explicit source evidence; do not claim that a plausible guess
+// supplies missing request fields or code.
+func evidenceNeeds(question string) []evidenceNeed {
+	q := strings.ToLower(question)
+	var needs []evidenceNeed
+	for _, literal := range []string{"line_items", "success_url", "cancel_url", "sql.exec", "wrangler", "sqlite"} {
+		if strings.Contains(q, literal) {
+			needs = append(needs, evidenceNeed{literal, []string{literal}})
+		}
+	}
+	if strings.Contains(q, "webhook") {
+		needs = append(needs, evidenceNeed{"webhook events", []string{"webhook", "checkout.session.completed"}})
+		if strings.Contains(q, "signature") || strings.Contains(q, "secure") {
+			needs = append(needs, evidenceNeed{"webhook signature verification", []string{"stripe-signature", "constructevent", "signature verification", "signatureverification"}})
+		}
+	}
+	if strings.Contains(q, "sql table") || strings.Contains(q, "creating a sql") {
+		needs = append(needs, evidenceNeed{"SQL table creation", []string{"create table"}})
+	}
+	if strings.Contains(q, "accesses the binding") || strings.Contains(q, "access the binding") {
+		needs = append(needs, evidenceNeed{"Worker binding access", []string{"getbyname", "env.my_durable_object", "env.durable_object"}})
+	}
+	if strings.Contains(q, "checkout") && (strings.Contains(q, "fulfill") || strings.Contains(q, "fulfil")) {
+		needs = append(needs, evidenceNeed{"Checkout fulfillment event", []string{"checkout.session.completed"}})
+	}
+	return needs
+}
+
+func missingResearchEvidence(question, text string) []string {
+	lower := strings.ToLower(text)
+	var missing []string
+	for _, need := range evidenceNeeds(question) {
+		found := false
+		for _, candidate := range need.alternatives {
+			if strings.Contains(lower, candidate) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			missing = append(missing, need.name)
+		}
+	}
+	return missing
+}

package/internal/app/research_navigation_test.go

@@ -0,0 +1,72 @@
+package app
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func TestStripeFocusedLinksPreferCheckoutAndWebhookDocs(t *testing.T) {
+	old := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = old })
+	index := "# Documentation\n\n" +
+		"- [Climate order webhooks](https://docs.stripe.com/climate/orders/webhooks.md)\n" +
+		"- [Create a Checkout Session](https://docs.stripe.com/api/checkout/sessions/create.md)\n" +
+		"- [Build a Stripe-hosted checkout page](https://docs.stripe.com/checkout/quickstart.md)\n" +
+		"- [Set up and deploy a webhook](https://docs.stripe.com/webhooks/quickstart.md)\n" +
+		"- [Fulfill orders](https://docs.stripe.com/checkout/fulfillment.md)\n" +
+		"- [Verify webhook signatures](https://docs.stripe.com/webhooks.md#verify-signature)\n"
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		if req.URL.Path == "/llms.txt" {
+			return testHTTPResponse(req, http.StatusOK, index, nil), nil
+		}
+		return testHTTPResponse(req, http.StatusNotFound, "", nil), nil
+	})}
+	question := "Implement Stripe-hosted Checkout: create a Checkout Session REST request and securely fulfill orders using webhooks, including signature verification"
+	primary := "# Stripe Checkout\n\n[Start building your checkout integration](https://docs.stripe.com/checkout/quickstart.md)"
+	got := researchCandidates("https://docs.stripe.com/payments/checkout", primary, question, 5)
+	var links []string
+	for _, c := range got {
+		links = append(links, c.url)
+	}
+	output := strings.Join(links, "\n")
+	for _, want := range []string{
+		"/checkout/quickstart.md",
+		"/api/checkout/sessions/create.md",
+		"/webhooks/quickstart.md",
+		"/checkout/fulfillment.md",
+	} {
+		if !strings.Contains(output, want) {
+			t.Errorf("expected %q among selected source links: %s", want, output)
+		}
+	}
+	if strings.Contains(output, "/climate/") {
+		t.Errorf("irrelevant Climate-specific webhook outranked Checkout sources: %s", output)
+	}
+}
+
+func TestCloudflareNewSQLiteQuestionPrefersGettingStarted(t *testing.T) {
+	old := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = old })
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		return testHTTPResponse(req, http.StatusForbidden, "", nil), nil
+	})}
+	primary := "# Cloudflare Durable Objects\n" +
+		"[Durable Object alarms](https://developers.cloudflare.com/durable-objects/api/alarms/)\n" +
+		"[SQLite storage](https://developers.cloudflare.com/durable-objects/api/sqlite-storage-api/)\n" +
+		"[Get started](https://developers.cloudflare.com/durable-objects/get-started/)\n" +
+		"> Documentation Index https://developers.cloudflare.com/durable-objects/llms.txt\n"
+	question := "For a new Cloudflare Worker using a SQLite-backed Durable Object, show complete wrangler configuration, Worker binding, and SQL table code."
+	got := researchCandidates("https://developers.cloudflare.com/durable-objects/", primary, question, 2)
+	if len(got) != 2 {
+		t.Fatalf("unexpected number of selected references: %+v", got)
+	}
+	var links []string
+	for _, c := range got {
+		links = append(links, c.url)
+	}
+	output := strings.Join(links, "\n")
+	if !strings.Contains(output, "/get-started/") || strings.Contains(output, "/alarms/") {
+		t.Fatalf("unrelated alarm material displaced getting-started guide: %s", output)
+	}
+}