webctx 0.2.2 → 0.2.3

package/internal/app/research.go

@@ -0,0 +1,526 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Research deliberately works with source text, not generated answers. The
+// coding agent gets exact excerpts and URLs it can independently inspect.
+// One follow-up level and fixed budgets prevent uncontrolled crawls.
+const (
+	// Coding agents value complete code and request contracts more than tiny
+	// snippets. Keep ordinary documentation pages whole when reasonably sized.
+	focusedPrimaryBudget = 22000
+	focusedRelatedBudget = 6500
+	searchSourceBudget   = 4000
+	maxRelatedSources    = 2
+	maxSearchSources     = 3
+)
+
+var (
+	mdLinkPattern     = regexp.MustCompile(`\[([^\]\n]{2,120})\]\(([^\s)]+)(?:\s+"[^"]*")?\)`)
+	wordPattern       = regexp.MustCompile(`[a-zA-Z][a-zA-Z0-9_-]{2,}`)
+	sourceWordPattern = regexp.MustCompile(`[a-zA-Z][a-zA-Z0-9]{1,}`)
+	headingPattern    = regexp.MustCompile(`^#{1,6}\s+(.+)$`)
+)
+
+type researchSection struct {
+	index   int
+	heading string
+	body    string
+	score   int
+}
+
+type sourceCandidate struct {
+	url   string
+	label string
+	score int
+	order int
+}
+
+type sourceRead struct {
+	url     string
+	content string
+	err     error
+}
+
+// ReadLinkFocused is backwards-compatible: URL-only calls still return the
+// complete original source. A question makes the engine select excerpts and
+// automatically inspect related documentation on the same host.
+func ReadLinkFocused(rawURL, question string) (string, error) {
+	return readLinkFocused(rawURL, question, ReadLink)
+}
+
+func readLinkFocused(rawURL, question string, reader func(string) (string, error)) (string, error) {
+	if strings.TrimSpace(question) == "" {
+		return reader(rawURL)
+	}
+	if err := validateSourceURL(rawURL); err != nil {
+		return "", err
+	}
+	question = strings.TrimSpace(question)
+	// REST contract discovery is independent of the page scrape. Start it
+	// immediately so a slow Firecrawl read and a large OpenAPI download overlap
+	// instead of approaching the hosted MCP function timeout sequentially.
+	var openAPIDone chan openAPIContext
+	if asksForAPIContracts(question) {
+		openAPIDone = make(chan openAPIContext, 1)
+		go func() { openAPIDone <- discoverOpenAPIContract(rawURL, question) }()
+	}
+	primary, err := reader(rawURL)
+	if err != nil {
+		return "", err
+	}
+
+	links := relatedSourceLinks(primary, rawURL, question, maxRelatedSources)
+	if len(links) < maxRelatedSources {
+		// Many doc systems publish llms.txt as their navigation index. It is a
+		// cheap way to find exact API reference pages not linked from this page.
+		for _, candidate := range linksFromDocsIndex(rawURL, question, maxRelatedSources) {
+			if len(links) == maxRelatedSources {
+				break
+			}
+			duplicate := candidate.url == rawURL
+			for _, chosen := range links {
+				duplicate = duplicate || chosen.url == candidate.url
+			}
+			if !duplicate {
+				links = append(links, candidate)
+			}
+		}
+	}
+	structuredDone := make(chan string, 1)
+	go func() { structuredDone <- embeddedStructuredContext(rawURL, question) }()
+	more := readSources(links, reader)
+	parts := []string{
+		"# Source-grounded context",
+		"",
+		"**Question:** " + question,
+		"",
+		"## Primary source",
+		"**Source:** " + rawURL,
+		"",
+		focusExcerpt(primary, question, focusedPrimaryBudget),
+	}
+	successful := 1
+	for _, source := range more {
+		if source.err != nil {
+			continue
+		}
+		successful++
+		parts = append(parts, "", "## Related source", "**Source:** "+source.url, "", focusExcerpt(source.content, question, focusedRelatedBudget))
+	}
+	if openAPIDone != nil {
+		if contract := <-openAPIDone; contract.Content != "" {
+			successful++
+			parts = append(parts, "", "## REST API contract", "**Source:** "+contract.URL, "", contract.Content)
+		}
+	}
+	if structured := <-structuredDone; structured != "" {
+		parts = append(parts, "", structured)
+	}
+	parts = append(parts, "", fmt.Sprintf("**Coverage:** %d source document(s) inspected. Extracts are source-grounded, not an independently verified synthesis. REST commands assembled from an OpenAPI operation and its published example are labeled as such.", successful))
+	if failed := len(more) - (successful - 1); failed > 0 {
+		parts = append(parts, fmt.Sprintf("%d related page(s) could not be retrieved; the primary source is preserved.", failed))
+	}
+	return strings.Join(parts, "\n"), nil
+}
+
+// enrichSearch gives agents real documentation content in their first search
+// response, with the original ranked source list retained for navigation.
+func enrichSearch(query string, ranked []SearchDoc) string {
+	return enrichSearchWithReader(query, ranked, ReadLink)
+}
+
+func enrichSearchWithReader(query string, ranked []SearchDoc, reader func(string) (string, error)) string {
+	chosen := chooseSearchSources(query, ranked, maxSearchSources)
+	if len(chosen) == 0 {
+		return ""
+	}
+	more := readSources(chosen, reader)
+	parts := []string{}
+	seen := make(map[string]bool)
+	for _, source := range more {
+		if source.err != nil {
+			continue
+		}
+		seen[source.url] = true
+		parts = append(parts, "### "+source.url, "", focusExcerpt(source.content, query, searchSourceBudget), "")
+	}
+	if len(parts) == 0 {
+		return ""
+	}
+	// Expand high-relevance documentation references from retrieved results,
+	// not arbitrary websites or unlimited recursive link trees.
+	var follow []sourceCandidate
+	for _, source := range more {
+		if source.err != nil || len(follow) >= 2 {
+			continue
+		}
+		for _, candidate := range relatedSourceLinks(source.content, source.url, query, 2) {
+			if seen[candidate.url] {
+				continue
+			}
+			seen[candidate.url] = true
+			follow = append(follow, candidate)
+			if len(follow) >= 2 {
+				break
+			}
+		}
+	}
+	for _, source := range readSources(follow, reader) {
+		if source.err == nil {
+			parts = append(parts, "### Supporting reference: "+source.url, "", focusExcerpt(source.content, query, focusedRelatedBudget), "")
+		}
+	}
+	return "## Retrieved source context\n\nThe following are excerpts from source pages, not an AI-generated answer. Compare versions and follow the URLs for full documents.\n\n" + strings.Join(parts, "\n")
+}
+
+func linksFromDocsIndex(rawURL, question string, limit int) []sourceCandidate {
+	u, err := url.Parse(rawURL)
+	if err != nil || u.Hostname() == "github.com" || directSourceLanguage(rawURL) != "" {
+		return nil
+	}
+	if !strings.HasPrefix(u.Hostname(), "docs.") && !strings.Contains(u.Path, "/docs/") && !strings.Contains(u.Path, "/reference/") {
+		return nil
+	}
+	indexURL := u.Scheme + "://" + u.Host + "/llms.txt"
+	if rawURL == indexURL {
+		return nil
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	index, err := fetchPublicText(ctx, indexURL, 300000)
+	if err != nil || !strings.Contains(index, "](") {
+		return nil
+	}
+	return relatedSourceLinks(index, indexURL, question, limit)
+}
+
+func readSources(candidates []sourceCandidate, reader func(string) (string, error)) []sourceRead {
+	result := make([]sourceRead, len(candidates))
+	var wg sync.WaitGroup
+	for i, candidate := range candidates {
+		wg.Add(1)
+		go func(i int, candidate sourceCandidate) {
+			defer wg.Done()
+			content, err := reader(candidate.url)
+			result[i] = sourceRead{url: candidate.url, content: content, err: err}
+		}(i, candidate)
+	}
+	wg.Wait()
+	return result
+}
+
+func chooseSearchSources(query string, ranked []SearchDoc, limit int) []sourceCandidate {
+	items := make([]sourceCandidate, 0, limit)
+	terms := queryTerms(query)
+	for i, source := range ranked {
+		if i >= 18 {
+			break
+		}
+		if err := validateSourceURL(source.URL); err != nil || disallowedSourcePath(source.URL) {
+			continue
+		}
+		u, _ := url.Parse(source.URL)
+		score := 18 - i + termScore(source.Title, terms)*2 + termScore(u.Hostname()+" "+u.Path, terms)
+		if strings.HasPrefix(u.Hostname(), "docs.") || strings.Contains(u.Path, "/docs/") || strings.Contains(u.Path, "/reference/") {
+			score += 7
+		}
+		if strings.Contains(u.Path, "/specification/") || strings.Contains(u.Path, "/api-reference/") || strings.Contains(u.Path, "/sdk/") {
+			score += 7
+		}
+		if u.Hostname() == "github.com" || strings.Contains(u.Path, "openapi") {
+			score += 5
+		}
+		if strings.Contains(u.Path, "/community/") || strings.Contains(u.Path, "/interest-groups/") {
+			score -= 16 // governance pages rarely contain usable API contracts
+		}
+		if strings.Contains(u.Path, "/blog/") || strings.HasPrefix(u.Hostname(), "blog.") {
+			score -= 5
+		}
+		if strings.Contains(u.Path, "/internal/") && u.Hostname() == "pkg.go.dev" {
+			score -= 12 // internal Go packages aren't public SDK contracts
+		}
+		if strings.Contains(strings.ToLower(query), "go sdk") && strings.Contains(strings.ToLower(source.Title+" "+u.Path), "go-sdk") {
+			score += 7
+		}
+		items = append(items, sourceCandidate{url: source.URL, label: source.Title, score: score, order: i})
+	}
+	sort.SliceStable(items, func(i, j int) bool { return items[i].score > items[j].score })
+	selected := make([]sourceCandidate, 0, limit)
+	seenHosts := make(map[string]int)
+	for _, item := range items {
+		if len(selected) == limit {
+			break
+		}
+		u, _ := url.Parse(item.url)
+		if seenHosts[u.Hostname()] >= 2 { // diversify while allowing two relevant docs pages
+			continue
+		}
+		seenHosts[u.Hostname()]++
+		selected = append(selected, item)
+	}
+	return selected
+}
+
+func relatedSourceLinks(markdown, baseURL, question string, limit int) []sourceCandidate {
+	base, err := url.Parse(baseURL)
+	if err != nil {
+		return nil
+	}
+	terms := queryTerms(question)
+	seen := map[string]bool{canonicalSourceURL(base): true}
+	var links []sourceCandidate
+	for i, match := range mdLinkPattern.FindAllStringSubmatch(markdown, 300) {
+		destination := strings.TrimSpace(match[2])
+		if strings.Contains(destination, "\\") { // malformed Markdown-escaped URLs
+			continue
+		}
+		ref, err := url.Parse(destination)
+		if err != nil {
+			continue
+		}
+		u := base.ResolveReference(ref)
+		if !strings.EqualFold(base.Hostname(), u.Hostname()) || base.Port() != u.Port() || u.User != nil {
+			continue
+		}
+		if err := validateSourceURL(u.String()); err != nil || disallowedSourcePath(u.String()) || sensitiveURL(u) {
+			continue
+		}
+		key := canonicalSourceURL(u)
+		if seen[key] {
+			continue
+		}
+		seen[key] = true
+		label := strings.TrimSpace(match[1])
+		score := termScore(label, terms)*4 + termScore(u.Path, terms)*3
+		if score == 0 { // no relevant relationship to the user's question
+			continue
+		}
+		if len(terms) >= 4 && len(matchedLinkTerms(label+" "+u.Path, terms)) == 1 {
+			term := matchedLinkTerms(label+" "+u.Path, terms)[0]
+			if len(term) <= 5 { // don't expand generic links matching only 'MCP' or 'tools'
+				continue
+			}
+		}
+		links = append(links, sourceCandidate{url: key, label: label, score: score, order: i})
+	}
+	sort.SliceStable(links, func(i, j int) bool {
+		if links[i].score == links[j].score {
+			return links[i].order < links[j].order
+		}
+		return links[i].score > links[j].score
+	})
+	if len(links) > limit {
+		links = links[:limit]
+	}
+	return links
+}
+
+func matchedLinkTerms(text string, terms []string) []string {
+	var matched []string
+	for _, term := range terms {
+		if termScore(text, []string{term}) > 0 {
+			matched = append(matched, term)
+		}
+	}
+	return matched
+}
+
+func canonicalSourceURL(u *url.URL) string {
+	copy := *u
+	copy.Fragment = ""
+	copy.RawFragment = ""
+	return copy.String()
+}
+
+func sensitiveURL(u *url.URL) bool {
+	for name := range u.Query() {
+		name = strings.ToLower(name)
+		if strings.Contains(name, "token") || strings.Contains(name, "secret") || strings.Contains(name, "key") || strings.Contains(name, "auth") || strings.Contains(name, "signature") {
+			return true
+		}
+	}
+	return false
+}
+
+func disallowedSourcePath(raw string) bool {
+	u, err := url.Parse(raw)
+	if err != nil {
+		return true
+	}
+	path := strings.ToLower(u.Path)
+	for _, suffix := range []string{".png", ".jpg", ".jpeg", ".svg", ".gif", ".webp", ".zip", ".mp4", ".mov", ".mp3", ".woff", ".woff2", ".pdf", ".dmg", ".exe"} {
+		if strings.HasSuffix(path, suffix) {
+			return true
+		}
+	}
+	return false
+}
+
+func queryTerms(query string) []string {
+	stop := map[string]bool{"the": true, "and": true, "for": true, "from": true, "with": true, "that": true, "this": true, "what": true, "how": true, "when": true, "where": true, "does": true, "which": true, "using": true, "about": true, "into": true, "can": true, "are": true, "get": true, "use": true, "you": true, "your": true, "its": true, "have": true}
+	seen := make(map[string]bool)
+	var terms []string
+	for _, term := range sourceWordPattern.FindAllString(strings.ToLower(query), 40) {
+		if len(term) < 3 {
+			continue
+		}
+		if stop[term] || seen[term] {
+			continue
+		}
+		seen[term] = true
+		terms = append(terms, term)
+	}
+	return terms
+}
+
+func termScore(text string, terms []string) int {
+	// Compare words instead of substrings. Without this, an "api" search
+	// incorrectly matches "scraping" and chooses unrelated pages.
+	words := sourceWordPattern.FindAllString(strings.ToLower(text), -1)
+	count := 0
+	for _, term := range terms {
+		// Approximate inflection matching: pagination / paginate, authenticated /
+		// authentication, etc. Exact matches are weighted slightly higher.
+		root := term
+		if len(root) > 5 {
+			root = root[:5]
+		}
+		for _, word := range words {
+			if word == term {
+				count += 2
+				break
+			}
+			if len(root) >= 4 && strings.HasPrefix(word, root) {
+				count++
+				break
+			}
+		}
+	}
+	return count
+}
+
+func splitResearchSections(markdown string, terms []string) []researchSection {
+	lines := strings.Split(markdown, "\n")
+	sections := make([]researchSection, 0, 20)
+	current := researchSection{heading: "Introduction"}
+	var body strings.Builder
+	inFence := false
+	flush := func() {
+		current.body = strings.TrimSpace(body.String())
+		if current.body != "" {
+			current.index = len(sections)
+			current.score = 5*termScore(current.heading, terms) + termScore(current.body, terms)
+			if strings.Contains(current.body, "```") && current.score > 0 {
+				current.score += 2
+			}
+			sections = append(sections, current)
+		}
+		body.Reset()
+	}
+	for _, line := range lines {
+		trimmed := strings.TrimSpace(line)
+		if !inFence {
+			if heading := headingPattern.FindStringSubmatch(trimmed); len(heading) == 2 {
+				flush()
+				current = researchSection{heading: heading[1]}
+			}
+		}
+		body.WriteString(line)
+		body.WriteByte('\n')
+		if strings.HasPrefix(trimmed, "```") || strings.HasPrefix(trimmed, "~~~") {
+			inFence = !inFence
+		}
+	}
+	flush()
+	return sections
+}
+
+func focusExcerpt(markdown, question string, budget int) string {
+	markdown = strings.TrimSpace(markdown)
+	if len(markdown) <= budget {
+		return markdown
+	}
+	if focused := focusedRawJSON(markdown, question, budget); focused != "" {
+		return focused
+	}
+	sections := splitResearchSections(markdown, queryTerms(question))
+	if len(sections) == 0 {
+		return boundedMarkdown(markdown, budget)
+	}
+	sorted := append([]researchSection(nil), sections...)
+	sort.SliceStable(sorted, func(i, j int) bool {
+		if sorted[i].score == sorted[j].score {
+			return sorted[i].index < sorted[j].index
+		}
+		return sorted[i].score > sorted[j].score
+	})
+	selected := make([]researchSection, 0, 5)
+	for _, section := range sorted {
+		if len(selected) >= 5 || (section.score == 0 && len(selected) > 0) {
+			break
+		}
+		selected = append(selected, section)
+	}
+	sort.Slice(selected, func(i, j int) bool { return selected[i].index < selected[j].index })
+	var parts []string
+	remaining := budget
+	for i, section := range selected {
+		if remaining < 160 {
+			break
+		}
+		// Allocate budget across selected sections so one huge section doesn't
+		// crowd out exact schemas or examples from the other sections.
+		allocation := remaining / (len(selected) - i)
+		if allocation < 500 && remaining >= 500 {
+			allocation = 500
+		}
+		text := atomicFocusedSection(section.body, question, allocation)
+		parts = append(parts, text)
+		remaining -= len(text)
+	}
+	result := strings.Join(parts, "\n\n")
+	return result + "\n\n*[Selected excerpts; open the source URL to inspect omitted sections.]*"
+}
+
+func boundedMarkdown(markdown string, budget int) string {
+	if len(markdown) <= budget {
+		return markdown
+	}
+	if budget <= 80 {
+		return "*[Section too large; consult source.]*"
+	}
+	lines := strings.Split(markdown, "\n")
+	var out strings.Builder
+	inFence := false
+	fence := "```"
+	for _, line := range lines {
+		if out.Len()+len(line)+len("\n…\n```") > budget {
+			break
+		}
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "```") || strings.HasPrefix(trimmed, "~~~") {
+			if !inFence {
+				fence = trimmed[:3]
+			}
+			inFence = !inFence
+		}
+		out.WriteString(line)
+		out.WriteByte('\n')
+	}
+	if inFence {
+		out.WriteString(fence)
+		out.WriteByte('\n')
+	}
+	out.WriteString("*[Excerpt shortened; see source for full content.]*")
+	return out.String()
+}

package/internal/app/research_excerpt.go

@@ -0,0 +1,149 @@
+package app
+
+import (
+	"sort"
+	"strings"
+)
+
+const backtickFence = "\x60\x60\x60"
+
+type evidenceBlock struct {
+	text  string
+	code  bool
+	score int
+	order int
+}
+
+// A code fence is indivisible evidence: do not silently truncate a command,
+// request body or SDK example and then invent a closing fence.
+func splitEvidenceBlocks(markdown, question string) []evidenceBlock {
+	var blocks []evidenceBlock
+	var content strings.Builder
+	terms := queryTerms(question)
+	inCode := false
+	fence := ""
+	flush := func(code bool) {
+		text := strings.TrimSpace(content.String())
+		content.Reset()
+		if text == "" {
+			return
+		}
+		score := termScore(text, terms)
+		if code {
+			score *= 3
+			if strings.Contains(strings.ToLower(question), "rest") &&
+				(strings.Contains(text, "curl") || strings.Contains(text, "POST ") || strings.Contains(text, "GET ")) {
+				score += 4
+			}
+		}
+		blocks = append(blocks, evidenceBlock{text: text, code: code, score: score, order: len(blocks)})
+	}
+	for _, line := range strings.Split(markdown, "\n") {
+		trimmed := strings.TrimSpace(line)
+		opening := !inCode && (strings.HasPrefix(trimmed, backtickFence) || strings.HasPrefix(trimmed, "~~~"))
+		if opening {
+			flush(false)
+			inCode = true
+			fence = trimmed[:3]
+		}
+		content.WriteString(line)
+		content.WriteByte('\n')
+		if inCode && !opening && strings.TrimSpace(strings.TrimPrefix(trimmed, fence)) == "" &&
+			strings.HasPrefix(trimmed, fence) {
+			flush(true)
+			inCode = false
+			fence = ""
+		}
+	}
+	// Unclosed code blocks are not valid examples; treat them as source prose.
+	flush(false)
+	return blocks
+}
+
+// Select complete, high-relevance examples and explanatory prose from long
+// sections, preserving original order and the hard context budget.
+func atomicFocusedSection(markdown, question string, budget int) string {
+	markdown = strings.TrimSpace(markdown)
+	if len(markdown) <= budget {
+		return markdown
+	}
+	if budget < 180 {
+		return "*[Section exceeds excerpt budget; follow the source URL.]*"
+	}
+	blocks := splitEvidenceBlocks(markdown, question)
+	const omitted = "\n\n*[Other source text omitted. Follow the original URL for full details.]*"
+	remaining := budget - len(omitted) - 8
+	if remaining < 80 {
+		return "*[Section exceeds excerpt budget; follow the source URL.]*"
+	}
+	selected := make(map[int]string)
+	var codes []evidenceBlock
+	var prose []evidenceBlock
+	for _, block := range blocks {
+		if block.code {
+			codes = append(codes, block)
+		} else {
+			prose = append(prose, block)
+		}
+	}
+	sort.SliceStable(codes, func(i, j int) bool {
+		if codes[i].score == codes[j].score {
+			return codes[i].order < codes[j].order
+		}
+		return codes[i].score > codes[j].score
+	})
+	// Reserve a little space for the section explanation.
+	codeAllowance := remaining * 85 / 100
+	for _, block := range codes {
+		size := len(block.text) + 2
+		if size <= codeAllowance {
+			selected[block.order] = block.text
+			codeAllowance -= size
+			remaining -= size
+		}
+	}
+	sort.SliceStable(prose, func(i, j int) bool {
+		if prose[i].score == prose[j].score {
+			return prose[i].order < prose[j].order
+		}
+		return prose[i].score > prose[j].score
+	})
+	for _, block := range prose {
+		if remaining < 100 {
+			break
+		}
+		if len(block.text)+2 <= remaining {
+			selected[block.order] = block.text
+			remaining -= len(block.text) + 2
+		} else if part := completeProseLines(block.text, remaining-4); part != "" {
+			selected[block.order] = part
+			remaining -= len(part) + 2
+		}
+	}
+	var output []string
+	for _, block := range blocks {
+		if text := selected[block.order]; text != "" {
+			output = append(output, text)
+		}
+	}
+	if len(output) == 0 {
+		return "*[Code examples exceed the excerpt budget; inspect source URL for the full examples.]*"
+	}
+	return strings.Join(output, "\n\n") + omitted
+}
+
+func completeProseLines(text string, maxBytes int) string {
+	if maxBytes <= 0 {
+		return ""
+	}
+	var lines []string
+	size := 0
+	for _, line := range strings.Split(text, "\n") {
+		if size+len(line)+1 > maxBytes {
+			break
+		}
+		lines = append(lines, line)
+		size += len(line) + 1
+	}
+	return strings.TrimSpace(strings.Join(lines, "\n"))
+}