webctx 0.2.2 → 0.2.3

package/internal/app/structured_source.go

@@ -0,0 +1,249 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+)
+
+const maxDirectSourceBytes = 1024 * 1024
+
+var (
+	jsonLDPattern   = regexp.MustCompile(`(?is)<script\b[^>]*type\s*=\s*["']application/ld\+json["'][^>]*>(.*?)</script\s*>`)
+	nextDataPattern = regexp.MustCompile(`(?is)<script\b[^>]*id\s*=\s*["']__NEXT_DATA__["'][^>]*>(.*?)</script\s*>`)
+)
+
+// These URLs are already structured source documents. Never scrape a raw
+// OpenAPI schema through HTML rendering when we can retrieve its actual bytes.
+func directSourceLanguage(rawURL string) string {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return ""
+	}
+	switch strings.ToLower(path.Ext(u.Path)) {
+	case ".json", ".jsonl":
+		return "json"
+	case ".yaml", ".yml":
+		return "yaml"
+	case ".toml":
+		return "toml"
+	case ".xml":
+		return "xml"
+	case ".graphql", ".gql":
+		return "graphql"
+	case ".txt":
+		return "text"
+	default:
+		return ""
+	}
+}
+
+// A bounded, redirect-checked direct read prevents accidental traversal from
+// an accepted public source to an internal address.
+func fetchPublicText(ctx context.Context, rawURL string, maxBytes int64) (string, error) {
+	if err := validateSourceURL(rawURL); err != nil {
+		return "", err
+	}
+	client := *http.DefaultClient
+	oldCheck := client.CheckRedirect
+	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		if err := validateSourceURL(req.URL.String()); err != nil {
+			return err
+		}
+		if oldCheck != nil {
+			return oldCheck(req, via)
+		}
+		if len(via) >= 10 {
+			return fmt.Errorf("too many redirects")
+		}
+		return nil
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("Accept", "application/json, text/markdown, text/plain, text/html, application/yaml, */*;q=0.5")
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return "", fmt.Errorf("source HTTP %d", resp.StatusCode)
+	}
+	if resp.ContentLength > maxBytes {
+		return "", fmt.Errorf("source is larger than the supported %d bytes", maxBytes)
+	}
+	body, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1))
+	if err != nil {
+		return "", err
+	}
+	if int64(len(body)) > maxBytes {
+		return "", fmt.Errorf("source exceeded the supported %d bytes", maxBytes)
+	}
+	return string(body), nil
+}
+
+func readDirectSource(rawURL string) (string, bool) {
+	language := directSourceLanguage(rawURL)
+	if language == "" {
+		return "", false
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
+	defer cancel()
+	body, err := fetchPublicText(ctx, rawURL, maxDirectSourceBytes)
+	if err != nil || strings.TrimSpace(body) == "" {
+		return "", false
+	}
+	if strings.Contains(strings.ToLower(body[:min(len(body), 256)]), "<html") {
+		return "", false
+	}
+	if language == "json" && !json.Valid([]byte(body)) { // don't label HTML/garbage as JSON
+		return "", false
+	}
+	return fmt.Sprintf("**URL:** %s\n\n```%s\n%s\n```", rawURL, language, strings.TrimSpace(body)), true
+}
+
+// Extract only public machine-readable metadata actually embedded in the
+// original HTML. Never execute arbitrary scripts or speculate about hidden APIs.
+func embeddedStructuredContext(rawURL, question string) string {
+	if question == "" || directSourceLanguage(rawURL) != "" {
+		return ""
+	}
+	if u, err := url.Parse(rawURL); err == nil && strings.EqualFold(u.Hostname(), "github.com") {
+		return "" // GitHub's native APIs are more faithful than page hydration data.
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)
+	defer cancel()
+	html, err := fetchPublicText(ctx, rawURL, maxDirectSourceBytes)
+	if err != nil || !strings.Contains(strings.ToLower(html), "<script") {
+		return ""
+	}
+	var results []string
+	for _, pattern := range []*regexp.Regexp{jsonLDPattern, nextDataPattern} {
+		for _, match := range pattern.FindAllStringSubmatch(html, 3) {
+			var raw any
+			if json.Unmarshal([]byte(strings.TrimSpace(match[1])), &raw) != nil {
+				continue
+			}
+			selection := selectedJSONFields(raw, question, 2500)
+			if selection != "" {
+				results = append(results, selection)
+			}
+		}
+	}
+	if len(results) == 0 {
+		return ""
+	}
+	return "### Embedded source metadata\n\nStructured JSON embedded in the source page (not interpreted as an API response):\n\n" + strings.Join(results, "\n\n")
+}
+
+// selectedJSONFields preserves exact JSON values and JSON Pointers rather
+// than returning syntactically broken chunks of a large schema or payload.
+func selectedJSONFields(raw any, question string, budget int) string {
+	type field struct {
+		path  string
+		value any
+		score int
+	}
+	terms := queryTerms(question)
+	fields := make([]field, 0, 32)
+	var walk func(any, string, int)
+	walk = func(value any, parent string, depth int) {
+		if depth > 9 || len(fields) >= 500 {
+			return
+		}
+		switch v := value.(type) {
+		case map[string]any:
+			for key, child := range v {
+				pointer := parent + "/" + strings.ReplaceAll(strings.ReplaceAll(key, "~", "~0"), "/", "~1")
+				score := termScore(key, terms)*4 + termScore(pointer, terms)
+				if score > 0 {
+					fields = append(fields, field{path: pointer, value: child, score: score})
+				}
+				walk(child, pointer, depth+1)
+			}
+		case []any:
+			for i, child := range v {
+				if i >= 20 { // never flatten unbounded payload collections
+					break
+				}
+				walk(child, fmt.Sprintf("%s/%d", parent, i), depth+1)
+			}
+		}
+	}
+	walk(raw, "", 0)
+	sort.SliceStable(fields, func(i, j int) bool {
+		if fields[i].score == fields[j].score {
+			return fields[i].path < fields[j].path
+		}
+		return fields[i].score > fields[j].score
+	})
+	var out []string
+	for _, f := range fields {
+		value, err := json.MarshalIndent(f.value, "", "  ")
+		if err != nil || len(value) > budget/2 || len(value) < 2 {
+			continue
+		}
+		item := fmt.Sprintf("**JSON Pointer:** `%s`\n\n```json\n%s\n```", f.path, value)
+		if len(strings.Join(out, "\n\n"))+len(item) > budget {
+			break
+		}
+		out = append(out, item)
+		if len(out) >= 5 {
+			break
+		}
+	}
+	if len(out) > 0 {
+		return strings.Join(out, "\n\n") + "\n\n*[Selected JSON fields; full document at source URL.]*"
+	}
+	pretty, err := json.MarshalIndent(raw, "", "  ")
+	if err != nil {
+		return ""
+	}
+	if len(pretty)+20 <= budget {
+		return "```json\n" + string(pretty) + "\n```"
+	}
+	// No relevant key matched: expose the shape without claiming completeness.
+	if obj, ok := raw.(map[string]any); ok {
+		keys := make([]string, 0, len(obj))
+		for key := range obj {
+			keys = append(keys, key)
+		}
+		sort.Strings(keys)
+		if len(keys) > 25 {
+			keys = keys[:25]
+		}
+		return "**Top-level JSON keys:** " + strings.Join(keys, ", ") + " (full document at source URL)"
+	}
+	return "*[Structured document is larger than the inline budget; inspect original source.]*"
+}
+
+func focusedRawJSON(markdown, question string, budget int) string {
+	begin := strings.Index(markdown, "```json\n")
+	if begin < 0 {
+		return ""
+	}
+	jsonStart := begin + len("```json\n")
+	end := strings.LastIndex(markdown, "\n```")
+	if end <= jsonStart {
+		return ""
+	}
+	var source any
+	if json.Unmarshal([]byte(markdown[jsonStart:end]), &source) != nil {
+		return ""
+	}
+	selected := selectedJSONFields(source, question, budget-250)
+	if selected == "" {
+		return ""
+	}
+	return strings.TrimSpace(markdown[:begin]) + "\n\n**Focused JSON evidence:**\n\n" + selected
+}

package/internal/app/structured_source_test.go

@@ -0,0 +1,70 @@
+package app
+
+import (
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func TestDirectOpenAPIJSONBypassesScraping(t *testing.T) {
+	old := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = old })
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		if req.URL.Host != "docs.example.com" || req.URL.Path != "/openapi.json" || req.Method != http.MethodGet {
+			t.Fatalf("unexpected direct source request: %s", req.URL)
+		}
+		return testHTTPResponse(req, http.StatusOK, `{"openapi":"3.1.0","paths":{"/v2/scrape":{}}}`, nil), nil
+	})}
+	got, err := ReadLink("https://docs.example.com/openapi.json")
+	if err != nil || !strings.Contains(got, `"openapi":"3.1.0"`) || !strings.Contains(got, "```json") {
+		t.Fatalf("did not fetch raw OpenAPI: %q, %v", got, err)
+	}
+}
+
+func TestDirectSourceRejectsPrivateRedirects(t *testing.T) {
+	old := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = old })
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		if req.URL.Host != "docs.example.com" {
+			t.Fatalf("followed unsafe redirect to %s", req.URL)
+		}
+		return testHTTPResponse(req, http.StatusFound, "", map[string]string{"Location": "http://127.0.0.1/private"}), nil
+	})}
+	if _, err := fetchPublicText(t.Context(), "https://docs.example.com/openapi.json", 100); err == nil {
+		t.Fatal("should reject public-to-private redirects")
+	}
+}
+
+func TestAlexandriaDiscoveryFallsBackWhenNotEnabled(t *testing.T) {
+	old := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = old })
+	var calls int
+	http.DefaultClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		body, _ := io.ReadAll(req.Body)
+		calls++
+		if calls == 1 {
+			if !strings.Contains(string(body), `"domainTools":true`) {
+				t.Fatalf("Alexandria discovery not enabled: %s", body)
+			}
+			return testHTTPResponse(req, http.StatusForbidden, `{"error":"Alexandria unavailable"}`, nil), nil
+		}
+		if !strings.Contains(string(body), `"domainTools":false`) {
+			t.Fatalf("fallback did not disable domain matching: %s", body)
+		}
+		return testHTTPResponse(req, http.StatusOK, `{"success":true,"data":{"markdown":"# Good source\n\nAuthoritative page content."}}`, nil), nil
+	})}
+	result, err := firecrawlScrape("https://docs.example.com/reference", "fake-test-key", "")
+	if err != nil || calls != 2 || !strings.Contains(result.Markdown, "Good source") {
+		t.Fatalf("fallback broke regular scraping: %#v, %v, calls=%d", result, err, calls)
+	}
+}
+
+func TestAlexandriaToolDescriptionIsDiscoveryOnly(t *testing.T) {
+	got := formatAlexandriaTools([]any{
+		map[string]any{"provider": "particle", "capability": "podcasts/episodes/search", "description": "Find episodes"},
+	})
+	if !strings.Contains(got, "particle/podcasts/episodes/search") || !strings.Contains(got, "not executed") {
+		t.Fatalf("bad tool discovery metadata: %s", got)
+	}
+}

package/internal/app/tools.go

@@ -51,6 +51,11 @@ func Search(params SearchParams) (string, error) {
 	allExcludedDomains := append(append([]string{}, defaultExcludedDomains...), params.ExcludeDomains...)
 	truncatedKeyword := truncateWords(params.IncludeKeyword, 5)
 
+	// Free Alexandria tool discovery proceeds alongside normal web search.
+	// It never executes a paid tool or replaces source-grounded web results.
+	alexandriaDone := make(chan string, 1)
+	go func() { alexandriaDone <- discoverAlexandria(params.Query) }()
+
 	type namedSearch struct {
 		name string
 		fn   func(context.Context) (SearchResult, error)
@@ -118,6 +123,15 @@ func Search(params SearchParams) (string, error) {
 	}
 
 	parts := []string{fmt.Sprintf("Total Results: %d\n", len(ranked))}
+	// The ranked index remains available, but agents also get useful source
+	// context without having to issue several follow-up read-link calls.
+	if context := enrichSearch(params.Query, ranked); context != "" {
+		parts = append(parts, context, "")
+	}
+	if tools := <-alexandriaDone; tools != "" {
+		parts = append(parts, "", "## Related data providers", tools, "")
+	}
+	parts = append(parts, "## Search results", "")
 	for _, doc := range ranked {
 		title := decodeHTML(doc.Title)
 		overview := decodeHTML(doc.Overview)
@@ -153,6 +167,10 @@ func ReadLink(rawURL string) (string, error) {
 		return "", native.Err
 	}
 
+	if direct, ok := readDirectSource(rawURL); ok {
+		return direct, nil
+	}
+
 	if ok, _ := checkMarkdownAvailable(rawURL); ok {
 		result, err := fetchMarkdownContent(rawURL)
 		if err == nil {
@@ -201,14 +219,14 @@ func scrapeLinkWithFirecrawl(rawURL string) (string, string, error) {
 	// scrape responds successfully but returns a challenge/empty document.
 	first, err := firecrawlScrape(rawURL, apiKey, "")
 	if err == nil && usableScrapedMarkdown(first.Markdown) {
-		return first.Title, first.Markdown, nil
+		return first.Title, withSourceTools(first.Markdown, first.ToolSummary), nil
 	}
 	lastError := err
 	if err == nil {
 		lastError = fmt.Errorf("Firecrawl returned empty or blocked page content")
 		second, enhancedErr := firecrawlScrape(rawURL, apiKey, "enhanced")
 		if enhancedErr == nil && usableScrapedMarkdown(second.Markdown) {
-			return second.Title, second.Markdown, nil
+			return second.Title, withSourceTools(second.Markdown, second.ToolSummary), nil
 		}
 		if enhancedErr != nil {
 			lastError = enhancedErr

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webctx",
-  "version": "0.2.2",
+  "version": "0.2.3",
   "description": "Pure Go web search and browsing CLI using Brave, Tavily, Exa, and Firecrawl",
   "license": "MIT",
   "author": "amxv",

package/pkg/origo/server.go

@@ -18,14 +18,20 @@ type linkInput struct {
 	URL string `json:"url" jsonschema:"Absolute HTTP(S) URL to read as source-grounded Markdown"`
 }
 
+type readLinkInput struct {
+	URL      string `json:"url" jsonschema:"Absolute HTTP(S) URL to read as source-grounded Markdown"`
+	Question string `json:"question,omitempty" jsonschema:"Optional question or task. When set, automatically follows relevant source links and returns focused, cited context."`
+}
+
 func makeServer() *mcp.Server {
 	s := mcp.NewServer(&mcp.Implementation{Name: "origo", Version: "0.1.0"}, nil)
 	mcp.AddTool(s, &mcp.Tool{
 		Name:        "read_link",
 		Title:       "Read link",
-		Description: "Read a specific web page as clean Markdown, preferring native GitHub or site Markdown and using Firecrawl proxies/browser fallback when needed. Does not search the web.",
-	}, func(_ context.Context, _ *mcp.CallToolRequest, input linkInput) (*mcp.CallToolResult, any, error) {
-		result, err := retrieval.ReadLink(input.URL)
+		Description: "Read a URL as source-grounded Markdown. Optionally provide a question to automatically explore related docs and return relevant API contracts, code examples and precise source URLs. No search query required; no paid Alexandria tool execution.",
+		Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true},
+	}, func(_ context.Context, _ *mcp.CallToolRequest, input readLinkInput) (*mcp.CallToolResult, any, error) {
+		result, err := retrieval.ReadLinkFocused(input.URL, input.Question)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -35,6 +41,7 @@ func makeServer() *mcp.Server {
 		Name:        "map_site",
 		Title:       "Map site",
 		Description: "Discover URLs on a site using Firecrawl's sitemap-based map API. Does not perform web search.",
+		Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true},
 	}, func(_ context.Context, _ *mcp.CallToolRequest, input linkInput) (*mcp.CallToolResult, any, error) {
 		result, err := retrieval.MapSite(input.URL)
 		if err != nil {

package/pkg/origo/server_test.go

@@ -60,4 +60,19 @@ func TestExactlyTwoToolsAndNoSearch(t *testing.T) {
 	if count := strings.Count(text, `"name":`); count != 2 {
 		t.Fatalf("expected exactly 2 tools, got %d: %s", count, text)
 	}
+	if count := strings.Count(text, `"readOnlyHint":true`); count != 2 {
+		t.Fatalf("expected both tools to declare read-only annotations, got %d: %s", count, text)
+	}
+}
+
+func TestQuestionIsOptionalOnReadLinkOnly(t *testing.T) {
+	t.Setenv("ORIGO_API_KEY", "secret")
+	response := request(t, "secret", `{"jsonrpc":"2.0","id":4,"method":"tools/list","params":{}}`)
+	if response.Code != http.StatusOK {
+		t.Fatalf("list tools: %s", response.Body.String())
+	}
+	text := response.Body.String()
+	if !strings.Contains(text, `"question"`) || !strings.Contains(text, `"required":["url"]`) {
+		t.Fatalf("question should be optional; URL remains required: %s", text)
+	}
 }

package/pkg/retrieval/retrieval.go

@@ -8,4 +8,10 @@ import "github.com/amxv/webctx/internal/app"
 
 func ReadLink(url string) (string, error) { return app.ReadLink(url) }
 
+// ReadLinkFocused follows related documentation when the caller has a question.
+// A URL-only call retains the original complete-document behavior.
+func ReadLinkFocused(url, question string) (string, error) {
+	return app.ReadLinkFocused(url, question)
+}
+
 func MapSite(url string) (string, error) { return app.MapSite(url) }