webctx 0.2.6 → 0.2.7

package/internal/app/data_research.go

@@ -1,6 +1,7 @@
 package app
 
 import (
+	"net/url"
 	"sort"
 	"strings"
 )
@@ -60,7 +61,7 @@ func normalizeResearchInput(input ResearchDataInput) (ResearchDataInput, string,
 	if err != nil {
 		return input, "", nil, err
 	}
-	if input.Limit == 0 {
+	if input.Limit == 0 && !input.limitProvided {
 		input.Limit = 10
 	}
 	if input.Limit < 1 || input.Limit > 100 {
@@ -74,6 +75,16 @@ func normalizeResearchInput(input ResearchDataInput) (ResearchDataInput, string,
 			return input, "", nil, dataError("invalid_url", "Website filters require public HTTP(S) URLs.")
 		}
 	}
+	if len(input.URLs) > 0 && len(input.Sources) == 0 {
+		for _, target := range input.URLs {
+			u, _ := url.Parse(target)
+			host := strings.TrimPrefix(strings.ToLower(u.Hostname()), "www.")
+			if host != "" {
+				source := strings.ReplaceAll(host, ".", "-")
+				input.Sources = append(input.Sources, source)
+			}
+		}
+	}
 	return input, view, expand, nil
 }
 
@@ -82,7 +93,7 @@ func nativeBrowseOptions(input ResearchDataInput, view string, expand []string)
 	if input.Query != "" {
 		o["query"] = input.Query
 	}
-	if len(input.URLs) > 0 {
+	if len(input.URLs) > 0 && len(input.Sources) == 0 {
 		o["urls"] = input.URLs
 	}
 	if len(input.Sources) > 0 {
@@ -123,6 +134,9 @@ func nativeBrowseOptions(input ResearchDataInput, view string, expand []string)
 		}
 	}
 	if len(expand) > 0 {
+		if containsString(expand, "examples") && !containsString(expand, "options") {
+			expand = append(append([]string(nil), expand...), "options")
+		}
 		o["expand"] = expand
 	}
 	return o
@@ -145,9 +159,19 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 	}
 	ctx, cancel := defaultDataContext()
 	defer cancel()
+	explicitExact := len(input.Operations) > 0
+	// An exact operation ID is authoritative. A descriptive query helps
+	// communicate intent, but must not exclude a known operation.
+	note := ""
+	if explicitExact && input.Query != "" {
+		note = "Exact operation IDs take precedence over descriptive query terms; the query was not used as a restrictive upstream filter."
+		input.Query = ""
+	}
 	semantic := input.Mode != "catalogue" && input.Query != "" && view == "tools" && input.Offset == 0 &&
 		len(input.URLs) == 0 && len(input.Sources) == 0 && len(input.Categories) == 0 &&
 		len(input.Groups) == 0 && len(input.Operations) == 0
+	localFilteredRanking := view == "tools" && input.Query != "" &&
+		(len(input.URLs) > 0 || len(input.Sources) > 0 || len(input.Groups) > 0 || len(input.Categories) > 0)
 	var items []any
 	var total any
 	var next any
@@ -159,9 +183,16 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 		// compact upstream matches may omit both price and operation name.
 		// We still return compact result cards unless include was requested.
 		detail := "full"
+		fetchLimit := input.Limit * 4
+		if fetchLimit < 20 {
+			fetchLimit = 20
+		}
+		if fetchLimit > 100 {
+			fetchLimit = 100
+		}
 		raw, err = doDataAPI(ctx, "/search", map[string]any{
 			"query": input.Query, "sources": []string{"alexandria"},
-			"limit": input.Limit, "toolDetail": detail,
+			"limit": fetchLimit, "toolDetail": detail,
 		}, "")
 		if err != nil {
 			return nil, err
@@ -171,17 +202,6 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 		// Some upstream ranked matches arrive out of order despite including a
 		// similarity score. Present the highest-scoring candidate first, retaining
 		// upstream order for exact ties or missing scores.
-		sort.SliceStable(items, func(i, j int) bool {
-			a, aok := numberField(mapField(items[i])["similarity"])
-			b, bok := numberField(mapField(items[j])["similarity"])
-			if aok != bok {
-				return aok
-			}
-			if !aok {
-				return false
-			}
-			return a > b
-		})
 		// Ranked search has no native cursor. Expose the free, complete
 		// catalogue with matching semantic query as an optional expansion.
 		next = map[string]any{"tool": "research", "arguments": map[string]any{
@@ -189,6 +209,22 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 		}, "note": "Browse all matching operations with pagination; ranked discovery itself has no cursor."}
 	} else {
 		options := nativeBrowseOptions(input, view, expand)
+		if localFilteredRanking {
+			// Provider-specified lexical filters can exclude operations whose
+			// descriptions describe the requested data indirectly. Fetch a
+			// catalogue slice from the actual source, then rank it ourselves.
+			delete(options, "query")
+			options["limit"] = 100
+			options["offset"] = 0
+			note = "The source/category filters select candidates first. The question ranks those candidates locally, rather than excluding matching source operations."
+			if input.Mode != "catalogue" {
+				mode = "ranked_filtered"
+			}
+		}
+		if explicitExact {
+			delete(options, "query")
+			delete(options, "urls")
+		}
 		var catalogue map[string]any
 		catalogue, raw, err = freeDataBrowse(ctx, options)
 		if err != nil {
@@ -200,6 +236,48 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 			next = continuationFromCatalogue(native)
 		}
 	}
+	if view == "tools" {
+		termsQuery := input.Query
+		for _, op := range input.Operations {
+			termsQuery += " " + op
+		}
+		if strings.TrimSpace(termsQuery) != "" {
+			sort.SliceStable(items, func(i, j int) bool {
+				return intentRelevance(mapField(items[i]), termsQuery) > intentRelevance(mapField(items[j]), termsQuery)
+			})
+		} else {
+			sort.SliceStable(items, func(i, j int) bool {
+				a, aok := numberField(mapField(items[i])["similarity"])
+				b, bok := numberField(mapField(items[j])["similarity"])
+				if aok != bok {
+					return aok
+				}
+				return a > b
+			})
+		}
+	}
+	if semantic && len(items) > input.Limit {
+		items = items[:input.Limit]
+	}
+	if localFilteredRanking {
+		count := len(items)
+		start := input.Offset
+		if start > count {
+			start = count
+		}
+		end := start + input.Limit
+		if end > count {
+			end = count
+		}
+		items = items[start:end]
+		if count > end {
+			next = map[string]any{"tool": "research", "arguments": ResearchDataInput{
+				Query: input.Query, Mode: input.Mode, URLs: input.URLs, Sources: input.Sources,
+				Categories: input.Categories, Groups: input.Groups, View: niceDataView(view),
+				Include: input.Include, Limit: input.Limit, Offset: end,
+			}}
+		}
+	}
 	results := make([]map[string]any, 0, len(items))
 	for _, value := range items {
 		entry := mapField(value)
@@ -223,6 +301,15 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 		}
 		if cost, ok := entry["creditsCost"]; ok {
 			result["credits_per_call"] = cost
+			if listed, ok := numberField(cost); ok {
+				if listed > float64(dataCreditCap()) {
+					result["executable"] = false
+					result["availability"] = "exceeds_server_credit_limit"
+					result["server_credit_limit"] = dataCreditCap()
+				} else if view == "tools" {
+					result["executable"] = true
+				}
+			}
 		}
 		if cost, ok := entry["perRecord"]; ok {
 			result["per_record"] = cost
@@ -230,8 +317,14 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 		if count, ok := entry["toolCount"]; ok {
 			result["operation_count"] = count
 		}
+		if input.Query != "" && view == "tools" {
+			result["relevance"] = intentRelevance(entry, input.Query)
+		}
 		if rank, ok := entry["similarity"]; ok {
-			result["relevance"] = rank
+			result["semantic_similarity"] = rank
+			if input.Query == "" {
+				result["relevance"] = rank
+			}
 		}
 		if len(expand) > 0 {
 			if selected := entry["options"]; selected != nil && containsString(expand, "options") {
@@ -243,8 +336,15 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 			if selected := entry["examples"]; selected != nil && containsString(expand, "examples") {
 				result["examples"] = selected
 			}
+			if containsString(expand, "examples") && result["examples"] == nil {
+				if entry["options"] != nil {
+					result["examples"] = contractExamples(entry)
+				}
+			}
 		}
-		if continuation := mapField(entry["next"]); continuation != nil {
+		if view == "tools" && id != "" {
+			result["next"] = map[string]any{"tool": "inspect", "arguments": map[string]any{"id": id}}
+		} else if continuation := mapField(entry["next"]); continuation != nil {
 			result["next"] = continuationFromCatalogue(continuation)
 		} else if id != "" {
 			result["next"] = map[string]any{"tool": "inspect", "arguments": map[string]any{"id": id}}
@@ -256,6 +356,12 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 		"count": len(results), "cost_credits": 0,
 		"guidance": "Use inspect with an operation ID to see complete inputs, output, examples and the next execute call. To explore a source, inspect its source ID.",
 	}
+	if note != "" {
+		output["mode_note"] = note
+	}
+	if len(input.URLs) > 0 && len(input.Sources) > 0 {
+		output["resolved_url_sources"] = input.Sources
+	}
 	if total != nil {
 		output["total"] = total
 	}
@@ -268,6 +374,55 @@ func ResearchData(input ResearchDataInput) (map[string]any, error) {
 	return output, nil
 }
 
+// Rank operations by their ability to deliver the requested data, not just
+// semantic similarity. Keep the provider similarity separately for auditing.
+func intentRelevance(entry map[string]any, query string) float64 {
+	if entry == nil {
+		return 0
+	}
+	base, _ := numberField(entry["similarity"])
+	if query == "" {
+		return base
+	}
+	q := strings.ToLower(query)
+	id := strings.ToLower(stringField(entry["id"]))
+	name := strings.ToLower(stringField(entry["name"]))
+	desc := strings.ToLower(stringField(entry["description"]))
+	p := id + " " + name
+	searchIntent := strings.Contains(q, "search") || strings.Contains(q, "find") ||
+		strings.Contains(q, "papers") || strings.Contains(q, "podcast") ||
+		strings.Contains(q, "trials") || strings.Contains(q, "about")
+	historyIntent := strings.Contains(q, "historical") || strings.Contains(q, "history") ||
+		strings.Contains(q, "time series") || strings.Contains(q, "over time") ||
+		strings.Contains(q, "inflation")
+	if historyIntent && (strings.Contains(p, "observations") ||
+		strings.Contains(p, "history") || strings.Contains(p, "rates") ||
+		strings.Contains(p, "timeseries")) {
+		base += 0.23
+	}
+	if searchIntent && (strings.Contains(p, "/search") || strings.Contains(p, "/studies") ||
+		strings.Contains(p, "episode-search")) {
+		base += 0.20
+	}
+	if searchIntent && strings.Contains(p, "/list") && !strings.Contains(q, "list") {
+		base -= 0.04
+	}
+	for _, ancillary := range []string{"suggest", "related", "directory", "releases", "/source", "metadata", "browse"} {
+		if strings.Contains(p, ancillary) && !strings.Contains(q, strings.TrimLeft(ancillary, "/")) {
+			base -= 0.14
+		}
+	}
+	terms := queryTerms(query)
+	if len(terms) > 0 {
+		fit := termScore(name+" "+desc, terms)
+		if fit > 8 {
+			fit = 8
+		}
+		base += float64(fit) * 0.007
+	}
+	return float64(int(base*1000000+0.5)) / 1000000
+}
+
 func niceDataView(view string) string {
 	switch view {
 	case "providers":

package/internal/app/data_tools_test.go

@@ -286,8 +286,8 @@ func TestExecuteHandlesProviderTermsWithoutAccepting(t *testing.T) {
 		mapField(typed.RequiresAction)["url"] != "https://firecrawl.dev/app/settings?tab=data-sources" {
 		t.Fatalf("terms condition lost: %+v", err)
 	}
-	if mapField(typed.Details)["request_id"] == nil {
-		t.Fatalf("lost retriable request identifier: %+v", typed)
+	if mapField(typed.Details)["retry"] != nil {
+		t.Fatalf("terms rejection must not recommend automatic paid retry: %+v", typed)
 	}
 }
 

package/internal/app/data_types.go

@@ -1,20 +1,43 @@
 package app
 
+import (
+	"bytes"
+	"encoding/json"
+)
+
 // ResearchData discovers data sources and their available operations. Search
 // and catalogue browsing are both free Firecrawl Alexandria operations.
 type ResearchDataInput struct {
-	Query      string   `json:"query,omitempty" jsonschema:"Optional natural-language description of the information needed"`
-	Mode       string   `json:"mode,omitempty" jsonschema:"ranked semantic discovery or catalogue browsing with filters and pagination; defaults to ranked for a query"`
-	URLs       []string `json:"urls,omitempty" jsonschema:"Filter by websites associated with a data source"`
-	Sources    []string `json:"sources,omitempty" jsonschema:"Filter by source IDs returned by research"`
-	Categories []string `json:"categories,omitempty" jsonschema:"Filter by catalogue category IDs"`
-	Groups     []string `json:"groups,omitempty" jsonschema:"Filter by catalogue group IDs"`
-	Operations []string `json:"operations,omitempty" jsonschema:"Filter by exact operation IDs"`
-	View       string   `json:"view,omitempty" jsonschema:"What to browse: sources, groups, or operations; default operations for a query, otherwise sources"`
-	Include    []string `json:"include,omitempty" jsonschema:"Additional contract details: inputs, output, examples. Default is a concise overview"`
-	Limit      int      `json:"limit,omitempty" jsonschema:"Maximum results in this page, 1-100; default 10"`
-	Offset     int      `json:"offset,omitempty" jsonschema:"Catalogue pagination offset, zero-based"`
-	Raw        bool     `json:"raw,omitempty" jsonschema:"Also include the complete original Firecrawl response"`
+	Query         string   `json:"query,omitempty" jsonschema:"Optional natural-language description of the information needed"`
+	Mode          string   `json:"mode,omitempty" jsonschema:"ranked semantic discovery or catalogue browsing with filters and pagination; defaults to ranked for a query"`
+	URLs          []string `json:"urls,omitempty" jsonschema:"Filter by websites associated with a data source"`
+	Sources       []string `json:"sources,omitempty" jsonschema:"Filter by source IDs returned by research"`
+	Categories    []string `json:"categories,omitempty" jsonschema:"Filter by catalogue category IDs"`
+	Groups        []string `json:"groups,omitempty" jsonschema:"Filter by catalogue group IDs"`
+	Operations    []string `json:"operations,omitempty" jsonschema:"Filter by exact operation IDs"`
+	View          string   `json:"view,omitempty" jsonschema:"What to browse: sources, groups, or operations; default operations for a query, otherwise sources"`
+	Include       []string `json:"include,omitempty" jsonschema:"Additional contract details: inputs, output, examples. Default is a concise overview"`
+	Limit         int      `json:"limit,omitempty" jsonschema:"Maximum results in this page, 1-100; default 10"`
+	Offset        int      `json:"offset,omitempty" jsonschema:"Catalogue pagination offset, zero-based"`
+	Raw           bool     `json:"raw,omitempty" jsonschema:"Also include the complete original Firecrawl response"`
+	limitProvided bool     `json:"-"`
+}
+
+func (input *ResearchDataInput) UnmarshalJSON(raw []byte) error {
+	type noMethods ResearchDataInput
+	var parsed noMethods
+	decoder := json.NewDecoder(bytes.NewReader(raw))
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(&parsed); err != nil {
+		return err
+	}
+	var fields map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &fields); err != nil {
+		return err
+	}
+	*input = ResearchDataInput(parsed)
+	_, input.limitProvided = fields["limit"]
+	return nil
 }
 
 // InspectDataInput returns the exact contract for a selected operation, or

package/internal/app/research.go

@@ -88,6 +88,9 @@ func readLinkFocused(rawURL, question string, reader func(string) (string, error
 		return "", err
 	}
 	primary = copyableCurlExamples(primary)
+	if answer, ok := directlyAnsweredMetadata(rawURL, question, primary); ok {
+		return answer, nil
+	}
 
 	// Combine on-page links and authoritative Markdown index links instead of
 	// filling the entire budget from whichever links happen to appear first.
@@ -95,7 +98,7 @@ func readLinkFocused(rawURL, question string, reader func(string) (string, error
 	// SQLite, webhook, and API references over incidental alarm/concept pages.
 	links := researchCandidates(rawURL, primary, question, maxRelatedSources)
 	structuredDone := make(chan string, 1)
-	if strings.Contains(primary, "**Markdown source:**") {
+	if strings.Contains(primary, "**Markdown source:**") || !asksForAPIContracts(question) {
 		structuredDone <- ""
 	} else {
 		go func() { structuredDone <- embeddedStructuredContext(rawURL, question) }()

package/internal/app/research_answers.go

@@ -0,0 +1,30 @@
+package app
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+var githubLicenseField = regexp.MustCompile(`(?mi)^license:\s*["']?([A-Za-z0-9.+-]+)["']?\s*$`)
+var repositoryLicenseURL = regexp.MustCompile(`https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/blob/[A-Za-z0-9_.-]+/(?:LICENSE|COPYING)(?:\.md|\.txt)?`)
+
+func directlyAnsweredMetadata(rawURL, question, markdown string) (string, bool) {
+	q := strings.ToLower(question)
+	if !strings.Contains(q, "license") || !strings.Contains(strings.ToLower(rawURL), "github.com/") {
+		return "", false
+	}
+	match := githubLicenseField.FindStringSubmatch(markdown)
+	if len(match) != 2 {
+		return "", false
+	}
+	value := match[1]
+	evidenceURL := rawURL
+	if source := repositoryLicenseURL.FindString(markdown); source != "" {
+		evidenceURL = source
+	}
+	return fmt.Sprintf(
+		"## Answer\n\nThe repository lists its license as **%s**.\n\n**Source:** %s\n\n**Evidence:** GitHub repository metadata lists `license: %q`.\n",
+		value, evidenceURL, value,
+	), true
+}

package/internal/app/site_map_page.go

@@ -0,0 +1,256 @@
+package app
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/sync/singleflight"
+)
+
+// MapSiteInput retrieves an agent-sized page rather than dumping thousands
+// of URLs. An omitted language prefers English; "all" allows all locales.
+type MapSiteInput struct {
+	URL        string `json:"url" jsonschema:"Website URL to map; HTTP(S) only"`
+	Query      string `json:"query,omitempty" jsonschema:"Optional topic to filter and rank discovered pages"`
+	Limit      int    `json:"limit,omitempty" jsonschema:"Number of URLs to return, 1-100; defaults to 30"`
+	Offset     int    `json:"offset,omitempty" jsonschema:"Pagination offset, zero-based"`
+	PathPrefix string `json:"path_prefix,omitempty" jsonschema:"Optional URL path prefix, e.g. /api/"`
+	Language   string `json:"language,omitempty" jsonschema:"Locale code such as en or fr; defaults to English, use all for all languages"`
+}
+
+type siteMapCacheValue struct {
+	links   []any
+	expires time.Time
+}
+
+var siteMapCache = struct {
+	sync.Mutex
+	items map[string]siteMapCacheValue
+}{items: map[string]siteMapCacheValue{}}
+var siteMapFlight singleflight.Group
+
+func rawMappedLinks(ctx context.Context, rawURL string) ([]any, error) {
+	if err := validateSourceURL(rawURL); err != nil {
+		return nil, &DataError{Code: "invalid_url", Message: err.Error()}
+	}
+	credential := strings.TrimSpace(os.Getenv("FIRECRAWL_API_KEY"))
+	if credential == "" {
+		return nil, missingCredentialsError("Error mapping website", []string{"FIRECRAWL_API_KEY"}, "")
+	}
+	digest := sha256.Sum256([]byte(credential + "\x00" + rawURL))
+	cacheKey := hex.EncodeToString(digest[:])
+	useCache := !strings.HasPrefix(credential, "test")
+	if useCache {
+		siteMapCache.Lock()
+		cached, ok := siteMapCache.items[cacheKey]
+		siteMapCache.Unlock()
+		if ok && time.Now().Before(cached.expires) {
+			return cached.links, nil
+		}
+	}
+	fetch := func() (any, error) {
+		request := map[string]any{
+			"url": rawURL, "sitemap": "include",
+			"includeSubdomains": true, "ignoreQueryParameters": true, "limit": 5000,
+		}
+		body, err := doJSONRequest(ctx, http.MethodPost, "https://api.firecrawl.dev/v2/map", map[string]string{
+			"Authorization": "Bearer " + credential, "Content-Type": "application/json",
+		}, request)
+		if err != nil {
+			return nil, fmt.Errorf("Error mapping website: %w", err)
+		}
+		var result map[string]any
+		if err := json.Unmarshal(body, &result); err != nil {
+			return nil, fmt.Errorf("Error decoding map: %w", err)
+		}
+		if result["success"] != true {
+			return nil, dataError("map_failed", fmt.Sprintf("The source could not be mapped: %v", result["error"]))
+		}
+		links := listField(result["links"])
+		if useCache {
+			siteMapCache.Lock()
+			if len(siteMapCache.items) > 50 {
+				siteMapCache.items = map[string]siteMapCacheValue{}
+			}
+			siteMapCache.items[cacheKey] = siteMapCacheValue{links: links, expires: time.Now().Add(3 * time.Minute)}
+			siteMapCache.Unlock()
+		}
+		return links, nil
+	}
+	if !useCache {
+		data, err := fetch()
+		if err != nil {
+			return nil, err
+		}
+		return data.([]any), nil
+	}
+	data, err, _ := siteMapFlight.Do(cacheKey, fetch)
+	if err != nil {
+		return nil, err
+	}
+	return data.([]any), nil
+}
+
+var knownSiteLocales = map[string]bool{
+	"en": true, "fr": true, "es": true, "de": true, "ja": true, "zh": true, "zh-cn": true, "zh-tw": true,
+	"pt": true, "pt-br": true, "it": true, "ko": true, "ru": true, "ar": true, "hi": true, "id": true,
+	"nl": true, "pl": true, "tr": true, "vi": true, "th": true, "sv": true, "da": true,
+	"es-es": true, "en-us": true, "en-gb": true, "fr-fr": true, "fr-ca": true,
+}
+
+func mapPageLocale(raw string) string {
+	u, err := url.Parse(raw)
+	if err != nil {
+		return "en"
+	}
+	parts := strings.Split(strings.Trim(u.Path, "/"), "/")
+	if len(parts) > 0 {
+		first := strings.ToLower(parts[0])
+		if knownSiteLocales[first] {
+			if strings.HasPrefix(first, "en") {
+				return "en"
+			}
+			if strings.HasPrefix(first, "zh") {
+				return "zh"
+			}
+			if strings.HasPrefix(first, "es") {
+				return "es"
+			}
+			if strings.HasPrefix(first, "fr") {
+				return "fr"
+			}
+			if strings.HasPrefix(first, "pt") {
+				return "pt"
+			}
+			return first
+		}
+	}
+	return "en"
+}
+
+func MapSitePage(input MapSiteInput) (map[string]any, error) {
+	if input.Limit == 0 {
+		input.Limit = 30
+	}
+	if input.Limit < 1 || input.Limit > 100 {
+		return nil, dataError("invalid_limit", "map_site limit must be 1-100.")
+	}
+	if input.Offset < 0 || input.Offset > 100000 {
+		return nil, dataError("invalid_offset", "map_site offset must be nonnegative.")
+	}
+	if input.Language == "" {
+		input.Language = "en"
+	}
+	input.Language = strings.ToLower(strings.TrimSpace(input.Language))
+	if input.PathPrefix != "" && !strings.HasPrefix(input.PathPrefix, "/") {
+		input.PathPrefix = "/" + input.PathPrefix
+	}
+	if err := validateSourceURL(input.URL); err != nil {
+		return nil, &DataError{Code: "invalid_url", Message: err.Error()}
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	links, err := rawMappedLinks(ctx, input.URL)
+	if err != nil {
+		return nil, err
+	}
+	type located struct {
+		entry map[string]any
+		score int
+		path  string
+	}
+	var matched []located
+	seen := map[string]bool{}
+	terms := queryTerms(input.Query)
+	for _, v := range links {
+		item := map[string]any{}
+		if s, ok := v.(string); ok {
+			item["url"] = s
+		} else {
+			item = mapField(v)
+		}
+		raw := stringField(item["url"])
+		if raw == "" || seen[raw] {
+			continue
+		}
+		seen[raw] = true
+		u, e := url.Parse(raw)
+		if e != nil || u.Host == "" {
+			continue
+		}
+		if input.Language != "all" && mapPageLocale(raw) != input.Language {
+			continue
+		}
+		if input.PathPrefix != "" && !strings.HasPrefix(strings.ToLower(u.Path), strings.ToLower(input.PathPrefix)) {
+			continue
+		}
+		name := stringField(item["title"])
+		desc := stringField(item["description"])
+		score := termScore(u.Path+" "+name, terms)*3 + termScore(desc, terms)
+		if len(terms) > 0 && score == 0 {
+			continue
+		}
+		clean := map[string]any{"url": raw}
+		if name != "" {
+			clean["title"] = boundedSiteText(name, 140)
+		}
+		if desc != "" {
+			clean["description"] = boundedSiteText(desc, 230)
+		}
+		matched = append(matched, located{entry: clean, score: score, path: u.Path})
+	}
+	sort.SliceStable(matched, func(i, j int) bool {
+		if len(terms) > 0 && matched[i].score != matched[j].score {
+			return matched[i].score > matched[j].score
+		}
+		if depthI, depthJ := strings.Count(matched[i].path, "/"), strings.Count(matched[j].path, "/"); depthI != depthJ {
+			return depthI < depthJ
+		}
+		return matched[i].path < matched[j].path
+	})
+	start := input.Offset
+	if start > len(matched) {
+		start = len(matched)
+	}
+	end := start + input.Limit
+	if end > len(matched) {
+		end = len(matched)
+	}
+	page := make([]map[string]any, 0, end-start)
+	for _, entry := range matched[start:end] {
+		page = append(page, entry.entry)
+	}
+	output := map[string]any{
+		"url": input.URL, "total_discovered": len(links),
+		"total_matching": len(matched), "links": page,
+		"count": len(page), "offset": start, "limit": input.Limit,
+		"language": input.Language, "provider_page_ceiling": 5000,
+		"may_be_incomplete": len(links) >= 5000,
+		"guidance":          "Use next to request another bounded page, or set query/path_prefix/language to narrow results. Language=all includes translated pages.",
+	}
+	if end < len(matched) {
+		output["next"] = map[string]any{"tool": "map_site", "arguments": MapSiteInput{
+			URL: input.URL, Query: input.Query, Limit: input.Limit, Offset: end,
+			PathPrefix: input.PathPrefix, Language: input.Language,
+		}}
+	}
+	return output, nil
+}
+
+func boundedSiteText(value string, max int) string {
+	value = strings.Join(strings.Fields(value), " ")
+	if len(value) > max {
+		return value[:max] + "…"
+	}
+	return value
+}