webctx 0.2.6 → 0.2.7

package/internal/app/tools.go

@@ -252,57 +252,27 @@ func scrapeLinkWithFirecrawl(rawURL string) (string, string, error) {
 }
 
 func MapSite(rawURL string) (string, error) {
-	if err := validateSourceURL(rawURL); err != nil {
+	data, err := MapSitePage(MapSiteInput{URL: rawURL})
+	if err != nil {
 		return "", err
 	}
-	apiKey := strings.TrimSpace(os.Getenv("FIRECRAWL_API_KEY"))
-	if apiKey == "" {
-		return "", missingCredentialsError("Error mapping website", []string{"FIRECRAWL_API_KEY"}, "")
+	parts := []string{
+		fmt.Sprintf("total urls found: %v", data["total_discovered"]),
+		fmt.Sprintf("matching pages: %v; displayed: %v (offset %v)", data["total_matching"], data["count"], data["offset"]),
+		"",
 	}
-
-	requestBody := map[string]any{
-		"url":                   rawURL,
-		"sitemap":               "include",
-		"includeSubdomains":     true,
-		"ignoreQueryParameters": true,
-		"limit":                 5000,
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
-	defer cancel()
-	body, err := doJSONRequest(ctx, http.MethodPost, "https://api.firecrawl.dev/v2/map", map[string]string{
-		"Authorization": "Bearer " + apiKey,
-		"Content-Type":  "application/json",
-	}, requestBody)
-	if err != nil {
-		return "", fmt.Errorf("Error mapping website: %v", err)
-	}
-
-	var parsed map[string]any
-	if err := json.Unmarshal(body, &parsed); err != nil {
-		return "", fmt.Errorf("Error mapping website: %v", err)
-	}
-	if success, _ := parsed["success"].(bool); !success {
-		return "", fmt.Errorf("Error mapping website: Mapping failed for %s: %v", rawURL, parsed["error"])
-	}
-
-	linksAny, _ := parsed["links"].([]any)
-	parts := []string{fmt.Sprintf("total urls found: %d", len(linksAny)), ""}
-	for _, item := range linksAny {
-		switch link := item.(type) {
-		case string:
-			parts = append(parts, "- "+link, "")
-		case map[string]any:
-			parts = append(parts, "- "+stringValue(link["url"]))
-			if v := stringValue(link["title"]); v != "" {
-				parts = append(parts, "- "+v)
-			}
-			if v := stringValue(link["description"]); v != "" {
-				parts = append(parts, "- "+v)
-			}
-			parts = append(parts, "")
+	for _, entry := range data["links"].([]map[string]any) {
+		parts = append(parts, "- "+stringField(entry["url"]))
+		if title := stringField(entry["title"]); title != "" {
+			parts = append(parts, "  - "+title)
+		}
+		if desc := stringField(entry["description"]); desc != "" {
+			parts = append(parts, "  - "+desc)
 		}
 	}
+	if data["next"] != nil {
+		parts = append(parts, "", "More pages available; use --offset with webctx map-site for the next page.")
+	}
 	return strings.Join(parts, "\n"), nil
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webctx",
-  "version": "0.2.6",
+  "version": "0.2.7",
   "description": "Go CLI for agent-friendly web research, structured-data discovery, and source-native querying",
   "license": "MIT",
   "author": "amxv",

package/pkg/origo/data_test.go

@@ -66,6 +66,32 @@ func TestDataToolErrorsAreActionableStructuredMCPResults(t *testing.T) {
 	}
 }
 
+func TestAuditMCPRejectsZeroLimitAndInvalidSourceURLsWithoutExceptions(t *testing.T) {
+	t.Setenv("ORIGO_API_KEY", "secret")
+	for _, tt := range []struct {
+		name string
+		args map[string]any
+		code string
+	}{
+		{"research", map[string]any{"mode": "catalogue", "view": "sources", "limit": 0}, "invalid_limit"},
+		{"read_link", map[string]any{"url": "file:///etc/passwd", "question": "summarize"}, "invalid_url"},
+		{"map_site", map[string]any{"url": "file:///etc/passwd"}, "invalid_url"},
+	} {
+		reply := toolResponse(t, tt.name, tt.args, 990)
+		if reply["isError"] == true {
+			t.Fatalf("%s surfaced a host error: %+v", tt.name, reply)
+		}
+		data, ok := reply["structuredContent"].(map[string]any)
+		if !ok || data["ok"] != false {
+			t.Fatalf("%s did not return a structured error: %+v", tt.name, reply)
+		}
+		failure := data["error"].(map[string]any)
+		if failure["code"] != tt.code {
+			t.Fatalf("%s expected %s, got %+v", tt.name, tt.code, failure)
+		}
+	}
+}
+
 func TestDataToolResearchInspectExecuteOverHTTP(t *testing.T) {
 	t.Setenv("ORIGO_API_KEY", "secret")
 	t.Setenv("FIRECRAWL_API_KEY", "test")

package/pkg/origo/server.go

@@ -36,22 +36,21 @@ func makeServer() *mcp.Server {
 	}, func(_ context.Context, _ *mcp.CallToolRequest, input readLinkInput) (*mcp.CallToolResult, any, error) {
 		result, err := retrieval.ReadLinkFocused(input.URL, input.Question)
 		if err != nil {
-			return nil, nil, err
+			code := "read_failed"
+			if strings.Contains(err.Error(), "absolute HTTP(S)") || strings.Contains(err.Error(), "readable URL") {
+				code = "invalid_url"
+			}
+			return dataMCPResult(nil, &retrieval.DataError{Code: code, Message: err.Error()})
 		}
 		return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: result}}}, nil, nil
 	})
-	mcp.AddTool(s, &mcp.Tool{
+	s.AddTool(&mcp.Tool{
 		Name:        "map_site",
 		Title:       "Map site",
-		Description: "Discover URLs on a site using Firecrawl's sitemap-based map API. Does not perform web search.",
+		Description: "Discover website pages in bounded, paginated groups (30 by default). Filter by topic, path prefix and language (en by default, all for translations). Returns structured titles, descriptions, counts, and copy-ready next-page arguments.",
 		Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true},
-	}, func(_ context.Context, _ *mcp.CallToolRequest, input linkInput) (*mcp.CallToolResult, any, error) {
-		result, err := retrieval.MapSite(input.URL)
-		if err != nil {
-			return nil, nil, err
-		}
-		return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: result}}}, nil, nil
-	})
+		InputSchema: mapSiteSchema(),
+	}, dataHandler(retrieval.MapSitePage))
 	s.AddTool(&mcp.Tool{
 		Name:        "research",
 		Title:       "Find data sources",
@@ -125,6 +124,14 @@ func executeDataSchema() *jsonschema.Schema {
 	return schema
 }
 
+func mapSiteSchema() *jsonschema.Schema {
+	schema, err := jsonschema.For[retrieval.MapInput](nil)
+	if err != nil {
+		panic(err)
+	}
+	return schema
+}
+
 // Manual validation keeps data errors in successful MCP tool responses rather
 // than surfacing them as client-side INVALID_ARGUMENT tool exceptions.
 func dataHandler[In any](fn func(In) (map[string]any, error)) mcp.ToolHandler {

package/pkg/retrieval/data.go

@@ -7,8 +7,10 @@ type InspectInput = app.InspectDataInput
 type ExecuteInput = app.ExecuteDataInput
 type DataCall = app.DataCall
 type DataError = app.DataError
+type MapInput = app.MapSiteInput
 
 func Research(input ResearchInput) (map[string]any, error) { return app.ResearchData(input) }
 func Inspect(input InspectInput) (map[string]any, error)   { return app.InspectData(input) }
 func Execute(input ExecuteInput) (map[string]any, error)   { return app.ExecuteData(input) }
 func DataErrorResult(err error) map[string]any             { return app.DataErrorResult(err) }
+func MapSitePage(input MapInput) (map[string]any, error)   { return app.MapSitePage(input) }