webctx 0.2.4 → 0.2.5

package/internal/app/data_types.go

@@ -0,0 +1,40 @@
+package app
+
+// ResearchData discovers data sources and their available operations. Search
+// and catalogue browsing are both free Firecrawl Alexandria operations.
+type ResearchDataInput struct {
+	Query      string   `json:"query,omitempty" jsonschema:"Optional natural-language description of the information needed"`
+	Mode       string   `json:"mode,omitempty" jsonschema:"ranked semantic discovery or catalogue browsing with filters and pagination; defaults to ranked for a query"`
+	URLs       []string `json:"urls,omitempty" jsonschema:"Filter by websites associated with a data source"`
+	Sources    []string `json:"sources,omitempty" jsonschema:"Filter by source IDs returned by research"`
+	Categories []string `json:"categories,omitempty" jsonschema:"Filter by catalogue category IDs"`
+	Groups     []string `json:"groups,omitempty" jsonschema:"Filter by catalogue group IDs"`
+	Operations []string `json:"operations,omitempty" jsonschema:"Filter by exact operation IDs"`
+	View       string   `json:"view,omitempty" jsonschema:"What to browse: sources, groups, or operations; default operations for a query, otherwise sources"`
+	Include    []string `json:"include,omitempty" jsonschema:"Additional contract details: inputs, output, examples. Default is a concise overview"`
+	Limit      int      `json:"limit,omitempty" jsonschema:"Maximum results in this page, 1-100; default 10"`
+	Offset     int      `json:"offset,omitempty" jsonschema:"Catalogue pagination offset, zero-based"`
+	Raw        bool     `json:"raw,omitempty" jsonschema:"Also include the complete original Firecrawl response"`
+}
+
+// InspectDataInput returns the exact contract for a selected operation, or
+// browses all operations of a source when passed just a source ID.
+type InspectDataInput struct {
+	ID      string   `json:"id" jsonschema:"Source or operation ID returned by research, e.g. particle/podcasts/episodes/search"`
+	Include []string `json:"include,omitempty" jsonschema:"Contract sections: inputs, output, examples; all by default"`
+	Raw     bool     `json:"raw,omitempty" jsonschema:"Include the complete original Firecrawl response"`
+}
+
+type DataCall struct {
+	ID     string         `json:"id" jsonschema:"Exact operation ID returned by research or inspect"`
+	Inputs map[string]any `json:"inputs,omitempty" jsonschema:"Provider-specific input fields copied from the inspected contract"`
+}
+
+// ExecuteDataInput supports up to ten calls in a single provider request.
+// It validates each call against the free catalogue before using credits.
+type ExecuteDataInput struct {
+	Calls      []DataCall `json:"calls" jsonschema:"One to ten operation calls; inspect each operation to discover exact input field names"`
+	MaxCredits int        `json:"max_credits,omitempty" jsonschema:"Maximum predicted credits for this request; defaults to WEBCTX_ALEXANDRIA_MAX_CREDITS or 100"`
+	RequestID  string     `json:"request_id,omitempty" jsonschema:"Optional stable idempotency identifier for retrying a paid request; reuse unchanged if retrying"`
+	Raw        bool       `json:"raw,omitempty" jsonschema:"Include the complete original provider response"`
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "webctx",
-  "version": "0.2.4",
-  "description": "Pure Go web search and browsing CLI using Brave, Tavily, Exa, and Firecrawl",
+  "version": "0.2.5",
+  "description": "Go CLI for agent-friendly web research, structured-data discovery, and source-native querying",
   "license": "MIT",
   "author": "amxv",
   "repository": {

package/pkg/origo/data_test.go

@@ -0,0 +1,134 @@
+package origo
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+type testDataRoundTripper func(*http.Request) (*http.Response, error)
+
+func (f testDataRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+func toolResponse(t *testing.T, name string, args any, id int) map[string]any {
+	t.Helper()
+	command, err := json.Marshal(map[string]any{
+		"jsonrpc": "2.0", "id": id, "method": "tools/call",
+		"params": map[string]any{"name": name, "arguments": args},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	response := request(t, "secret", string(command))
+	if response.Code != 200 {
+		t.Fatalf("MCP status %d: %s", response.Code, response.Body.String())
+	}
+	var decoded map[string]any
+	if err := json.Unmarshal(response.Body.Bytes(), &decoded); err != nil {
+		t.Fatalf("MCP JSON decode: %v", err)
+	}
+	if decoded["error"] != nil {
+		t.Fatalf("MCP protocol error: %s", response.Body.String())
+	}
+	result, ok := decoded["result"].(map[string]any)
+	if !ok {
+		t.Fatalf("missing result: %s", response.Body.String())
+	}
+	return result
+}
+
+func TestDataToolErrorsAreActionableStructuredMCPResults(t *testing.T) {
+	t.Setenv("ORIGO_API_KEY", "secret")
+	for _, tc := range []struct {
+		name  string
+		input map[string]any
+		code  string
+	}{
+		{"research", map[string]any{"view": "invalid"}, "invalid_view"},
+		{"inspect", map[string]any{"id": "not-a-valid-operation/"}, "invalid_operation"},
+		{"execute", map[string]any{"calls": []any{}}, "invalid_calls"},
+	} {
+		response := toolResponse(t, tc.name, tc.input, 11)
+		if response["isError"] != true {
+			t.Fatalf("%s should return isError: %#v", tc.name, response)
+		}
+		output, ok := response["structuredContent"].(map[string]any)
+		if !ok {
+			t.Fatalf("%s did not preserve structuredContent: %#v", tc.name, response)
+		}
+		errorResult, ok := output["error"].(map[string]any)
+		if !ok || errorResult["code"] != tc.code {
+			t.Fatalf("%s error should be %s: %#v", tc.name, tc.code, response)
+		}
+	}
+}
+
+func TestDataToolResearchInspectExecuteOverHTTP(t *testing.T) {
+	t.Setenv("ORIGO_API_KEY", "secret")
+	t.Setenv("FIRECRAWL_API_KEY", "test")
+	t.Setenv("WEBCTX_ALEXANDRIA_PAID", "true")
+	original := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = original })
+	var paidCount int
+	http.DefaultClient = &http.Client{Transport: testDataRoundTripper(func(req *http.Request) (*http.Response, error) {
+		body, err := io.ReadAll(req.Body)
+		if err != nil {
+			return nil, err
+		}
+		var args map[string]any
+		if json.Unmarshal(body, &args) != nil {
+			t.Fatalf("bad upstream request: %s", body)
+		}
+		var data map[string]any
+		switch {
+		case req.URL.Path == "/v2/search":
+			data = map[string]any{"success": true, "data": map[string]any{"tools": []any{
+				map[string]any{"id": "example/data/list", "provider": "example", "capability": "data/list", "name": "List example data", "creditsCost": 2, "description": "Demo read-only data"},
+			}}}
+		case req.URL.Path == "/v2/scrape" && args["alexandria"] != nil:
+			switch args["alexandria"].(type) {
+			case map[string]any:
+				data = map[string]any{"success": true, "data": map[string]any{"alexandria": []any{
+					map[string]any{"provider": "firecrawl", "capability": "find-tools", "data": map[string]any{
+						"items": []any{map[string]any{
+							"id": "example/data/list", "provider": "example", "capability": "data/list", "name": "List example data",
+							"creditsCost": 2, "perRecord": false, "options": []any{map[string]any{"name": "query", "type": "string", "required": true}},
+							"response": map[string]any{"key": "data"},
+						}},
+					}},
+				}}}
+			case []any:
+				paidCount++
+				if req.Header.Get("x-request-id") == "" {
+					t.Fatal("paid call missing idempotency header")
+				}
+				data = map[string]any{"success": true, "data": map[string]any{"creditsCost": 2, "alexandria": []any{
+					map[string]any{"provider": "example", "capability": "data/list", "creditsCost": 2, "alexandriaId": "receipt-1", "data": map[string]any{"data": []any{map[string]any{"answer": 42}}}},
+				}}}
+			}
+		}
+		if data == nil {
+			t.Fatalf("unhandled upstream operation %s", req.URL)
+		}
+		b, _ := json.Marshal(data)
+		return &http.Response{StatusCode: 200, Status: "200 OK", Body: io.NopCloser(strings.NewReader(string(b))), Header: make(http.Header), Request: req}, nil
+	})}
+	discovery := toolResponse(t, "research", map[string]any{"query": "example data"}, 12)
+	if discovery["isError"] == true || discovery["structuredContent"] == nil {
+		t.Fatalf("research failed: %#v", discovery)
+	}
+	contract := toolResponse(t, "inspect", map[string]any{"id": "example/data/list"}, 13)
+	structured := contract["structuredContent"].(map[string]any)
+	if structured["id"] != "example/data/list" || structured["next"] == nil {
+		t.Fatalf("inspect failed: %#v", structured)
+	}
+	result := toolResponse(t, "execute", map[string]any{"calls": []any{
+		map[string]any{"id": "example/data/list", "inputs": map[string]any{"query": "forty two"}},
+	}, "max_credits": 3}, 14)
+	final := result["structuredContent"].(map[string]any)
+	if result["isError"] == true || final["credits_used"] != float64(2) || paidCount != 1 {
+		t.Fatalf("paid query failed: %#v calls=%d", final, paidCount)
+	}
+}

package/pkg/origo/server.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"crypto/subtle"
+	"encoding/json"
 	"net/http"
 	"os"
 	"strings"
@@ -49,9 +50,54 @@ func makeServer() *mcp.Server {
 		}
 		return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: result}}}, nil, nil
 	})
+	mcp.AddTool(s, &mcp.Tool{
+		Name:        "research",
+		Title:       "Find data sources",
+		Description: "Discover real-world information sources and their available operations. Search semantically, browse all sources, groups or operations, filter by websites/source IDs/categories, and paginate. Free; returns IDs and copy-ready inspect calls. Use read_link for regular documentation.",
+		Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true},
+	}, func(_ context.Context, _ *mcp.CallToolRequest, input retrieval.ResearchInput) (*mcp.CallToolResult, any, error) {
+		result, err := retrieval.Research(input)
+		return dataMCPResult(result, err)
+	})
+	mcp.AddTool(s, &mcp.Tool{
+		Name:        "inspect",
+		Title:       "Inspect a data source or operation",
+		Description: "Explore operations in a source, or inspect one operation's EXACT input types, required fields, constraints, output contract, pricing, and examples. Returns the next execute call with the exact input keys to fill. Free.",
+		Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true},
+	}, func(_ context.Context, _ *mcp.CallToolRequest, input retrieval.InspectInput) (*mcp.CallToolResult, any, error) {
+		result, err := retrieval.Inspect(input)
+		return dataMCPResult(result, err)
+	})
+	mcp.AddTool(s, &mcp.Tool{
+		Name:        "execute",
+		Title:       "Query a data source (uses credits)",
+		Description: "Execute 1-10 inspected structured-data operations using their exact source IDs and inputs. Charges Firecrawl Alexandria credits after server-side schema and credit-budget checks. Supports pagination/continuations, stable request_id for safe retries, and original provider JSON. No third-party terms are accepted automatically.",
+		Annotations: &mcp.ToolAnnotations{ReadOnlyHint: false, DestructiveHint: func() *bool { v := false; return &v }()},
+	}, func(_ context.Context, _ *mcp.CallToolRequest, input retrieval.ExecuteInput) (*mcp.CallToolResult, any, error) {
+		result, err := retrieval.Execute(input)
+		return dataMCPResult(result, err)
+	})
 	return s
 }
 
+func dataMCPResult(result map[string]any, err error) (*mcp.CallToolResult, any, error) {
+	failed := err != nil
+	if failed {
+		// Use the shared error serializer to preserve codes, corrective details
+		// and provider-term acceptance URLs as structured tool output.
+		result = retrieval.DataErrorResult(err)
+	}
+	bytes, marshalErr := json.MarshalIndent(result, "", "  ")
+	if marshalErr != nil {
+		return nil, nil, marshalErr
+	}
+	return &mcp.CallToolResult{
+		Content:           []mcp.Content{&mcp.TextContent{Text: string(bytes)}},
+		StructuredContent: result,
+		IsError:           failed,
+	}, nil, nil
+}
+
 var transport = mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server {
 	return makeServer()
 }, &mcp.StreamableHTTPOptions{Stateless: true, JSONResponse: true})

package/pkg/origo/server_test.go

@@ -47,21 +47,26 @@ func TestServerAuthentication(t *testing.T) {
 	}
 }
 
-func TestExactlyTwoToolsAndNoSearch(t *testing.T) {
+func TestFiveToolsWithProgressiveDataInterface(t *testing.T) {
 	t.Setenv("ORIGO_API_KEY", "secret")
 	response := request(t, "secret", `{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}`)
 	if response.Code != http.StatusOK {
 		t.Fatalf("tools/list failed: %d %s", response.Code, response.Body.String())
 	}
 	text := response.Body.String()
-	if !strings.Contains(text, `"name":"read_link"`) || !strings.Contains(text, `"name":"map_site"`) || strings.Contains(text, `"name":"search"`) {
+	if !strings.Contains(text, `"name":"read_link"`) || !strings.Contains(text, `"name":"map_site"`) ||
+		!strings.Contains(text, `"name":"research"`) || !strings.Contains(text, `"name":"inspect"`) ||
+		!strings.Contains(text, `"name":"execute"`) || strings.Contains(text, `"name":"search"`) {
 		t.Fatalf("unexpected tools: %s", text)
 	}
-	if count := strings.Count(text, `"name":`); count != 2 {
-		t.Fatalf("expected exactly 2 tools, got %d: %s", count, text)
+	if count := strings.Count(text, `"name":`); count != 5 {
+		t.Fatalf("expected exactly 5 tools, got %d: %s", count, text)
 	}
-	if count := strings.Count(text, `"readOnlyHint":true`); count != 2 {
-		t.Fatalf("expected both tools to declare read-only annotations, got %d: %s", count, text)
+	if count := strings.Count(text, `"readOnlyHint":true`); count != 4 {
+		t.Fatalf("expected the four free retrieval tools to be read-only, got %d: %s", count, text)
+	}
+	if count := strings.Count(text, `"readOnlyHint":false`); count != 1 {
+		t.Fatalf("expected only paid execute to indicate effects on billing, got %d: %s", count, text)
 	}
 }
 

package/pkg/retrieval/data.go

@@ -0,0 +1,13 @@
+package retrieval
+
+import "github.com/amxv/webctx/internal/app"
+
+type ResearchInput = app.ResearchDataInput
+type InspectInput = app.InspectDataInput
+type ExecuteInput = app.ExecuteDataInput
+type DataCall = app.DataCall
+
+func Research(input ResearchInput) (map[string]any, error) { return app.ResearchData(input) }
+func Inspect(input InspectInput) (map[string]any, error)   { return app.InspectData(input) }
+func Execute(input ExecuteInput) (map[string]any, error)   { return app.ExecuteData(input) }
+func DataErrorResult(err error) map[string]any             { return app.DataErrorResult(err) }