webctx 0.2.4 → 0.2.6

package/pkg/origo/data_test.go

@@ -0,0 +1,134 @@
+package origo
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+type testDataRoundTripper func(*http.Request) (*http.Response, error)
+
+func (f testDataRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+func toolResponse(t *testing.T, name string, args any, id int) map[string]any {
+	t.Helper()
+	command, err := json.Marshal(map[string]any{
+		"jsonrpc": "2.0", "id": id, "method": "tools/call",
+		"params": map[string]any{"name": name, "arguments": args},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	response := request(t, "secret", string(command))
+	if response.Code != 200 {
+		t.Fatalf("MCP status %d: %s", response.Code, response.Body.String())
+	}
+	var decoded map[string]any
+	if err := json.Unmarshal(response.Body.Bytes(), &decoded); err != nil {
+		t.Fatalf("MCP JSON decode: %v", err)
+	}
+	if decoded["error"] != nil {
+		t.Fatalf("MCP protocol error: %s", response.Body.String())
+	}
+	result, ok := decoded["result"].(map[string]any)
+	if !ok {
+		t.Fatalf("missing result: %s", response.Body.String())
+	}
+	return result
+}
+
+func TestDataToolErrorsAreActionableStructuredMCPResults(t *testing.T) {
+	t.Setenv("ORIGO_API_KEY", "secret")
+	for _, tc := range []struct {
+		name  string
+		input map[string]any
+		code  string
+	}{
+		{"research", map[string]any{"view": "invalid"}, "invalid_view"},
+		{"inspect", map[string]any{"id": "not-a-valid-operation/"}, "invalid_operation"},
+		{"execute", map[string]any{"calls": []any{}}, "invalid_calls"},
+		{"execute", map[string]any{"calls": []any{map[string]any{"id": "example/data/list"}}, "max_credits": 1}, "invalid_arguments"},
+	} {
+		response := toolResponse(t, tc.name, tc.input, 11)
+		if response["isError"] == true {
+			t.Fatalf("%s should return normal structured response, not an MCP error: %#v", tc.name, response)
+		}
+		output, ok := response["structuredContent"].(map[string]any)
+		if !ok {
+			t.Fatalf("%s did not preserve structuredContent: %#v", tc.name, response)
+		}
+		errorResult, ok := output["error"].(map[string]any)
+		if !ok || errorResult["code"] != tc.code || output["ok"] != false {
+			t.Fatalf("%s error should be %s: %#v", tc.name, tc.code, response)
+		}
+	}
+}
+
+func TestDataToolResearchInspectExecuteOverHTTP(t *testing.T) {
+	t.Setenv("ORIGO_API_KEY", "secret")
+	t.Setenv("FIRECRAWL_API_KEY", "test")
+	original := http.DefaultClient
+	t.Cleanup(func() { http.DefaultClient = original })
+	var paidCount int
+	http.DefaultClient = &http.Client{Transport: testDataRoundTripper(func(req *http.Request) (*http.Response, error) {
+		body, err := io.ReadAll(req.Body)
+		if err != nil {
+			return nil, err
+		}
+		var args map[string]any
+		if json.Unmarshal(body, &args) != nil {
+			t.Fatalf("bad upstream request: %s", body)
+		}
+		var data map[string]any
+		switch {
+		case req.URL.Path == "/v2/search":
+			data = map[string]any{"success": true, "data": map[string]any{"tools": []any{
+				map[string]any{"id": "example/data/list", "provider": "example", "capability": "data/list", "name": "List example data", "creditsCost": 2, "description": "Demo read-only data"},
+			}}}
+		case req.URL.Path == "/v2/scrape" && args["alexandria"] != nil:
+			switch args["alexandria"].(type) {
+			case map[string]any:
+				data = map[string]any{"success": true, "data": map[string]any{"alexandria": []any{
+					map[string]any{"provider": "firecrawl", "capability": "find-tools", "data": map[string]any{
+						"items": []any{map[string]any{
+							"id": "example/data/list", "provider": "example", "capability": "data/list", "name": "List example data",
+							"creditsCost": 2, "perRecord": false, "options": []any{map[string]any{"name": "query", "type": "string", "required": true}},
+							"response": map[string]any{"key": "data"},
+						}},
+					}},
+				}}}
+			case []any:
+				paidCount++
+				if req.Header.Get("x-request-id") == "" {
+					t.Fatal("paid call missing idempotency header")
+				}
+				data = map[string]any{"success": true, "data": map[string]any{"creditsCost": 2, "alexandria": []any{
+					map[string]any{"provider": "example", "capability": "data/list", "creditsCost": 2, "alexandriaId": "receipt-1", "data": map[string]any{"data": []any{map[string]any{"answer": 42}}}},
+				}}}
+			}
+		}
+		if data == nil {
+			t.Fatalf("unhandled upstream operation %s", req.URL)
+		}
+		b, _ := json.Marshal(data)
+		return &http.Response{StatusCode: 200, Status: "200 OK", Body: io.NopCloser(strings.NewReader(string(b))), Header: make(http.Header), Request: req}, nil
+	})}
+	discovery := toolResponse(t, "research", map[string]any{"query": "example data"}, 12)
+	if discovery["isError"] == true || discovery["structuredContent"] == nil {
+		t.Fatalf("research failed: %#v", discovery)
+	}
+	contract := toolResponse(t, "inspect", map[string]any{"id": "example/data/list"}, 13)
+	structured := contract["structuredContent"].(map[string]any)
+	if structured["id"] != "example/data/list" || structured["next"] == nil {
+		t.Fatalf("inspect failed: %#v", structured)
+	}
+	result := toolResponse(t, "execute", map[string]any{"calls": []any{
+		map[string]any{"id": "example/data/list", "inputs": map[string]any{"query": "forty two"}},
+	}}, 14)
+	final := result["structuredContent"].(map[string]any)
+	if result["isError"] == true || final["credits_used"] != float64(2) || paidCount != 1 {
+		t.Fatalf("paid query failed: %#v calls=%d", final, paidCount)
+	}
+}

package/pkg/origo/server.go

@@ -3,14 +3,17 @@
 package origo
 
 import (
+	"bytes"
 	"context"
 	"crypto/sha256"
 	"crypto/subtle"
+	"encoding/json"
 	"net/http"
 	"os"
 	"strings"
 
 	"github.com/amxv/webctx/pkg/retrieval"
+	"github.com/google/jsonschema-go/jsonschema"
 	"github.com/modelcontextprotocol/go-sdk/mcp"
 )
 
@@ -49,9 +52,123 @@ func makeServer() *mcp.Server {
 		}
 		return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: result}}}, nil, nil
 	})
+	s.AddTool(&mcp.Tool{
+		Name:        "research",
+		Title:       "Find data sources",
+		Description: "Discover real-world information sources and their available operations. Search semantically, browse all sources, groups or operations, filter by websites/source IDs/categories, and paginate. Free; returns IDs and copy-ready inspect calls. Use read_link for regular documentation.",
+		Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true},
+		InputSchema: researchDataSchema(),
+	}, dataHandler(retrieval.Research))
+	s.AddTool(&mcp.Tool{
+		Name:        "inspect",
+		Title:       "Inspect a data source or operation",
+		Description: "Explore operations in a source, or inspect one operation's EXACT input types, required fields, constraints, output contract, pricing, and examples. Returns the next execute call with the exact input keys to fill. Free.",
+		Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true},
+		InputSchema: inspectDataSchema(),
+	}, dataHandler(retrieval.Inspect))
+	s.AddTool(&mcp.Tool{
+		Name:        "execute",
+		Title:       "Query a data source (uses credits)",
+		Description: "Execute 1-10 inspected structured-data operations using exact operation IDs and inputs. Alexandria credits are charged automatically; the server enforces its configured 200-credit per-request ceiling. Actual credits are reported. Supports pagination, safe request_id retries and provider-native JSON. Does not accept third-party terms.",
+		Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true},
+		InputSchema: executeDataSchema(),
+	}, dataHandler(retrieval.Execute))
 	return s
 }
 
+// Keep client-facing types narrow and self-describing. The SDK's default
+// reflection makes optional slices nullable and string enums unconstrained;
+// the catalogue operations support neither ambiguity.
+func researchDataSchema() *jsonschema.Schema {
+	schema, err := jsonschema.For[retrieval.ResearchInput](nil)
+	if err != nil {
+		panic(err)
+	}
+	schema.Properties["mode"].Enum = []any{"ranked", "catalogue"}
+	schema.Properties["view"].Enum = []any{"sources", "groups", "operations"}
+	for _, key := range []string{"mode", "view"} {
+		schema.Properties[key].Type = "string"
+		schema.Properties[key].Types = nil
+	}
+	for _, key := range []string{"urls", "sources", "categories", "groups", "operations", "include"} {
+		p := schema.Properties[key]
+		p.Type = "array"
+		p.Types = nil
+	}
+	schema.Properties["include"].Items.Enum = []any{"inputs", "output", "examples"}
+	return schema
+}
+
+func inspectDataSchema() *jsonschema.Schema {
+	schema, err := jsonschema.For[retrieval.InspectInput](nil)
+	if err != nil {
+		panic(err)
+	}
+	p := schema.Properties["include"]
+	p.Type = "array"
+	p.Types = nil
+	p.Items.Enum = []any{"inputs", "output", "examples"}
+	return schema
+}
+
+func executeDataSchema() *jsonschema.Schema {
+	schema, err := jsonschema.For[retrieval.ExecuteInput](nil)
+	if err != nil {
+		panic(err)
+	}
+	countMin, countMax := 1, 10
+	p := schema.Properties["calls"]
+	p.Type = "array"
+	p.Types = nil
+	p.MinItems = &countMin
+	p.MaxItems = &countMax
+	return schema
+}
+
+// Manual validation keeps data errors in successful MCP tool responses rather
+// than surfacing them as client-side INVALID_ARGUMENT tool exceptions.
+func dataHandler[In any](fn func(In) (map[string]any, error)) mcp.ToolHandler {
+	return func(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) {
+		var input In
+		args := req.Params.Arguments
+		if len(args) == 0 {
+			args = []byte("{}")
+		}
+		decoder := json.NewDecoder(bytes.NewReader(args))
+		decoder.DisallowUnknownFields()
+		if err := decoder.Decode(&input); err != nil {
+			response, _, e := dataMCPResult(nil, &retrieval.DataError{Code: "invalid_arguments", Message: err.Error()})
+			return response, e
+		}
+		result, err := fn(input)
+		response, _, e := dataMCPResult(result, err)
+		return response, e
+	}
+}
+
+func dataMCPResult(result map[string]any, err error) (*mcp.CallToolResult, any, error) {
+	failed := err != nil
+	if failed {
+		// Use the shared error serializer to preserve codes, corrective details
+		// and provider-term acceptance URLs as structured tool output.
+		result = retrieval.DataErrorResult(err)
+		result["ok"] = false
+	} else {
+		result["ok"] = true
+	}
+	bytes, marshalErr := json.MarshalIndent(result, "", "  ")
+	if marshalErr != nil {
+		return nil, nil, marshalErr
+	}
+	return &mcp.CallToolResult{
+		Content:           []mcp.Content{&mcp.TextContent{Text: string(bytes)}},
+		StructuredContent: result,
+		// Keep validation/provider failures in the structured result. Clients
+		// commonly convert isError=true to opaque INVALID_ARGUMENT exceptions.
+		IsError: false,
+	}, nil, nil
+}
+
 var transport = mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server {
 	return makeServer()
 }, &mcp.StreamableHTTPOptions{Stateless: true, JSONResponse: true})

package/pkg/origo/server_test.go

@@ -1,6 +1,7 @@
 package origo
 
 import (
+	"encoding/json"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -22,6 +23,57 @@ func request(t *testing.T, key, body string) *httptest.ResponseRecorder {
 	return response
 }
 
+func TestDataSchemasExposeExactEnumsAndNonNullableCalls(t *testing.T) {
+	t.Setenv("ORIGO_API_KEY", "secret")
+	result := request(t, "secret", `{"jsonrpc":"2.0","id":12,"method":"tools/list","params":{}}`)
+	if result.Code != 200 {
+		t.Fatalf("tools/list failed: %s", result.Body.String())
+	}
+	var decoded struct {
+		Result struct {
+			Tools []struct {
+				Name        string         `json:"name"`
+				InputSchema map[string]any `json:"inputSchema"`
+				Annotations map[string]any `json:"annotations"`
+			} `json:"tools"`
+		} `json:"result"`
+	}
+	if err := json.Unmarshal(result.Body.Bytes(), &decoded); err != nil {
+		t.Fatal(err)
+	}
+	schemas := map[string]map[string]any{}
+	for _, tool := range decoded.Result.Tools {
+		if tool.Annotations["readOnlyHint"] != true {
+			t.Fatalf("%s was not marked read-only", tool.Name)
+		}
+		if _, exists := tool.Annotations["openWorldHint"]; exists {
+			t.Fatalf("%s should omit openWorldHint", tool.Name)
+		}
+		schemas[tool.Name] = tool.InputSchema
+	}
+	if len(schemas) != 5 {
+		t.Fatalf("expected five tools, got %d", len(schemas))
+	}
+	props := func(name string) map[string]any { return schemas[name]["properties"].(map[string]any) }
+	mode := props("research")["mode"].(map[string]any)
+	view := props("research")["view"].(map[string]any)
+	if mode["type"] != "string" || len(mode["enum"].([]any)) != 2 ||
+		view["type"] != "string" || len(view["enum"].([]any)) != 3 {
+		t.Fatalf("research discovery schemas are not typed: mode=%#v view=%#v", mode, view)
+	}
+	incl := props("research")["include"].(map[string]any)
+	if incl["type"] != "array" || len(incl["items"].(map[string]any)["enum"].([]any)) != 3 {
+		t.Fatalf("include does not constrain allowed options: %#v", incl)
+	}
+	calls := props("execute")["calls"].(map[string]any)
+	if _, present := props("execute")["max_credits"]; present {
+		t.Fatal("execute still exposes credit-budget control")
+	}
+	if calls["type"] != "array" || calls["minItems"] != float64(1) || calls["maxItems"] != float64(10) {
+		t.Fatalf("execute calls must be non-null and 1–10 items: %#v", calls)
+	}
+}
+
 func TestServerRequiresConfiguredKey(t *testing.T) {
 	t.Setenv("ORIGO_API_KEY", "")
 	response := request(t, "somekey", `{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}`)
@@ -47,21 +99,26 @@ func TestServerAuthentication(t *testing.T) {
 	}
 }
 
-func TestExactlyTwoToolsAndNoSearch(t *testing.T) {
+func TestFiveToolsWithProgressiveDataInterface(t *testing.T) {
 	t.Setenv("ORIGO_API_KEY", "secret")
 	response := request(t, "secret", `{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}`)
 	if response.Code != http.StatusOK {
 		t.Fatalf("tools/list failed: %d %s", response.Code, response.Body.String())
 	}
 	text := response.Body.String()
-	if !strings.Contains(text, `"name":"read_link"`) || !strings.Contains(text, `"name":"map_site"`) || strings.Contains(text, `"name":"search"`) {
+	if !strings.Contains(text, `"name":"read_link"`) || !strings.Contains(text, `"name":"map_site"`) ||
+		!strings.Contains(text, `"name":"research"`) || !strings.Contains(text, `"name":"inspect"`) ||
+		!strings.Contains(text, `"name":"execute"`) || strings.Contains(text, `"name":"search"`) {
 		t.Fatalf("unexpected tools: %s", text)
 	}
-	if count := strings.Count(text, `"name":`); count != 2 {
-		t.Fatalf("expected exactly 2 tools, got %d: %s", count, text)
+	if count := strings.Count(text, `"name":`); count != 5 {
+		t.Fatalf("expected exactly 5 tools, got %d: %s", count, text)
+	}
+	if count := strings.Count(text, `"readOnlyHint":true`); count != 5 {
+		t.Fatalf("expected all five MCP tools to declare readOnlyHint=true, got %d: %s", count, text)
 	}
-	if count := strings.Count(text, `"readOnlyHint":true`); count != 2 {
-		t.Fatalf("expected both tools to declare read-only annotations, got %d: %s", count, text)
+	if strings.Contains(text, `"readOnlyHint":false`) || strings.Contains(text, `"openWorldHint"`) || strings.Contains(text, `"destructiveHint"`) {
+		t.Fatalf("unexpected non-read-only or open-world/destructive annotations: %s", text)
 	}
 }
 

package/pkg/retrieval/data.go

@@ -0,0 +1,14 @@
+package retrieval
+
+import "github.com/amxv/webctx/internal/app"
+
+type ResearchInput = app.ResearchDataInput
+type InspectInput = app.InspectDataInput
+type ExecuteInput = app.ExecuteDataInput
+type DataCall = app.DataCall
+type DataError = app.DataError
+
+func Research(input ResearchInput) (map[string]any, error) { return app.ResearchData(input) }
+func Inspect(input InspectInput) (map[string]any, error)   { return app.InspectData(input) }
+func Execute(input ExecuteInput) (map[string]any, error)   { return app.ExecuteData(input) }
+func DataErrorResult(err error) map[string]any             { return app.DataErrorResult(err) }