webctx 0.2.1

package/docs/porting-status.md

@@ -0,0 +1,173 @@
+# webctx TypeScript -> Go porting status
+
+This document is the handoff reference for future agents working on `amxv/webctx`.
+
+## Goal
+
+Port the CLI behavior from `amxv/webctx-ts` into pure Go while keeping the command-line interface and provider behavior effectively one-to-one for the CLI use case.
+
+Scope intentionally excludes the MCP/server/dashboard pieces from the TypeScript repo.
+
+## Source areas reviewed in `webctx-ts`
+
+- `cli.ts`
+- `tools/search.ts`
+- `tools/read-link.ts`
+- `tools/map-site.ts`
+- `lib/search/brave.ts`
+- `lib/search/tavily.ts`
+- `lib/search/exa.ts`
+- `lib/ranking.ts`
+- `lib/utils.ts`
+- `lib/scraping.ts`
+- `lib/firecrawl-queue.ts`
+- `lib/rate-limiter.ts`
+
+## Completed
+
+### CLI surface
+
+Implemented in Go:
+
+- `webctx search <query> [--exclude domain1,domain2] [--keyword phrase]`
+- `webctx read-link <url>`
+- `webctx map-site <url>`
+- `webctx --help`
+- `webctx --version`
+
+Notes:
+
+- `--version` now prints the bare version string, matching the TypeScript CLI.
+- Error handling is exit-code based in Go rather than promise rejection based.
+
+### Search port
+
+Implemented:
+
+- Brave HTTP client
+- Tavily HTTP client using direct HTTP API instead of the TypeScript SDK
+- Exa HTTP client
+- provider fan-out with per-provider timeout
+- duplicate-aware reranking
+- excluded-domain filtering
+- HTML entity decoding
+- keyword truncation to 5 words for Exa include-text mode
+- top 35 result output limit
+
+Current behavior matches the TypeScript CLI design:
+
+- normal search mode queries Brave + Tavily + Exa
+- keyword mode queries Exa only
+- user/domain exclusions are applied after provider collection, matching the TypeScript tool flow
+
+### Read-link port
+
+Implemented:
+
+- GitHub raw-content fast path
+- `.md` fast path with HEAD probe
+- Firecrawl scrape fallback with the same agent-oriented request settings
+- PDF parser enablement for `.pdf` URLs
+
+Kept settings aligned with the TypeScript CLI:
+
+- `formats: ["markdown"]`
+- `onlyMainContent: true`
+- `skipTlsVerification: true`
+- `blockAds: true`
+- `removeBase64Images: true`
+- `maxAge: 600000`
+- same excluded tags list
+
+### Map-site port
+
+Implemented with the same Firecrawl map settings:
+
+- `sitemap: "include"`
+- `includeSubdomains: true`
+- `ignoreQueryParameters: true`
+- `limit: 5000`
+
+### Firecrawl queue / rate limiting
+
+Implemented in Go:
+
+- singleton-style queue wrapper
+- token bucket limiter at 10 requests/minute
+- serialized queue processing for Firecrawl operations
+
+This is not a literal line-by-line port, but preserves the same operational intent.
+
+### Release/publish setup
+
+Updated to be release-ready for the real `webctx` CLI:
+
+- GitHub Actions workflow now builds `webctx` instead of the old template placeholder
+- release binaries embed the tagged version into `internal/buildinfo.Version`
+- npm metadata now describes the actual CLI instead of the template
+- README / agent / maintainer docs updated for the real repo
+
+## Intentionally not ported
+
+- MCP/server behavior
+- Next.js app/dashboard code
+- database/logging layers unrelated to the CLI
+
+These can be added later only if explicitly requested.
+
+## Current repo files of interest
+
+- `cmd/webctx/main.go`
+- `internal/app/app.go`
+- `internal/app/tools.go`
+- `internal/app/scrape.go`
+- `internal/app/app_test.go`
+- `.github/workflows/release.yml`
+- `scripts/postinstall.js`
+- `README.md`
+
+## Verification already completed
+
+- `go test ./...`
+- `go build ./cmd/webctx`
+
+## Live validation notes
+
+Live CLI validation was run against a real `.env.local` on the Sprite machine.
+
+Confirmed working live:
+
+- combined `search` path returns real web results
+- public GitHub blob `read-link` fast path works
+- Firecrawl-backed `read-link` works
+- Firecrawl-backed `map-site` works
+
+Observed external/provider constraints during live validation:
+
+- `search --keyword` currently depends on Exa-only results and could not be fully validated because the live Exa account returned `NO_MORE_CREDITS`
+- private GitHub blob URLs are not readable via unauthenticated raw-content fetch, so they fall through to the general scrape path
+
+These findings were from live provider behavior, not from compile/test failures in the Go port.
+
+## Good next checks for future agents
+
+1. Run live end-to-end checks against real provider keys for:
+   - normal multi-provider search
+   - Exa keyword-only search mode
+   - GitHub raw-content read-link
+   - `.md` fast path read-link
+   - Firecrawl scrape fallback
+   - Firecrawl map-site
+
+2. Compare a handful of live outputs from `webctx-ts` and Go `webctx` for formatting parity.
+
+3. If performance tuning is needed, focus on:
+   - HTTP client reuse
+   - provider timeout tuning
+   - Firecrawl queue behavior under concurrent use
+
+## Constraints to preserve
+
+- Keep the CLI output simple and agent-friendly.
+- Keep the Firecrawl request settings stable unless explicitly asked to change them.
+- Keep the release asset naming contract stable unless postinstall/workflow are updated together.

package/go.mod

@@ -0,0 +1,3 @@
+module github.com/amxv/webctx
+
+go 1.26

package/internal/app/app.go

@@ -0,0 +1,139 @@
+package app
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/amxv/webctx/internal/buildinfo"
+)
+
+const commandName = "webctx"
+
+var version = buildinfo.CurrentVersion()
+
+func Run(args []string, stdout, stderr io.Writer) int {
+	loadEnvLocal()
+
+	if len(args) == 0 || isHelpArg(args[0]) {
+		_, _ = fmt.Fprintln(stdout, usageText())
+		return 0
+	}
+
+	if args[0] == "--version" || args[0] == "-v" {
+		_, _ = fmt.Fprintln(stdout, version)
+		return 0
+	}
+
+	tool := args[0]
+	flags, positional := parseArgs(args[1:])
+	input := ""
+	if len(positional) > 0 {
+		input = positional[0]
+	}
+
+	switch tool {
+	case "search":
+		query := strings.Join(positional, " ")
+		if strings.TrimSpace(query) == "" {
+			_, _ = fmt.Fprintln(stderr, "Error: search requires a query")
+			_, _ = fmt.Fprintln(stdout, "Usage: webctx search <query> [--exclude domains] [--keyword phrase]")
+			return 1
+		}
+		excludeDomains := splitCSV(flags["exclude"])
+		text, err := Search(SearchParams{Query: query, ExcludeDomains: excludeDomains, IncludeKeyword: flags["keyword"]})
+		if err != nil {
+			_, _ = fmt.Fprintln(stderr, err.Error())
+			return 1
+		}
+		_, _ = fmt.Fprintln(stdout, text)
+		return 0
+	case "read-link":
+		if strings.TrimSpace(input) == "" {
+			_, _ = fmt.Fprintln(stderr, "Error: read-link requires a URL")
+			_, _ = fmt.Fprintln(stdout, "Usage: webctx read-link <url>")
+			return 1
+		}
+		text, err := ReadLink(input)
+		if err != nil {
+			_, _ = fmt.Fprintln(stderr, err.Error())
+			return 1
+		}
+		_, _ = fmt.Fprintln(stdout, text)
+		return 0
+	case "map-site":
+		if strings.TrimSpace(input) == "" {
+			_, _ = fmt.Fprintln(stderr, "Error: map-site requires a URL")
+			_, _ = fmt.Fprintln(stdout, "Usage: webctx map-site <url>")
+			return 1
+		}
+		text, err := MapSite(input)
+		if err != nil {
+			_, _ = fmt.Fprintln(stderr, err.Error())
+			return 1
+		}
+		_, _ = fmt.Fprintln(stdout, text)
+		return 0
+	default:
+		_, _ = fmt.Fprintln(stderr, "Unknown tool:", tool)
+		_, _ = fmt.Fprintln(stdout, usageText())
+		return 1
+	}
+}
+
+func usageText() string {
+	return fmt.Sprintf(`webctx v%s - Web search & browsing CLI
+
+Usage:
+  webctx search <query> [--exclude domain1,domain2] [--keyword phrase]
+  webctx read-link <url>
+  webctx map-site <url>
+
+Examples:
+  webctx search "next.js server components"
+  webctx search "react hooks" --exclude youtube.com,vimeo.com
+  webctx search "drizzle orm" --keyword "migration guide"
+  webctx read-link https://docs.example.com/guide
+  webctx map-site https://example.com`, version)
+}
+
+func parseArgs(args []string) (map[string]string, []string) {
+	flags := map[string]string{}
+	positional := make([]string, 0, len(args))
+	for i := 0; i < len(args); i++ {
+		if strings.HasPrefix(args[i], "--") && i+1 < len(args) {
+			flags[strings.TrimPrefix(args[i], "--")] = args[i+1]
+			i++
+			continue
+		}
+		positional = append(positional, args[i])
+	}
+	return flags, positional
+}
+
+func splitCSV(v string) []string {
+	if strings.TrimSpace(v) == "" {
+		return nil
+	}
+	parts := strings.Split(v, ",")
+	out := make([]string, 0, len(parts))
+	for _, part := range parts {
+		trimmed := strings.TrimSpace(part)
+		if trimmed != "" {
+			out = append(out, trimmed)
+		}
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	return out
+}
+
+func isHelpArg(v string) bool {
+	switch v {
+	case "-h", "--help", "help":
+		return true
+	default:
+		return false
+	}
+}

package/internal/app/app_test.go

@@ -0,0 +1,194 @@
+package app
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+func TestRunRootHelp(t *testing.T) {
+	var out bytes.Buffer
+	var errBuf bytes.Buffer
+
+	code := Run([]string{"--help"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("Run returned code %d", code)
+	}
+	if !strings.Contains(out.String(), "webctx v") || !strings.Contains(out.String(), "read-link") {
+		t.Fatalf("unexpected help output: %q", out.String())
+	}
+}
+
+func TestRunVersion(t *testing.T) {
+	var out bytes.Buffer
+	var errBuf bytes.Buffer
+
+	code := Run([]string{"--version"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("Run returned code %d", code)
+	}
+	if strings.TrimSpace(out.String()) == "" {
+		t.Fatalf("unexpected empty version output")
+	}
+	if strings.Contains(out.String(), "webctx ") {
+		t.Fatalf("expected bare version output, got: %q", out.String())
+	}
+}
+
+func TestRunSearchWithoutQuery(t *testing.T) {
+	var out bytes.Buffer
+	var errBuf bytes.Buffer
+
+	code := Run([]string{"search"}, &out, &errBuf)
+	if code != 1 {
+		t.Fatalf("expected exit code 1, got %d", code)
+	}
+	if !strings.Contains(errBuf.String(), "search requires a query") {
+		t.Fatalf("unexpected stderr: %q", errBuf.String())
+	}
+}
+
+func TestNormalizeURL(t *testing.T) {
+	got := normalizeURL("https://Example.com/docs/?utm_source=x&ref=y&q=go")
+	want := "https://example.com/docs?q=go"
+	if got != want {
+		t.Fatalf("normalizeURL mismatch: got %q want %q", got, want)
+	}
+}
+
+func TestScoreAndRankResultsDuplicateBoost(t *testing.T) {
+	results, duplicatesRemoved := scoreAndRankResults([]providerDocs{
+		{Provider: "Brave", Docs: []SearchDoc{{URL: "https://a.com", Title: "A"}, {URL: "https://b.com", Title: "B"}}},
+		{Provider: "Tavily", Docs: []SearchDoc{{URL: "https://b.com", Title: "B2"}, {URL: "https://c.com", Title: "C"}}},
+	})
+	if duplicatesRemoved != 1 {
+		t.Fatalf("expected 1 duplicate removed, got %d", duplicatesRemoved)
+	}
+	if len(results) == 0 || results[0].URL != "https://b.com" {
+		t.Fatalf("expected duplicate URL to rank first, got %#v", results)
+	}
+}
+
+func TestScoreAndRankResultsPreservesInsertionOrderForTies(t *testing.T) {
+	results, _ := scoreAndRankResults([]providerDocs{
+		{Provider: "Brave", Docs: []SearchDoc{{URL: "https://z.example", Title: "Z"}}},
+		{Provider: "Tavily", Docs: []SearchDoc{{URL: "https://a.example", Title: "A"}}},
+	})
+
+	if len(results) != 2 || results[0].URL != "https://z.example" || results[1].URL != "https://a.example" {
+		t.Fatalf("expected insertion order for tied results, got %#v", results)
+	}
+}
+
+func TestSearchMissingCredentialsErrorIsHelpful(t *testing.T) {
+	t.Setenv("BRAVE_API_KEY", "")
+	t.Setenv("TAVILY_API_KEY", "")
+	t.Setenv("EXA_API_KEY", "")
+
+	text, err := Search(SearchParams{Query: "openai api"})
+	if err == nil {
+		t.Fatalf("expected error, got text %q", text)
+	}
+	if !strings.Contains(err.Error(), "missing BRAVE_API_KEY, EXA_API_KEY, TAVILY_API_KEY") {
+		t.Fatalf("unexpected error: %q", err.Error())
+	}
+	if !strings.Contains(err.Error(), "macOS Keychain") {
+		t.Fatalf("expected keychain guidance, got %q", err.Error())
+	}
+}
+
+func TestMapSiteMissingCredentialErrorIsHelpful(t *testing.T) {
+	t.Setenv("FIRECRAWL_API_KEY", "")
+
+	text, err := MapSite("https://example.com")
+	if err == nil {
+		t.Fatalf("expected error, got text %q", text)
+	}
+	if !strings.Contains(err.Error(), "missing FIRECRAWL_API_KEY") {
+		t.Fatalf("unexpected error: %q", err.Error())
+	}
+	if !strings.Contains(err.Error(), ".env.local next to the binary") {
+		t.Fatalf("expected .env.local guidance, got %q", err.Error())
+	}
+}
+
+func TestLoadDotEnvFileDoesNotOverrideExistingEnv(t *testing.T) {
+	tmpDir := t.TempDir()
+	envPath := filepath.Join(tmpDir, ".env.local")
+	if err := os.WriteFile(envPath, []byte("BRAVE_API_KEY=from-file\nEXA_API_KEY=from-file\n"), 0o644); err != nil {
+		t.Fatalf("write env file: %v", err)
+	}
+
+	t.Setenv("BRAVE_API_KEY", "from-env")
+	t.Setenv("EXA_API_KEY", "")
+
+	loadDotEnvFile(envPath)
+
+	if got := os.Getenv("BRAVE_API_KEY"); got != "from-env" {
+		t.Fatalf("expected existing env to win, got %q", got)
+	}
+	if got := os.Getenv("EXA_API_KEY"); got != "" {
+		t.Fatalf("expected explicitly empty env to remain untouched, got %q", got)
+	}
+}
+
+func TestEnvLocalCandidatesPreferExecutableDir(t *testing.T) {
+	originalGetwd := getwdFunc
+	originalExecutable := executableFunc
+	t.Cleanup(func() {
+		getwdFunc = originalGetwd
+		executableFunc = originalExecutable
+	})
+
+	getwdFunc = func() (string, error) { return "/tmp/project", nil }
+	executableFunc = func() (string, error) { return "/opt/webctx/bin/webctx", nil }
+
+	got := envLocalCandidates()
+	want := []string{
+		filepath.Join("/opt/webctx/bin", ".env.local"),
+		filepath.Join("/opt/webctx", ".env.local"),
+		filepath.Join("/tmp/project", ".env.local"),
+	}
+	if runtime.GOOS == "windows" {
+		for i := range want {
+			want[i] = filepath.Clean(want[i])
+		}
+	}
+
+	if len(got) != len(want) {
+		t.Fatalf("unexpected candidate count: got %v want %v", got, want)
+	}
+	for i := range want {
+		if filepath.Clean(got[i]) != filepath.Clean(want[i]) {
+			t.Fatalf("candidate %d mismatch: got %q want %q", i, got[i], want[i])
+		}
+	}
+}
+
+func TestLoadKeychainEnvLoadsMissingOnly(t *testing.T) {
+	originalLookup := keychainLookup
+	t.Cleanup(func() { keychainLookup = originalLookup })
+
+	keychainLookup = func(key string) (string, error) {
+		return "from-keychain-" + key, nil
+	}
+
+	t.Setenv("BRAVE_API_KEY", "already-set")
+	t.Setenv("TAVILY_API_KEY", "")
+	t.Setenv("EXA_API_KEY", "")
+	t.Setenv("FIRECRAWL_API_KEY", "")
+	t.Setenv("GH_TOKEN", "")
+	t.Setenv("GITHUB_TOKEN", "")
+
+	loadKeychainEnv()
+
+	if got := os.Getenv("BRAVE_API_KEY"); got != "already-set" {
+		t.Fatalf("expected existing env to remain, got %q", got)
+	}
+	if got := os.Getenv("TAVILY_API_KEY"); got != "" {
+		t.Fatalf("expected explicitly empty env to remain untouched, got %q", got)
+	}
+}