webctx 0.1.0

package/internal/app/app.go

@@ -0,0 +1,139 @@
+package app
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/amxv/webctx/internal/buildinfo"
+)
+
+const commandName = "webctx"
+
+var version = buildinfo.CurrentVersion()
+
+func Run(args []string, stdout, stderr io.Writer) int {
+	loadEnvLocal()
+
+	if len(args) == 0 || isHelpArg(args[0]) {
+		_, _ = fmt.Fprintln(stdout, usageText())
+		return 0
+	}
+
+	if args[0] == "--version" || args[0] == "-v" {
+		_, _ = fmt.Fprintln(stdout, version)
+		return 0
+	}
+
+	tool := args[0]
+	flags, positional := parseArgs(args[1:])
+	input := ""
+	if len(positional) > 0 {
+		input = positional[0]
+	}
+
+	switch tool {
+	case "search":
+		query := strings.Join(positional, " ")
+		if strings.TrimSpace(query) == "" {
+			_, _ = fmt.Fprintln(stderr, "Error: search requires a query")
+			_, _ = fmt.Fprintln(stdout, "Usage: webctx search <query> [--exclude domains] [--keyword phrase]")
+			return 1
+		}
+		excludeDomains := splitCSV(flags["exclude"])
+		text, err := Search(SearchParams{Query: query, ExcludeDomains: excludeDomains, IncludeKeyword: flags["keyword"]})
+		if err != nil {
+			_, _ = fmt.Fprintln(stderr, err.Error())
+			return 1
+		}
+		_, _ = fmt.Fprintln(stdout, text)
+		return 0
+	case "read-link":
+		if strings.TrimSpace(input) == "" {
+			_, _ = fmt.Fprintln(stderr, "Error: read-link requires a URL")
+			_, _ = fmt.Fprintln(stdout, "Usage: webctx read-link <url>")
+			return 1
+		}
+		text, err := ReadLink(input)
+		if err != nil {
+			_, _ = fmt.Fprintln(stderr, err.Error())
+			return 1
+		}
+		_, _ = fmt.Fprintln(stdout, text)
+		return 0
+	case "map-site":
+		if strings.TrimSpace(input) == "" {
+			_, _ = fmt.Fprintln(stderr, "Error: map-site requires a URL")
+			_, _ = fmt.Fprintln(stdout, "Usage: webctx map-site <url>")
+			return 1
+		}
+		text, err := MapSite(input)
+		if err != nil {
+			_, _ = fmt.Fprintln(stderr, err.Error())
+			return 1
+		}
+		_, _ = fmt.Fprintln(stdout, text)
+		return 0
+	default:
+		_, _ = fmt.Fprintln(stderr, "Unknown tool:", tool)
+		_, _ = fmt.Fprintln(stdout, usageText())
+		return 1
+	}
+}
+
+func usageText() string {
+	return fmt.Sprintf(`webctx v%s - Web search & browsing CLI
+
+Usage:
+  webctx search <query> [--exclude domain1,domain2] [--keyword phrase]
+  webctx read-link <url>
+  webctx map-site <url>
+
+Examples:
+  webctx search "next.js server components"
+  webctx search "react hooks" --exclude youtube.com,vimeo.com
+  webctx search "drizzle orm" --keyword "migration guide"
+  webctx read-link https://docs.example.com/guide
+  webctx map-site https://example.com`, version)
+}
+
+func parseArgs(args []string) (map[string]string, []string) {
+	flags := map[string]string{}
+	positional := make([]string, 0, len(args))
+	for i := 0; i < len(args); i++ {
+		if strings.HasPrefix(args[i], "--") && i+1 < len(args) {
+			flags[strings.TrimPrefix(args[i], "--")] = args[i+1]
+			i++
+			continue
+		}
+		positional = append(positional, args[i])
+	}
+	return flags, positional
+}
+
+func splitCSV(v string) []string {
+	if strings.TrimSpace(v) == "" {
+		return nil
+	}
+	parts := strings.Split(v, ",")
+	out := make([]string, 0, len(parts))
+	for _, part := range parts {
+		trimmed := strings.TrimSpace(part)
+		if trimmed != "" {
+			out = append(out, trimmed)
+		}
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	return out
+}
+
+func isHelpArg(v string) bool {
+	switch v {
+	case "-h", "--help", "help":
+		return true
+	default:
+		return false
+	}
+}

package/internal/app/app_test.go

@@ -0,0 +1,77 @@
+package app
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+)
+
+func TestRunRootHelp(t *testing.T) {
+	var out bytes.Buffer
+	var errBuf bytes.Buffer
+
+	code := Run([]string{"--help"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("Run returned code %d", code)
+	}
+	if !strings.Contains(out.String(), "webctx v") || !strings.Contains(out.String(), "read-link") {
+		t.Fatalf("unexpected help output: %q", out.String())
+	}
+}
+
+func TestRunVersion(t *testing.T) {
+	var out bytes.Buffer
+	var errBuf bytes.Buffer
+
+	code := Run([]string{"--version"}, &out, &errBuf)
+	if code != 0 {
+		t.Fatalf("Run returned code %d", code)
+	}
+	if strings.TrimSpace(out.String()) == "" {
+		t.Fatalf("unexpected empty version output")
+	}
+	if strings.Contains(out.String(), "webctx ") {
+		t.Fatalf("expected bare version output, got: %q", out.String())
+	}
+}
+
+func TestRunSearchWithoutQuery(t *testing.T) {
+	var out bytes.Buffer
+	var errBuf bytes.Buffer
+
+	code := Run([]string{"search"}, &out, &errBuf)
+	if code != 1 {
+		t.Fatalf("expected exit code 1, got %d", code)
+	}
+	if !strings.Contains(errBuf.String(), "search requires a query") {
+		t.Fatalf("unexpected stderr: %q", errBuf.String())
+	}
+}
+
+func TestNormalizeURL(t *testing.T) {
+	got := normalizeURL("https://Example.com/docs/?utm_source=x&ref=y&q=go")
+	want := "https://example.com/docs?q=go"
+	if got != want {
+		t.Fatalf("normalizeURL mismatch: got %q want %q", got, want)
+	}
+}
+
+func TestScoreAndRankResultsDuplicateBoost(t *testing.T) {
+	results, duplicatesRemoved := scoreAndRankResults([]providerDocs{
+		{Provider: "Brave", Docs: []SearchDoc{{URL: "https://a.com", Title: "A"}, {URL: "https://b.com", Title: "B"}}},
+		{Provider: "Tavily", Docs: []SearchDoc{{URL: "https://b.com", Title: "B2"}, {URL: "https://c.com", Title: "C"}}},
+	})
+	if duplicatesRemoved != 1 {
+		t.Fatalf("expected 1 duplicate removed, got %d", duplicatesRemoved)
+	}
+	if len(results) == 0 || results[0].URL != "https://b.com" {
+		t.Fatalf("expected duplicate URL to rank first, got %#v", results)
+	}
+}
+
+func TestParseGitHubURL(t *testing.T) {
+	info := parseGitHubURL("https://github.com/amxv/webctx-ts/blob/main/cli.ts")
+	if info == nil || !info.IsFile || info.Owner != "amxv" || info.Repo != "webctx-ts" || info.Branch != "main" || info.Path != "cli.ts" {
+		t.Fatalf("unexpected parse result: %#v", info)
+	}
+}

package/internal/app/scrape.go

@@ -0,0 +1,310 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+type MarkdownResult struct {
+	URL      string
+	Title    string
+	Markdown string
+}
+
+type githubURLInfo struct {
+	Owner  string
+	Repo   string
+	Branch string
+	Path   string
+	IsFile bool
+}
+
+func parseGitHubURL(raw string) *githubURLInfo {
+	parsed, err := url.Parse(raw)
+	if err != nil || parsed.Hostname() != "github.com" {
+		return nil
+	}
+	parts := strings.FieldsFunc(strings.TrimPrefix(parsed.Path, "/"), func(r rune) bool { return r == '/' })
+	if len(parts) < 2 {
+		return nil
+	}
+	info := &githubURLInfo{Owner: parts[0], Repo: parts[1], IsFile: true}
+	if len(parts) == 2 {
+		return info
+	}
+	switch parts[2] {
+	case "tree":
+		info.IsFile = false
+		if len(parts) > 3 {
+			info.Branch = parts[3]
+		}
+		if len(parts) > 4 {
+			info.Path = strings.Join(parts[4:], "/")
+		}
+		return info
+	case "blob":
+		if len(parts) > 3 {
+			info.Branch = parts[3]
+		}
+		if len(parts) > 4 {
+			info.Path = strings.Join(parts[4:], "/")
+		}
+		return info
+	default:
+		return nil
+	}
+}
+
+func convertToRawGitHubURL(info *githubURLInfo) string {
+	if info.Path == "" {
+		return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/HEAD/README.md", info.Owner, info.Repo)
+	}
+	branch := info.Branch
+	if branch == "" {
+		branch = "HEAD"
+	}
+	return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", info.Owner, info.Repo, branch, info.Path)
+}
+
+func fetchGitHubRawContent(raw string) (*MarkdownResult, error) {
+	info := parseGitHubURL(raw)
+	if info == nil || !info.IsFile {
+		return nil, nil
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	content, status, err := fetchText(ctx, convertToRawGitHubURL(info))
+	if err != nil || status < 200 || status >= 300 {
+		if info.Path == "" && status == http.StatusNotFound {
+			for _, alt := range []string{"readme.md", "Readme.md", "README"} {
+				altURL := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/HEAD/%s", info.Owner, info.Repo, alt)
+				content, status, err = fetchText(ctx, altURL)
+				if err == nil && status >= 200 && status < 300 {
+					title := firstHeadingOrFallback(content, info.Owner+"/"+info.Repo)
+					return &MarkdownResult{URL: raw, Title: title, Markdown: content}, nil
+				}
+			}
+		}
+		return nil, nil
+	}
+	title := firstHeadingOrFallback(content, fallbackGitHubTitle(info))
+	return &MarkdownResult{URL: raw, Title: title, Markdown: content}, nil
+}
+
+func checkMarkdownAvailable(raw string) (bool, error) {
+	mdURL := raw
+	if !strings.HasSuffix(strings.ToLower(mdURL), ".md") {
+		mdURL += ".md"
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, http.MethodHead, mdURL, nil)
+	if err != nil {
+		return false, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return false, nil
+	}
+	contentType := strings.ToLower(resp.Header.Get("Content-Type"))
+	contentLength := resp.ContentLength
+	return (strings.Contains(contentType, "markdown") || strings.Contains(contentType, "text/plain")) && contentLength > 50, nil
+}
+
+func fetchMarkdownContent(raw string) (*MarkdownResult, error) {
+	mdURL := raw
+	if !strings.HasSuffix(strings.ToLower(mdURL), ".md") {
+		mdURL += ".md"
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	content, status, err := fetchText(ctx, mdURL)
+	if err != nil {
+		return nil, err
+	}
+	if status < 200 || status >= 300 {
+		return nil, fmt.Errorf("Failed to fetch markdown: %d", status)
+	}
+	title := firstHeadingOrFallback(content, filepath.Base(raw))
+	return &MarkdownResult{URL: raw, Title: title, Markdown: content}, nil
+}
+
+func fetchText(ctx context.Context, rawURL string) (string, int, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return "", 0, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", 0, err
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", resp.StatusCode, err
+	}
+	return string(body), resp.StatusCode, nil
+}
+
+func firstHeadingOrFallback(markdown, fallback string) string {
+	for _, line := range strings.Split(markdown, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "# ") {
+			return strings.TrimSpace(strings.TrimPrefix(trimmed, "# "))
+		}
+	}
+	if strings.TrimSpace(fallback) == "" {
+		return "Document"
+	}
+	return fallback
+}
+
+func fallbackGitHubTitle(info *githubURLInfo) string {
+	if info.Path != "" {
+		parts := strings.Split(info.Path, "/")
+		return parts[len(parts)-1]
+	}
+	return info.Owner + "/" + info.Repo
+}
+
+type tokenBucketRateLimiter struct {
+	mu             sync.Mutex
+	tokens         int
+	maxTokens      int
+	refillRate     int
+	refillInterval time.Duration
+	lastRefill     time.Time
+}
+
+func newFirecrawlRateLimiter() *tokenBucketRateLimiter {
+	return &tokenBucketRateLimiter{tokens: 10, maxTokens: 10, refillRate: 1, refillInterval: 6 * time.Second, lastRefill: time.Now()}
+}
+
+func (r *tokenBucketRateLimiter) refill() {
+	now := time.Now()
+	elapsed := now.Sub(r.lastRefill)
+	if elapsed < r.refillInterval {
+		return
+	}
+	intervals := int(elapsed / r.refillInterval)
+	if intervals <= 0 {
+		return
+	}
+	r.tokens += intervals * r.refillRate
+	if r.tokens > r.maxTokens {
+		r.tokens = r.maxTokens
+	}
+	r.lastRefill = r.lastRefill.Add(time.Duration(intervals) * r.refillInterval)
+}
+
+func (r *tokenBucketRateLimiter) acquire(ctx context.Context) error {
+	for {
+		r.mu.Lock()
+		r.refill()
+		if r.tokens > 0 {
+			r.tokens--
+			r.mu.Unlock()
+			return nil
+		}
+		wait := r.refillInterval - time.Since(r.lastRefill) + 100*time.Millisecond
+		r.mu.Unlock()
+		if wait < 100*time.Millisecond {
+			wait = 100 * time.Millisecond
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(wait):
+		}
+	}
+}
+
+type firecrawlQueue struct {
+	rateLimiter *tokenBucketRateLimiter
+	mu          sync.Mutex
+}
+
+var (
+	queueOnce sync.Once
+	queueInst *firecrawlQueue
+)
+
+func getFirecrawlQueue() *firecrawlQueue {
+	queueOnce.Do(func() {
+		queueInst = &firecrawlQueue{rateLimiter: newFirecrawlRateLimiter()}
+	})
+	return queueInst
+}
+
+func (q *firecrawlQueue) enqueue(_ string, requestFn func() (map[string]any, error)) (map[string]any, error) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+	defer cancel()
+	if err := q.rateLimiter.acquire(ctx); err != nil {
+		return nil, err
+	}
+	return requestFn()
+}
+
+func loadEnvLocal() {
+	for _, candidate := range envLocalCandidates() {
+		loadDotEnvFile(candidate)
+	}
+}
+
+func envLocalCandidates() []string {
+	candidates := []string{}
+	if cwd, err := os.Getwd(); err == nil {
+		candidates = append(candidates, filepath.Join(cwd, ".env.local"))
+	}
+	if exe, err := os.Executable(); err == nil {
+		exeDir := filepath.Dir(exe)
+		candidates = append(candidates, filepath.Join(exeDir, ".env.local"), filepath.Join(filepath.Dir(exeDir), ".env.local"))
+	}
+	seen := map[string]struct{}{}
+	unique := []string{}
+	for _, c := range candidates {
+		if _, ok := seen[c]; ok {
+			continue
+		}
+		seen[c] = struct{}{}
+		unique = append(unique, c)
+	}
+	return unique
+}
+
+func loadDotEnvFile(path string) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return
+	}
+	for _, line := range strings.Split(string(data), "\n") {
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" || strings.HasPrefix(trimmed, "#") {
+			continue
+		}
+		trimmed = strings.TrimPrefix(trimmed, "export ")
+		key, value, ok := strings.Cut(trimmed, "=")
+		if !ok {
+			continue
+		}
+		key = strings.TrimSpace(key)
+		value = strings.Trim(strings.TrimSpace(value), `"'`)
+		if key != "" {
+			_ = os.Setenv(key, value)
+		}
+	}
+}