recursive-llm-ts 5.0.0 → 5.0.2

package/go/rlm/tokenizer_test.go

@@ -1,305 +0,0 @@
-package rlm
-
-import (
-	"strings"
-	"testing"
-)
-
-// ─── Tiktoken BPE Tokenizer Tests ────────────────────────────────────────────
-
-func TestTiktokenTokenizer_English(t *testing.T) {
-	tok, err := NewTiktokenTokenizer("gpt-4o")
-	if err != nil {
-		t.Fatalf("failed to create tokenizer: %v", err)
-	}
-
-	text := "Hello, world! This is a test of the tokenizer."
-	bpeCount := tok.CountTokens(text)
-	heuristic := (&HeuristicTokenizer{}).CountTokens(text)
-
-	t.Logf("English text: BPE=%d, Heuristic=%d", bpeCount, heuristic)
-
-	if bpeCount <= 0 {
-		t.Error("BPE count should be > 0")
-	}
-	// BPE and heuristic should give different counts for most text
-	// (they may coincidentally match for very short strings, so just verify BPE is reasonable)
-	if bpeCount > len(text) {
-		t.Errorf("BPE count %d should not exceed character count %d", bpeCount, len(text))
-	}
-}
-
-func TestTiktokenTokenizer_Code(t *testing.T) {
-	tok, err := NewTiktokenTokenizer("gpt-4o")
-	if err != nil {
-		t.Fatalf("failed to create tokenizer: %v", err)
-	}
-
-	code := `func main() {
-	fmt.Println("Hello, World!")
-	for i := 0; i < 100; i++ {
-		result = append(result, processItem(items[i]))
-	}
-}`
-	bpeCount := tok.CountTokens(code)
-	heuristic := (&HeuristicTokenizer{}).CountTokens(code)
-
-	t.Logf("Code: BPE=%d, Heuristic=%d (chars=%d)", bpeCount, heuristic, len(code))
-
-	if bpeCount <= 0 {
-		t.Error("BPE count should be > 0 for code")
-	}
-}
-
-func TestTiktokenTokenizer_JSON(t *testing.T) {
-	tok, err := NewTiktokenTokenizer("gpt-4o")
-	if err != nil {
-		t.Fatalf("failed to create tokenizer: %v", err)
-	}
-
-	jsonData := `{"name": "test", "values": [1, 2, 3, 4, 5], "nested": {"key": "value", "count": 42}}`
-	bpeCount := tok.CountTokens(jsonData)
-	heuristic := (&HeuristicTokenizer{}).CountTokens(jsonData)
-
-	t.Logf("JSON: BPE=%d, Heuristic=%d (chars=%d)", bpeCount, heuristic, len(jsonData))
-
-	if bpeCount <= 0 {
-		t.Error("BPE count should be > 0 for JSON")
-	}
-}
-
-func TestTiktokenTokenizer_CJK(t *testing.T) {
-	tok, err := NewTiktokenTokenizer("gpt-4o")
-	if err != nil {
-		t.Fatalf("failed to create tokenizer: %v", err)
-	}
-
-	// Chinese, Japanese, and Korean text
-	cjkText := "这是一个测试。日本語のテスト。한국어 테스트입니다。"
-	bpeCount := tok.CountTokens(cjkText)
-	heuristic := (&HeuristicTokenizer{}).CountTokens(cjkText)
-
-	t.Logf("CJK text: BPE=%d, Heuristic=%d (chars=%d, bytes=%d)", bpeCount, heuristic, len([]rune(cjkText)), len(cjkText))
-
-	if bpeCount <= 0 {
-		t.Error("BPE count should be > 0 for CJK")
-	}
-	// CJK text has ~1.5 chars per token but heuristic assumes ~3.5
-	// So BPE should count MORE tokens than heuristic for CJK
-	if bpeCount <= heuristic {
-		t.Logf("WARNING: BPE (%d) should typically be > heuristic (%d) for CJK text", bpeCount, heuristic)
-	}
-}
-
-func TestTiktokenTokenizer_Empty(t *testing.T) {
-	tok, err := NewTiktokenTokenizer("gpt-4o")
-	if err != nil {
-		t.Fatalf("failed to create tokenizer: %v", err)
-	}
-
-	if tok.CountTokens("") != 0 {
-		t.Error("empty string should return 0 tokens")
-	}
-}
-
-func TestTiktokenTokenizer_EncodingSelection(t *testing.T) {
-	tests := []struct {
-		model    string
-		expected string
-	}{
-		{"gpt-4o", "o200k_base"},
-		{"gpt-4o-mini", "o200k_base"},
-		{"gpt-4o-mini-2024-07-18", "o200k_base"},
-		{"gpt-4", "cl100k_base"},
-		{"gpt-4-turbo", "cl100k_base"},
-		{"gpt-3.5-turbo", "cl100k_base"},
-		{"claude-3-opus", "cl100k_base"},
-		{"claude-sonnet-4", "cl100k_base"},
-		{"o1", "o200k_base"},
-		{"o3-mini", "o200k_base"},
-		{"llama-3.1", "cl100k_base"},
-		{"unknown-model", "cl100k_base"},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.model, func(t *testing.T) {
-			enc := encodingForModel(tt.model)
-			if enc != tt.expected {
-				t.Errorf("encodingForModel(%q) = %q, want %q", tt.model, enc, tt.expected)
-			}
-		})
-	}
-}
-
-// ─── Heuristic Tokenizer Tests ───────────────────────────────────────────────
-
-func TestHeuristicTokenizer_Fallback(t *testing.T) {
-	h := &HeuristicTokenizer{}
-
-	if h.CountTokens("") != 0 {
-		t.Error("empty string should be 0")
-	}
-
-	// "Hello" = 5 chars, ceil(5/3.5) = 2
-	count := h.CountTokens("Hello")
-	if count <= 0 {
-		t.Error("should count > 0 tokens for 'Hello'")
-	}
-
-	// Longer text
-	longText := strings.Repeat("word ", 1000)
-	longCount := h.CountTokens(longText)
-	expected := (len(longText)*10 + 34) / 35
-	if longCount != expected {
-		t.Errorf("heuristic count %d != expected %d", longCount, expected)
-	}
-}
-
-// ─── Cached Tokenizer Tests ─────────────────────────────────────────────────
-
-func TestCachedTokenizer_CacheHit(t *testing.T) {
-	callCount := 0
-	inner := &countingTokenizer{fn: func(text string) int {
-		callCount++
-		return len(text) / 4
-	}}
-
-	cached := NewCachedTokenizer(inner)
-
-	text := "This is a test string for caching"
-
-	// First call: cache miss
-	count1 := cached.CountTokens(text)
-	if callCount != 1 {
-		t.Errorf("expected 1 inner call, got %d", callCount)
-	}
-
-	// Second call: cache hit (inner should NOT be called again)
-	count2 := cached.CountTokens(text)
-	if callCount != 1 {
-		t.Errorf("expected still 1 inner call after cache hit, got %d", callCount)
-	}
-
-	if count1 != count2 {
-		t.Errorf("cache returned different values: %d vs %d", count1, count2)
-	}
-
-	if cached.CacheSize() != 1 {
-		t.Errorf("cache size should be 1, got %d", cached.CacheSize())
-	}
-}
-
-func TestCachedTokenizer_Empty(t *testing.T) {
-	inner := &HeuristicTokenizer{}
-	cached := NewCachedTokenizer(inner)
-
-	if cached.CountTokens("") != 0 {
-		t.Error("empty string should return 0 without caching")
-	}
-	if cached.CacheSize() != 0 {
-		t.Error("cache should not store empty strings")
-	}
-}
-
-func TestCachedTokenizer_DifferentStrings(t *testing.T) {
-	inner := &HeuristicTokenizer{}
-	cached := NewCachedTokenizer(inner)
-
-	cached.CountTokens("string one")
-	cached.CountTokens("string two")
-	cached.CountTokens("string three")
-
-	if cached.CacheSize() != 3 {
-		t.Errorf("cache size should be 3, got %d", cached.CacheSize())
-	}
-}
-
-func TestCachedTokenizer_Inner(t *testing.T) {
-	inner := &HeuristicTokenizer{}
-	cached := NewCachedTokenizer(inner)
-
-	if cached.Inner() != inner {
-		t.Error("Inner() should return the wrapped tokenizer")
-	}
-}
-
-// countingTokenizer tracks how many times CountTokens is called.
-type countingTokenizer struct {
-	fn func(string) int
-}
-
-func (c *countingTokenizer) CountTokens(text string) int {
-	return c.fn(text)
-}
-
-// ─── Global Default Tokenizer Tests ──────────────────────────────────────────
-
-func TestSetDefaultTokenizer_KnownModel(t *testing.T) {
-	defer ResetDefaultTokenizer()
-
-	SetDefaultTokenizer("gpt-4o")
-	tok := GetTokenizer()
-
-	// Should be a CachedTokenizer wrapping a TiktokenTokenizer
-	cached, ok := tok.(*CachedTokenizer)
-	if !ok {
-		t.Fatalf("expected CachedTokenizer, got %T", tok)
-	}
-
-	inner, ok := cached.Inner().(*TiktokenTokenizer)
-	if !ok {
-		t.Fatalf("expected inner TiktokenTokenizer, got %T", cached.Inner())
-	}
-
-	if inner.EncodingName() != "o200k_base" {
-		t.Errorf("expected o200k_base encoding, got %s", inner.EncodingName())
-	}
-
-	// Verify it actually counts tokens
-	count := tok.CountTokens("Hello, world!")
-	if count <= 0 {
-		t.Error("tokenizer should count > 0 tokens")
-	}
-}
-
-func TestSetDefaultTokenizer_UnknownModel(t *testing.T) {
-	defer ResetDefaultTokenizer()
-
-	// Even unknown models should work because we default to cl100k_base
-	SetDefaultTokenizer("totally-unknown-model-xyz")
-	tok := GetTokenizer()
-
-	count := tok.CountTokens("Hello, world!")
-	if count <= 0 {
-		t.Error("tokenizer should count > 0 tokens even for unknown model")
-	}
-}
-
-func TestEstimateTokens_UsesDefaultTokenizer(t *testing.T) {
-	defer ResetDefaultTokenizer()
-
-	// With heuristic default
-	heuristicCount := EstimateTokens("Hello, world! This is a test.")
-
-	// Switch to BPE
-	SetDefaultTokenizer("gpt-4o")
-	bpeCount := EstimateTokens("Hello, world! This is a test.")
-
-	t.Logf("EstimateTokens: heuristic=%d, bpe=%d", heuristicCount, bpeCount)
-
-	// Both should be > 0
-	if heuristicCount <= 0 || bpeCount <= 0 {
-		t.Errorf("both counts should be > 0: heuristic=%d, bpe=%d", heuristicCount, bpeCount)
-	}
-}
-
-func TestResetDefaultTokenizer(t *testing.T) {
-	SetDefaultTokenizer("gpt-4o")
-	ResetDefaultTokenizer()
-
-	tok := GetTokenizer()
-	_, ok := tok.(*HeuristicTokenizer)
-	if !ok {
-		t.Errorf("after reset, expected HeuristicTokenizer, got %T", tok)
-	}
-}

package/go/test_mock.sh

@@ -1,90 +0,0 @@
-#!/bin/bash
-# Test Go binary with mock responses (no API key needed)
-
-set -e
-
-echo "🧪 RLM Go Mock Tests (No API Required)"
-echo "================================"
-echo ""
-
-# Test 1: Binary accepts input
-echo "📝 Test 1: Binary accepts and parses JSON input"
-echo "-----------------------------------"
-RESULT=$(cat <<EOF | ./rlm 2>&1
-{
-  "model": "test-model",
-  "query": "test",
-  "context": "test",
-  "config": {}
-}
-EOF
-)
-
-if echo "$RESULT" | grep -q "error"; then
-    echo "✅ Binary correctly handles missing API key"
-    echo "Error output: $(echo $RESULT | head -c 100)..."
-else
-    echo "⚠️  Unexpected output (but binary ran)"
-fi
-echo ""
-
-# Test 2: Config parsing
-echo "📝 Test 2: Config parsing works"
-echo "-----------------------------------"
-RESULT=$(cat <<EOF | ./rlm 2>&1
-{
-  "model": "gpt-4o-mini",
-  "query": "test",
-  "context": "test",
-  "config": {
-    "max_depth": 3,
-    "max_iterations": 15,
-    "temperature": 0.5
-  }
-}
-EOF
-)
-
-if echo "$RESULT" | grep -q "error\|Error"; then
-    echo "✅ Binary accepts configuration"
-else
-    echo "⚠️  Unexpected response"
-fi
-echo ""
-
-# Test 3: Binary executable and responds
-echo "📝 Test 3: Binary is executable"
-echo "-----------------------------------"
-if [ -x "./rlm" ]; then
-    echo "✅ Binary is executable"
-    SIZE=$(ls -lh ./rlm | awk '{print $5}')
-    echo "Binary size: $SIZE"
-else
-    echo "❌ Binary is not executable"
-    exit 1
-fi
-echo ""
-
-# Test 4: Help/version (invalid input)
-echo "📝 Test 4: Binary handles invalid input"
-echo "-----------------------------------"
-RESULT=$(echo "invalid json" | ./rlm 2>&1 || true)
-if echo "$RESULT" | grep -q "Failed to parse input JSON\|error"; then
-    echo "✅ Binary properly handles invalid input"
-else
-    echo "⚠️  Unexpected error handling"
-fi
-echo ""
-
-echo "================================"
-echo "✅ All mock tests passed!"
-echo ""
-echo "Summary:"
-echo "  - Binary is built and executable"
-echo "  - Accepts JSON input via stdin"
-echo "  - Parses configuration correctly"
-echo "  - Handles errors gracefully"
-echo "  - Returns JSON output on stdout"
-echo ""
-echo "⚠️  Cannot test with real API (quota exceeded)"
-echo "    But binary structure is validated ✅"

package/go/test_rlm.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-# Test script for Go RLM implementation
-
-set -e
-
-echo "=== Testing Go RLM Implementation ==="
-echo ""
-
-# Test 1: Simple FINAL response
-echo "Test 1: Simple FINAL response"
-cat <<'EOF' | ./rlm
-{
-  "model": "test-model",
-  "query": "What is the answer?",
-  "context": "The answer is 42.",
-  "config": {
-    "api_base": "http://mock-api.example.com",
-    "api_key": "test-key",
-    "max_depth": 5,
-    "max_iterations": 30
-  }
-}
-EOF
-echo ""
-
-# Test 2: Code extraction from markdown
-echo "Test 2: JavaScript code execution"
-cat <<'EOF' | ./rlm
-{
-  "model": "test-model",
-  "query": "Count words",
-  "context": "Hello World Test Document",
-  "config": {
-    "max_depth": 3,
-    "max_iterations": 10
-  }
-}
-EOF
-echo ""
-
-echo "=== All tests completed ==="

package/go/test_simple.sh

@@ -1,78 +0,0 @@
-#!/bin/bash
-# Simple integration tests
-
-set -e
-
-echo "🧪 Simple RLM Integration Tests"
-echo "================================"
-echo ""
-
-if [ -z "$OPENAI_API_KEY" ]; then
-    echo "❌ Set OPENAI_API_KEY first"
-    exit 1
-fi
-
-# Test 1
-echo "Test 1: Count word occurrences"
-cat > /tmp/test1.json <<'JSON'
-{
-  "model": "gpt-4o-mini",
-  "query": "How many times does the word 'test' appear?",
-  "context": "This is a test. Another test here. Final test.",
-  "config": {
-    "max_iterations": 10
-  }
-}
-JSON
-
-echo "Running..."
-RESULT=$(cat /tmp/test1.json | OPENAI_API_KEY="$OPENAI_API_KEY" ./rlm)
-echo "✅ Test 1 Result:"
-echo "$RESULT" | jq -r '.result'
-echo "Stats:" $(echo "$RESULT" | jq '.stats')
-echo ""
-
-# Test 2
-echo "Test 2: Simple counting"
-cat > /tmp/test2.json <<'JSON'
-{
-  "model": "gpt-4o-mini",
-  "query": "How many words are in this text?",
-  "context": "One two three four five",
-  "config": {
-    "max_iterations": 10,
-    "temperature": 0.1
-  }
-}
-JSON
-
-echo "Running..."
-RESULT=$(cat /tmp/test2.json | OPENAI_API_KEY="$OPENAI_API_KEY" ./rlm)
-echo "✅ Test 2 Result:"
-echo "$RESULT" | jq -r '.result'
-echo "Stats:" $(echo "$RESULT" | jq '.stats')
-echo ""
-
-# Test 3
-echo "Test 3: Extract information"
-cat > /tmp/test3.json <<'JSON'
-{
-  "model": "gpt-4o-mini",
-  "query": "List all the numbers mentioned",
-  "context": "I have 5 apples, 10 oranges, and 3 bananas.",
-  "config": {
-    "max_iterations": 10
-  }
-}
-JSON
-
-echo "Running..."
-RESULT=$(cat /tmp/test3.json | OPENAI_API_KEY="$OPENAI_API_KEY" ./rlm)
-echo "✅ Test 3 Result:"
-echo "$RESULT" | jq -r '.result'
-echo "Stats:" $(echo "$RESULT" | jq '.stats')
-echo ""
-
-echo "================================"
-echo "✅ All tests passed!"
-rm -f /tmp/test*.json