recursive-llm-ts 2.0.11 → 3.0.1

package/go/internal/rlm/parser_test.go

@@ -0,0 +1,202 @@
+package rlm
+
+import (
+	"testing"
+)
+
+func TestIsFinal(t *testing.T) {
+	tests := []struct {
+		name     string
+		response string
+		want     bool
+	}{
+		{"FINAL with double quotes", `FINAL("answer")`, true},
+		{"FINAL with single quotes", `FINAL('answer')`, true},
+		{"FINAL with triple double quotes", `FINAL("""answer""")`, true},
+		{"FINAL_VAR", `FINAL_VAR(result)`, true},
+		{"No FINAL", `x = 1`, false},
+		{"Contains FINAL as substring", `This is FINALLY done`, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsFinal(tt.response); got != tt.want {
+				t.Errorf("IsFinal() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestExtractFinal(t *testing.T) {
+	tests := []struct {
+		name     string
+		response string
+		want     string
+		wantOk   bool
+	}{
+		{
+			name:     "Double quotes",
+			response: `FINAL("The answer is 42")`,
+			want:     "The answer is 42",
+			wantOk:   true,
+		},
+		{
+			name:     "Single quotes",
+			response: `FINAL('The answer is 42')`,
+			want:     "The answer is 42",
+			wantOk:   true,
+		},
+		{
+			name:     "Triple double quotes",
+			response: `FINAL("""The answer is 42""")`,
+			want:     "The answer is 42",
+			wantOk:   true,
+		},
+		{
+			name:     "Triple single quotes",
+			response: `FINAL('''The answer is 42''')`,
+			want:     "The answer is 42",
+			wantOk:   true,
+		},
+		{
+			name: "Multiline with triple quotes",
+			response: `FINAL("""Line 1
+Line 2
+Line 3""")`,
+			want:   "Line 1\nLine 2\nLine 3",
+			wantOk: true,
+		},
+		{
+			name:     "With whitespace",
+			response: `FINAL(  "The answer"  )`,
+			want:     "The answer",
+			wantOk:   true,
+		},
+		{
+			name:     "No FINAL",
+			response: `x = 1`,
+			want:     "",
+			wantOk:   false,
+		},
+		{
+			name:     "FINAL_VAR should not match",
+			response: `FINAL_VAR(result)`,
+			want:     "",
+			wantOk:   false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, gotOk := extractFinal(tt.response)
+			if gotOk != tt.wantOk {
+				t.Errorf("extractFinal() ok = %v, want %v", gotOk, tt.wantOk)
+			}
+			if got != tt.want {
+				t.Errorf("extractFinal() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestExtractFinalVar(t *testing.T) {
+	tests := []struct {
+		name     string
+		response string
+		env      map[string]interface{}
+		want     string
+		wantOk   bool
+	}{
+		{
+			name:     "Simple variable",
+			response: `FINAL_VAR(result)`,
+			env:      map[string]interface{}{"result": "The answer"},
+			want:     "The answer",
+			wantOk:   true,
+		},
+		{
+			name:     "Integer variable",
+			response: `FINAL_VAR(count)`,
+			env:      map[string]interface{}{"count": 42},
+			want:     "42",
+			wantOk:   true,
+		},
+		{
+			name:     "Variable not found",
+			response: `FINAL_VAR(missing)`,
+			env:      map[string]interface{}{"result": "The answer"},
+			want:     "",
+			wantOk:   false,
+		},
+		{
+			name:     "With whitespace",
+			response: `FINAL_VAR( result )`,
+			env:      map[string]interface{}{"result": "The answer"},
+			want:     "The answer",
+			wantOk:   true,
+		},
+		{
+			name:     "No FINAL_VAR",
+			response: `x = 1`,
+			env:      map[string]interface{}{},
+			want:     "",
+			wantOk:   false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, gotOk := extractFinalVar(tt.response, tt.env)
+			if gotOk != tt.wantOk {
+				t.Errorf("extractFinalVar() ok = %v, want %v", gotOk, tt.wantOk)
+			}
+			if got != tt.want {
+				t.Errorf("extractFinalVar() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseResponse(t *testing.T) {
+	tests := []struct {
+		name     string
+		response string
+		env      map[string]interface{}
+		want     string
+		wantOk   bool
+	}{
+		{
+			name:     "FINAL takes precedence",
+			response: `FINAL("Direct answer")`,
+			env:      map[string]interface{}{"result": "Var answer"},
+			want:     "Direct answer",
+			wantOk:   true,
+		},
+		{
+			name:     "FINAL_VAR fallback",
+			response: `FINAL_VAR(result)`,
+			env:      map[string]interface{}{"result": "Var answer"},
+			want:     "Var answer",
+			wantOk:   true,
+		},
+		{
+			name:     "Neither",
+			response: `x = 1`,
+			env:      map[string]interface{}{},
+			want:     "",
+			wantOk:   false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, gotOk := ParseResponse(tt.response, tt.env)
+			if gotOk != tt.wantOk {
+				t.Errorf("ParseResponse() ok = %v, want %v", gotOk, tt.wantOk)
+			}
+			if got != tt.want {
+				t.Errorf("ParseResponse() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}

package/go/internal/rlm/prompt.go

@@ -0,0 +1,68 @@
+package rlm
+
+import "fmt"
+
+// BuildSystemPrompt creates the system prompt for RLM
+// useMetacognitive enables step-by-step reasoning guidance
+func BuildSystemPrompt(contextSize int, depth int, query string, useMetacognitive bool) string {
+	if useMetacognitive {
+		return buildMetacognitivePrompt(contextSize, depth, query)
+	}
+	return buildMinimalPrompt(contextSize, depth, query)
+}
+
+func buildMinimalPrompt(contextSize int, depth int, query string) string {
+	return fmt.Sprintf(`You are a Recursive Language Model. You interact with context through a JavaScript REPL environment.
+
+The context is stored in variable "context" (not in this prompt). Size: %d characters.
+
+Available in environment:
+- context: string (the document to analyze)
+- query: string (the question: %q)
+- recursive_llm(sub_query, sub_context) -> string (recursively process sub-context)
+- re: regex helper with findall(pattern, text) and search(pattern, text)
+- print(value, ...) -> output text
+- len(value) -> length of arrays/strings
+- json: helper with loads() and dumps()
+- math: Math helper
+- datetime: Date helper
+- Counter(iterable) -> object of counts
+- defaultdict(defaultFactory) -> object with defaults
+
+Write JavaScript code to answer the query. The last expression or console.log output will be shown to you.
+
+Examples:
+- console.log(context.slice(0, 100)); // See first 100 chars
+- const errors = re.findall("ERROR", context);
+- const count = errors.length; console.log(count);
+
+When you have the answer, use FINAL("answer") - this is NOT a function, just write it as text.
+
+Depth: %d`, contextSize, query, depth)
+}
+
+func buildMetacognitivePrompt(contextSize int, depth int, query string) string {
+	return fmt.Sprintf(`You are a Recursive Language Model. You interact with context through a JavaScript REPL environment.
+
+The context is stored in variable "context" (not in this prompt). Size: %d characters.
+
+Available in environment:
+- context: string (the document to analyze)
+- query: string (the question: %q)
+- recursive_llm(sub_query, sub_context) -> string (recursively process sub-context)
+- re: regex helper with findall(pattern, text) and search(pattern, text)
+- print(value, ...) -> output text
+- len(value) -> length of arrays/strings
+- json: helper with loads() and dumps()
+- math: Math helper
+- Counter(iterable) -> object of counts
+
+STRATEGY TIP: You can peek at context first to understand its structure before processing.
+Example: console.log(context.slice(0, 100))
+
+Write JavaScript code to answer the query. The last expression or console.log output will be shown to you.
+
+When you have the answer, use FINAL("answer") - this is NOT a function, just write it as text.
+
+Depth: %d`, contextSize, query, depth)
+}

package/go/internal/rlm/repl.go

@@ -0,0 +1,260 @@
+package rlm
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/dop251/goja"
+)
+
+type REPLExecutor struct {
+	maxOutputChars int
+}
+
+func NewREPLExecutor() *REPLExecutor {
+	return &REPLExecutor{
+		maxOutputChars: 2000,
+	}
+}
+
+func (r *REPLExecutor) Execute(code string, env map[string]interface{}) (string, error) {
+	code = extractCode(code)
+	if strings.TrimSpace(code) == "" {
+		return "No code to execute", nil
+	}
+
+	vm := goja.New()
+	var output strings.Builder
+
+	for key, value := range env {
+		vm.Set(key, value)
+	}
+
+	writeOutput := func(call goja.FunctionCall) goja.Value {
+		parts := make([]string, 0, len(call.Arguments))
+		for _, arg := range call.Arguments {
+			parts = append(parts, arg.String())
+		}
+		output.WriteString(strings.Join(parts, " "))
+		output.WriteString("\n")
+		return goja.Undefined()
+	}
+
+	console := map[string]func(goja.FunctionCall) goja.Value{
+		"log": writeOutput,
+	}
+
+	vm.Set("console", console)
+	vm.Set("print", writeOutput)
+	vm.Set("len", func(value goja.Value) int {
+		if value == nil || value == goja.Undefined() || value == goja.Null() {
+			return 0
+		}
+		if exported := value.Export(); exported != nil {
+			switch typed := exported.(type) {
+			case string:
+				return len(typed)
+			case []interface{}:
+				return len(typed)
+			case map[string]interface{}:
+				return len(typed)
+			}
+		}
+		return len(value.String())
+	})
+
+	if _, err := vm.RunString(jsBootstrap); err != nil {
+		return "", NewREPLError("Bootstrap execution error", jsBootstrap, err)
+	}
+
+	if _, err := vm.RunString(code); err != nil {
+		return "", NewREPLError("Code execution error", code, err)
+	}
+
+	if output.Len() == 0 {
+		lastLine := getLastLine(code)
+		if looksLikeExpression(lastLine) {
+			value, err := vm.RunString(lastLine)
+			if err == nil && value != nil && value != goja.Undefined() {
+				output.WriteString(value.String())
+				output.WriteString("\n")
+			}
+		}
+	}
+
+	if output.Len() == 0 {
+		return "Code executed successfully (no output)", nil
+	}
+
+	rawOutput := output.String()
+	trimmedOutput := strings.TrimSpace(rawOutput)
+	if len(rawOutput) > r.maxOutputChars {
+		truncated := rawOutput[:r.maxOutputChars]
+		return fmt.Sprintf("%s\n\n[Output truncated: %d chars total, showing first %d]", strings.TrimSpace(truncated), len(rawOutput), r.maxOutputChars), nil
+	}
+
+	return trimmedOutput, nil
+}
+
+func extractCode(text string) string {
+	if strings.Contains(text, "```python") {
+		return extractBlock(text, "```python")
+	}
+	if strings.Contains(text, "```javascript") {
+		return extractBlock(text, "```javascript")
+	}
+	if strings.Contains(text, "```js") {
+		return extractBlock(text, "```js")
+	}
+	if strings.Contains(text, "```") {
+		return extractBlock(text, "```")
+	}
+	return text
+}
+
+const jsBootstrap = `
+const json = {
+  loads: (text) => JSON.parse(text),
+  dumps: (value, replacer, space) => JSON.stringify(value, replacer, space),
+};
+const math = Math;
+const datetime = Date;
+const Counter = (iterable) => {
+  const counts = {};
+  if (iterable == null) {
+    return counts;
+  }
+  const items = typeof iterable === "string" ? iterable.split("") : iterable;
+  for (const item of items) {
+    const key = String(item);
+    counts[key] = (counts[key] || 0) + 1;
+  }
+  return counts;
+};
+const defaultdict = (defaultFactory) => new Proxy({}, {
+  get(target, prop) {
+    if (!(prop in target)) {
+      target[prop] = typeof defaultFactory === "function" ? defaultFactory() : defaultFactory;
+    }
+    return target[prop];
+  },
+});
+const range = (start, stop, step) => {
+  if (stop === undefined) {
+    stop = start;
+    start = 0;
+  }
+  if (step === undefined) {
+    step = 1;
+  }
+  if (step === 0) {
+    return [];
+  }
+  const result = [];
+  if (step > 0) {
+    for (let i = start; i < stop; i += step) {
+      result.push(i);
+    }
+  } else {
+    for (let i = start; i > stop; i += step) {
+      result.push(i);
+    }
+  }
+  return result;
+};
+const sorted = (iterable, compareFn) => [...iterable].sort(compareFn);
+const sum = (iterable) => (iterable || []).reduce((acc, value) => acc + Number(value), 0);
+const min = (iterable) => Math.min(...iterable);
+const max = (iterable) => Math.max(...iterable);
+const enumerate = (iterable) => (iterable || []).map((value, index) => [index, value]);
+const zip = (...iterables) => {
+  const length = Math.min(...iterables.map((items) => items.length));
+  const result = [];
+  for (let i = 0; i < length; i++) {
+    result.push(iterables.map((items) => items[i]));
+  }
+  return result;
+};
+const any = (iterable) => (iterable || []).some(Boolean);
+const all = (iterable) => (iterable || []).every(Boolean);
+`
+
+func extractBlock(text string, marker string) string {
+	start := strings.Index(text, marker)
+	if start == -1 {
+		return text
+	}
+	start += len(marker)
+	end := strings.Index(text[start:], "```")
+	if end == -1 {
+		return text[start:]
+	}
+	return strings.TrimSpace(text[start : start+end])
+}
+
+func getLastLine(code string) string {
+	lines := strings.Split(strings.TrimSpace(code), "\n")
+	if len(lines) == 0 {
+		return ""
+	}
+	lastLine := strings.TrimSpace(lines[len(lines)-1])
+	// Handle multiple statements on same line separated by semicolon
+	if strings.Contains(lastLine, ";") {
+		parts := strings.Split(lastLine, ";")
+		// Get the last non-empty part
+		for i := len(parts) - 1; i >= 0; i-- {
+			if trimmed := strings.TrimSpace(parts[i]); trimmed != "" {
+				return trimmed
+			}
+		}
+	}
+	return lastLine
+}
+
+func looksLikeExpression(line string) bool {
+	if line == "" {
+		return false
+	}
+	// Check for statements that shouldn't be evaluated as expressions
+	keywords := []string{"const ", "let ", "var ", "function ", "if ", "for ", "while ", "class ", "return "}
+	for _, keyword := range keywords {
+		if strings.HasPrefix(strings.TrimSpace(line), keyword) {
+			return false
+		}
+	}
+	// Check for assignment (but not == or === comparison)
+	if strings.Contains(line, "=") && !strings.Contains(line, "==") && !strings.Contains(line, "===") {
+		return false
+	}
+	return true
+}
+
+func NewRegexHelper() map[string]func(string, string) interface{} {
+	return map[string]func(string, string) interface{}{
+		"findall": func(pattern string, text string) interface{} {
+			re, err := regexpFromPattern(pattern)
+			if err != nil {
+				return []string{}
+			}
+			return re.FindAllString(text, -1)
+		},
+		"search": func(pattern string, text string) interface{} {
+			re, err := regexpFromPattern(pattern)
+			if err != nil {
+				return ""
+			}
+			match := re.FindString(text)
+			return match
+		},
+	}
+}
+
+func regexpFromPattern(pattern string) (*regexp.Regexp, error) {
+	re, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, errors.New("invalid regex pattern")
+	}
+	return re, nil
+}