recursive-llm-ts 4.8.0 → 5.0.0

package/go/rlm/tokenizer.go

@@ -0,0 +1,216 @@
+package rlm
+
+import (
+	"strings"
+	"sync"
+	"sync/atomic"
+
+	"github.com/cespare/xxhash/v2"
+	tiktoken "github.com/pkoukk/tiktoken-go"
+)
+
+// ─── Tokenizer Interface ─────────────────────────────────────────────────────
+// Provides accurate token counting with model-specific BPE encoding.
+// Replaces the heuristic ~3.5 chars/token estimation with real tokenization.
+
+// Tokenizer counts tokens in text strings.
+type Tokenizer interface {
+	CountTokens(text string) int
+}
+
+// ─── Tiktoken BPE Tokenizer ──────────────────────────────────────────────────
+
+// TiktokenTokenizer uses the tiktoken BPE encoding for accurate token counting.
+type TiktokenTokenizer struct {
+	encoding *tiktoken.Tiktoken
+	name     string
+}
+
+// modelEncodingMap maps model name prefixes to their tiktoken encoding names.
+var modelEncodingMap = map[string]string{
+	// OpenAI o200k_base models
+	"gpt-4o":     "o200k_base",
+	"gpt-4o-":    "o200k_base",
+	"o1":         "o200k_base",
+	"o3":         "o200k_base",
+	"o4":         "o200k_base",
+	// OpenAI cl100k_base models
+	"gpt-4-":     "cl100k_base",
+	"gpt-4":      "cl100k_base",
+	"gpt-3.5":    "cl100k_base",
+	// Anthropic (closest approximation)
+	"claude":     "cl100k_base",
+	// Meta Llama
+	"llama":      "cl100k_base",
+	// Mistral
+	"mistral":    "cl100k_base",
+	"mixtral":    "cl100k_base",
+	// Qwen
+	"qwen":       "cl100k_base",
+}
+
+// encodingForModel returns the tiktoken encoding name for a given model.
+func encodingForModel(model string) string {
+	lower := strings.ToLower(model)
+
+	// Try exact match first
+	if enc, ok := modelEncodingMap[lower]; ok {
+		return enc
+	}
+
+	// Try prefix matching (longest prefix wins)
+	bestMatch := ""
+	bestEnc := "cl100k_base"
+	for prefix, enc := range modelEncodingMap {
+		if strings.HasPrefix(lower, prefix) && len(prefix) > len(bestMatch) {
+			bestMatch = prefix
+			bestEnc = enc
+		}
+	}
+
+	return bestEnc
+}
+
+// NewTiktokenTokenizer creates a tokenizer using the appropriate BPE encoding for the model.
+// Returns nil and an error if the encoding cannot be loaded.
+func NewTiktokenTokenizer(model string) (*TiktokenTokenizer, error) {
+	encName := encodingForModel(model)
+	enc, err := tiktoken.GetEncoding(encName)
+	if err != nil {
+		return nil, err
+	}
+	return &TiktokenTokenizer{
+		encoding: enc,
+		name:     encName,
+	}, nil
+}
+
+// CountTokens returns the exact BPE token count for the text.
+func (t *TiktokenTokenizer) CountTokens(text string) int {
+	if len(text) == 0 {
+		return 0
+	}
+	tokens := t.encoding.Encode(text, nil, nil)
+	return len(tokens)
+}
+
+// EncodingName returns the name of the encoding used.
+func (t *TiktokenTokenizer) EncodingName() string {
+	return t.name
+}
+
+// ─── Heuristic Tokenizer (Fallback) ──────────────────────────────────────────
+
+// HeuristicTokenizer uses the character-to-token ratio heuristic.
+// This is the original EstimateTokens logic, kept as a fallback
+// when tiktoken encodings are unavailable.
+type HeuristicTokenizer struct{}
+
+// CountTokens provides a fast approximation of token count.
+// Uses ~3.5 chars/token ratio, intentionally conservative (over-estimates).
+func (h *HeuristicTokenizer) CountTokens(text string) int {
+	if len(text) == 0 {
+		return 0
+	}
+	return (len(text)*10 + 34) / 35
+}
+
+// ─── Cached Tokenizer ────────────────────────────────────────────────────────
+
+const maxCacheSize = 10000
+
+// CachedTokenizer wraps another Tokenizer with an LRU-style cache.
+// Uses xxhash for fast key hashing and sync.Map for concurrent access.
+type CachedTokenizer struct {
+	inner    Tokenizer
+	cache    sync.Map // map[uint64]int
+	size     atomic.Int64
+}
+
+// NewCachedTokenizer wraps a tokenizer with caching.
+func NewCachedTokenizer(inner Tokenizer) *CachedTokenizer {
+	return &CachedTokenizer{
+		inner: inner,
+	}
+}
+
+// CountTokens returns the cached token count, computing and caching on miss.
+func (c *CachedTokenizer) CountTokens(text string) int {
+	if len(text) == 0 {
+		return 0
+	}
+
+	// Hash the text for cache key
+	key := xxhash.Sum64String(text)
+
+	// Check cache
+	if val, ok := c.cache.Load(key); ok {
+		return val.(int)
+	}
+
+	// Compute
+	count := c.inner.CountTokens(text)
+
+	// Store if under max size; evict all if over (simple strategy)
+	if c.size.Load() < maxCacheSize {
+		if _, loaded := c.cache.LoadOrStore(key, count); !loaded {
+			c.size.Add(1)
+		}
+	} else {
+		// Simple eviction: clear the cache when full
+		c.cache.Clear()
+		c.size.Store(0)
+		c.cache.Store(key, count)
+		c.size.Add(1)
+	}
+
+	return count
+}
+
+// CacheSize returns the current number of cached entries.
+func (c *CachedTokenizer) CacheSize() int64 {
+	return c.size.Load()
+}
+
+// Inner returns the underlying tokenizer.
+func (c *CachedTokenizer) Inner() Tokenizer {
+	return c.inner
+}
+
+// ─── Global Default Tokenizer ────────────────────────────────────────────────
+
+var (
+	defaultTokenizer Tokenizer = &HeuristicTokenizer{}
+	tokenizerMu      sync.RWMutex
+)
+
+// SetDefaultTokenizer configures the global tokenizer for a given model.
+// Tries tiktoken BPE first, falls back to heuristic on failure.
+// The tokenizer is wrapped with caching for performance.
+func SetDefaultTokenizer(model string) {
+	tokenizerMu.Lock()
+	defer tokenizerMu.Unlock()
+
+	tok, err := NewTiktokenTokenizer(model)
+	if err != nil {
+		// Fall back to heuristic with caching
+		defaultTokenizer = NewCachedTokenizer(&HeuristicTokenizer{})
+		return
+	}
+
+	defaultTokenizer = NewCachedTokenizer(tok)
+}
+
+// GetTokenizer returns the current global tokenizer.
+func GetTokenizer() Tokenizer {
+	tokenizerMu.RLock()
+	defer tokenizerMu.RUnlock()
+	return defaultTokenizer
+}
+
+// ResetDefaultTokenizer resets to the heuristic tokenizer (used in tests).
+func ResetDefaultTokenizer() {
+	tokenizerMu.Lock()
+	defer tokenizerMu.Unlock()
+	defaultTokenizer = &HeuristicTokenizer{}
+}

package/go/rlm/tokenizer_test.go

@@ -0,0 +1,305 @@
+package rlm
+
+import (
+	"strings"
+	"testing"
+)
+
+// ─── Tiktoken BPE Tokenizer Tests ────────────────────────────────────────────
+
+func TestTiktokenTokenizer_English(t *testing.T) {
+	tok, err := NewTiktokenTokenizer("gpt-4o")
+	if err != nil {
+		t.Fatalf("failed to create tokenizer: %v", err)
+	}
+
+	text := "Hello, world! This is a test of the tokenizer."
+	bpeCount := tok.CountTokens(text)
+	heuristic := (&HeuristicTokenizer{}).CountTokens(text)
+
+	t.Logf("English text: BPE=%d, Heuristic=%d", bpeCount, heuristic)
+
+	if bpeCount <= 0 {
+		t.Error("BPE count should be > 0")
+	}
+	// BPE and heuristic should give different counts for most text
+	// (they may coincidentally match for very short strings, so just verify BPE is reasonable)
+	if bpeCount > len(text) {
+		t.Errorf("BPE count %d should not exceed character count %d", bpeCount, len(text))
+	}
+}
+
+func TestTiktokenTokenizer_Code(t *testing.T) {
+	tok, err := NewTiktokenTokenizer("gpt-4o")
+	if err != nil {
+		t.Fatalf("failed to create tokenizer: %v", err)
+	}
+
+	code := `func main() {
+	fmt.Println("Hello, World!")
+	for i := 0; i < 100; i++ {
+		result = append(result, processItem(items[i]))
+	}
+}`
+	bpeCount := tok.CountTokens(code)
+	heuristic := (&HeuristicTokenizer{}).CountTokens(code)
+
+	t.Logf("Code: BPE=%d, Heuristic=%d (chars=%d)", bpeCount, heuristic, len(code))
+
+	if bpeCount <= 0 {
+		t.Error("BPE count should be > 0 for code")
+	}
+}
+
+func TestTiktokenTokenizer_JSON(t *testing.T) {
+	tok, err := NewTiktokenTokenizer("gpt-4o")
+	if err != nil {
+		t.Fatalf("failed to create tokenizer: %v", err)
+	}
+
+	jsonData := `{"name": "test", "values": [1, 2, 3, 4, 5], "nested": {"key": "value", "count": 42}}`
+	bpeCount := tok.CountTokens(jsonData)
+	heuristic := (&HeuristicTokenizer{}).CountTokens(jsonData)
+
+	t.Logf("JSON: BPE=%d, Heuristic=%d (chars=%d)", bpeCount, heuristic, len(jsonData))
+
+	if bpeCount <= 0 {
+		t.Error("BPE count should be > 0 for JSON")
+	}
+}
+
+func TestTiktokenTokenizer_CJK(t *testing.T) {
+	tok, err := NewTiktokenTokenizer("gpt-4o")
+	if err != nil {
+		t.Fatalf("failed to create tokenizer: %v", err)
+	}
+
+	// Chinese, Japanese, and Korean text
+	cjkText := "这是一个测试。日本語のテスト。한국어 테스트입니다。"
+	bpeCount := tok.CountTokens(cjkText)
+	heuristic := (&HeuristicTokenizer{}).CountTokens(cjkText)
+
+	t.Logf("CJK text: BPE=%d, Heuristic=%d (chars=%d, bytes=%d)", bpeCount, heuristic, len([]rune(cjkText)), len(cjkText))
+
+	if bpeCount <= 0 {
+		t.Error("BPE count should be > 0 for CJK")
+	}
+	// CJK text has ~1.5 chars per token but heuristic assumes ~3.5
+	// So BPE should count MORE tokens than heuristic for CJK
+	if bpeCount <= heuristic {
+		t.Logf("WARNING: BPE (%d) should typically be > heuristic (%d) for CJK text", bpeCount, heuristic)
+	}
+}
+
+func TestTiktokenTokenizer_Empty(t *testing.T) {
+	tok, err := NewTiktokenTokenizer("gpt-4o")
+	if err != nil {
+		t.Fatalf("failed to create tokenizer: %v", err)
+	}
+
+	if tok.CountTokens("") != 0 {
+		t.Error("empty string should return 0 tokens")
+	}
+}
+
+func TestTiktokenTokenizer_EncodingSelection(t *testing.T) {
+	tests := []struct {
+		model    string
+		expected string
+	}{
+		{"gpt-4o", "o200k_base"},
+		{"gpt-4o-mini", "o200k_base"},
+		{"gpt-4o-mini-2024-07-18", "o200k_base"},
+		{"gpt-4", "cl100k_base"},
+		{"gpt-4-turbo", "cl100k_base"},
+		{"gpt-3.5-turbo", "cl100k_base"},
+		{"claude-3-opus", "cl100k_base"},
+		{"claude-sonnet-4", "cl100k_base"},
+		{"o1", "o200k_base"},
+		{"o3-mini", "o200k_base"},
+		{"llama-3.1", "cl100k_base"},
+		{"unknown-model", "cl100k_base"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.model, func(t *testing.T) {
+			enc := encodingForModel(tt.model)
+			if enc != tt.expected {
+				t.Errorf("encodingForModel(%q) = %q, want %q", tt.model, enc, tt.expected)
+			}
+		})
+	}
+}
+
+// ─── Heuristic Tokenizer Tests ───────────────────────────────────────────────
+
+func TestHeuristicTokenizer_Fallback(t *testing.T) {
+	h := &HeuristicTokenizer{}
+
+	if h.CountTokens("") != 0 {
+		t.Error("empty string should be 0")
+	}
+
+	// "Hello" = 5 chars, ceil(5/3.5) = 2
+	count := h.CountTokens("Hello")
+	if count <= 0 {
+		t.Error("should count > 0 tokens for 'Hello'")
+	}
+
+	// Longer text
+	longText := strings.Repeat("word ", 1000)
+	longCount := h.CountTokens(longText)
+	expected := (len(longText)*10 + 34) / 35
+	if longCount != expected {
+		t.Errorf("heuristic count %d != expected %d", longCount, expected)
+	}
+}
+
+// ─── Cached Tokenizer Tests ─────────────────────────────────────────────────
+
+func TestCachedTokenizer_CacheHit(t *testing.T) {
+	callCount := 0
+	inner := &countingTokenizer{fn: func(text string) int {
+		callCount++
+		return len(text) / 4
+	}}
+
+	cached := NewCachedTokenizer(inner)
+
+	text := "This is a test string for caching"
+
+	// First call: cache miss
+	count1 := cached.CountTokens(text)
+	if callCount != 1 {
+		t.Errorf("expected 1 inner call, got %d", callCount)
+	}
+
+	// Second call: cache hit (inner should NOT be called again)
+	count2 := cached.CountTokens(text)
+	if callCount != 1 {
+		t.Errorf("expected still 1 inner call after cache hit, got %d", callCount)
+	}
+
+	if count1 != count2 {
+		t.Errorf("cache returned different values: %d vs %d", count1, count2)
+	}
+
+	if cached.CacheSize() != 1 {
+		t.Errorf("cache size should be 1, got %d", cached.CacheSize())
+	}
+}
+
+func TestCachedTokenizer_Empty(t *testing.T) {
+	inner := &HeuristicTokenizer{}
+	cached := NewCachedTokenizer(inner)
+
+	if cached.CountTokens("") != 0 {
+		t.Error("empty string should return 0 without caching")
+	}
+	if cached.CacheSize() != 0 {
+		t.Error("cache should not store empty strings")
+	}
+}
+
+func TestCachedTokenizer_DifferentStrings(t *testing.T) {
+	inner := &HeuristicTokenizer{}
+	cached := NewCachedTokenizer(inner)
+
+	cached.CountTokens("string one")
+	cached.CountTokens("string two")
+	cached.CountTokens("string three")
+
+	if cached.CacheSize() != 3 {
+		t.Errorf("cache size should be 3, got %d", cached.CacheSize())
+	}
+}
+
+func TestCachedTokenizer_Inner(t *testing.T) {
+	inner := &HeuristicTokenizer{}
+	cached := NewCachedTokenizer(inner)
+
+	if cached.Inner() != inner {
+		t.Error("Inner() should return the wrapped tokenizer")
+	}
+}
+
+// countingTokenizer tracks how many times CountTokens is called.
+type countingTokenizer struct {
+	fn func(string) int
+}
+
+func (c *countingTokenizer) CountTokens(text string) int {
+	return c.fn(text)
+}
+
+// ─── Global Default Tokenizer Tests ──────────────────────────────────────────
+
+func TestSetDefaultTokenizer_KnownModel(t *testing.T) {
+	defer ResetDefaultTokenizer()
+
+	SetDefaultTokenizer("gpt-4o")
+	tok := GetTokenizer()
+
+	// Should be a CachedTokenizer wrapping a TiktokenTokenizer
+	cached, ok := tok.(*CachedTokenizer)
+	if !ok {
+		t.Fatalf("expected CachedTokenizer, got %T", tok)
+	}
+
+	inner, ok := cached.Inner().(*TiktokenTokenizer)
+	if !ok {
+		t.Fatalf("expected inner TiktokenTokenizer, got %T", cached.Inner())
+	}
+
+	if inner.EncodingName() != "o200k_base" {
+		t.Errorf("expected o200k_base encoding, got %s", inner.EncodingName())
+	}
+
+	// Verify it actually counts tokens
+	count := tok.CountTokens("Hello, world!")
+	if count <= 0 {
+		t.Error("tokenizer should count > 0 tokens")
+	}
+}
+
+func TestSetDefaultTokenizer_UnknownModel(t *testing.T) {
+	defer ResetDefaultTokenizer()
+
+	// Even unknown models should work because we default to cl100k_base
+	SetDefaultTokenizer("totally-unknown-model-xyz")
+	tok := GetTokenizer()
+
+	count := tok.CountTokens("Hello, world!")
+	if count <= 0 {
+		t.Error("tokenizer should count > 0 tokens even for unknown model")
+	}
+}
+
+func TestEstimateTokens_UsesDefaultTokenizer(t *testing.T) {
+	defer ResetDefaultTokenizer()
+
+	// With heuristic default
+	heuristicCount := EstimateTokens("Hello, world! This is a test.")
+
+	// Switch to BPE
+	SetDefaultTokenizer("gpt-4o")
+	bpeCount := EstimateTokens("Hello, world! This is a test.")
+
+	t.Logf("EstimateTokens: heuristic=%d, bpe=%d", heuristicCount, bpeCount)
+
+	// Both should be > 0
+	if heuristicCount <= 0 || bpeCount <= 0 {
+		t.Errorf("both counts should be > 0: heuristic=%d, bpe=%d", heuristicCount, bpeCount)
+	}
+}
+
+func TestResetDefaultTokenizer(t *testing.T) {
+	SetDefaultTokenizer("gpt-4o")
+	ResetDefaultTokenizer()
+
+	tok := GetTokenizer()
+	_, ok := tok.(*HeuristicTokenizer)
+	if !ok {
+		t.Errorf("after reset, expected HeuristicTokenizer, got %T", tok)
+	}
+}

package/go/rlm/types.go

@@ -6,10 +6,13 @@ import (
 )
 
 type RLMStats struct {
-	LlmCalls       int `json:"llm_calls"`
-	Iterations     int `json:"iterations"`
-	Depth          int `json:"depth"`
-	ParsingRetries int `json:"parsing_retries,omitempty"`
+	LlmCalls         int `json:"llm_calls"`
+	Iterations       int `json:"iterations"`
+	Depth            int `json:"depth"`
+	ParsingRetries   int `json:"parsing_retries,omitempty"`
+	TotalTokens      int `json:"total_tokens,omitempty"`
+	PromptTokens     int `json:"prompt_tokens,omitempty"`
+	CompletionTokens int `json:"completion_tokens,omitempty"`
 }
 
 type JSONSchema struct {
@@ -67,6 +70,7 @@ type Config struct {
 	MetaAgent         *MetaAgentConfig
 	Observability     *ObservabilityConfig
 	ContextOverflow   *ContextOverflowConfig
+	LCM              *LCMConfig // Lossless Context Management configuration
 }
 
 func ConfigFromMap(config map[string]interface{}) Config {
@@ -123,6 +127,27 @@ func ConfigFromMap(config map[string]interface{}) Config {
 		parsed.ContextOverflow = &co
 	}
 
+	// Extract LCM config
+	if lcmConfig, ok := config["lcm"].(map[string]interface{}); ok {
+		lcm := DefaultLCMConfig()
+		if v, ok := lcmConfig["enabled"].(bool); ok {
+			lcm.Enabled = v
+		}
+		if v, ok := toInt(lcmConfig["soft_threshold"]); ok {
+			lcm.SoftThreshold = v
+		}
+		if v, ok := toInt(lcmConfig["hard_threshold"]); ok {
+			lcm.HardThreshold = v
+		}
+		if v, ok := toInt(lcmConfig["compaction_block_size"]); ok {
+			lcm.CompactionBlockSize = v
+		}
+		if v, ok := toInt(lcmConfig["summary_target_tokens"]); ok {
+			lcm.SummaryTargetTokens = v
+		}
+		parsed.LCM = &lcm
+	}
+
 	for key, value := range config {
 		switch key {
 		case "recursive_model":
@@ -156,7 +181,7 @@ func ConfigFromMap(config map[string]interface{}) Config {
 			"trace_endpoint", "service_name", "log_output",
 			"langfuse_enabled", "langfuse_public_key",
 			"langfuse_secret_key", "langfuse_host",
-			"context_overflow":
+			"context_overflow", "lcm":
 			// ignore bridge-only config, meta_agent, observability, context_overflow (handled above/separately)
 		default:
 			parsed.ExtraParams[key] = value

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "recursive-llm-ts",
-  "version": "4.8.0",
+  "version": "5.0.0",
   "description": "TypeScript bridge for recursive-llm: Recursive Language Models for unbounded context processing with structured outputs",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -48,12 +48,12 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/jbeck018/recursive-llm-ts.git"
+    "url": "git+https://github.com/howlerops/recursive-llm-ts.git"
   },
   "bugs": {
-    "url": "https://github.com/jbeck018/recursive-llm-ts/issues"
+    "url": "https://github.com/howlerops/recursive-llm-ts/issues"
   },
-  "homepage": "https://github.com/jbeck018/recursive-llm-ts#readme",
+  "homepage": "https://github.com/howlerops/recursive-llm-ts#readme",
   "dependencies": {
     "zod": "^4.3.6"
   },