recursive-llm-ts 4.8.0 → 5.0.0

package/go/rlm/meta_agent.go

@@ -74,7 +74,15 @@ func (ma *MetaAgent) OptimizeQuery(query string, context string) (string, error)
 		return query, nil
 	}
 
-	optimized := strings.TrimSpace(result)
+	// Track meta-agent token usage in the parent RLM's stats
+	if result.Usage != nil {
+		ma.rlm.stats.PromptTokens += result.Usage.PromptTokens
+		ma.rlm.stats.CompletionTokens += result.Usage.CompletionTokens
+		ma.rlm.stats.TotalTokens += result.Usage.TotalTokens
+	}
+	ma.rlm.stats.LlmCalls++
+
+	optimized := strings.TrimSpace(result.Content)
 	ma.obs.Debug("meta_agent", "Optimized query: %s", truncateStr(optimized, 200))
 	ma.obs.Event("meta_agent.query_optimized", map[string]string{
 		"original_length":  fmt.Sprintf("%d", len(query)),
@@ -136,7 +144,15 @@ func (ma *MetaAgent) OptimizeForStructured(query string, context string, schema
 		return query, nil
 	}
 
-	optimized := strings.TrimSpace(result)
+	// Track meta-agent token usage in the parent RLM's stats
+	if result.Usage != nil {
+		ma.rlm.stats.PromptTokens += result.Usage.PromptTokens
+		ma.rlm.stats.CompletionTokens += result.Usage.CompletionTokens
+		ma.rlm.stats.TotalTokens += result.Usage.TotalTokens
+	}
+	ma.rlm.stats.LlmCalls++
+
+	optimized := strings.TrimSpace(result.Content)
 	ma.obs.Debug("meta_agent", "Optimized structured query: %s", truncateStr(optimized, 200))
 	ma.obs.Event("meta_agent.structured_query_optimized", map[string]string{
 		"original_length":  fmt.Sprintf("%d", len(query)),

package/go/rlm/observability.go

@@ -445,6 +445,12 @@ func FormatStatsWithObservability(stats RLMStats, obs *Observer) map[string]inte
 		result["parsing_retries"] = stats.ParsingRetries
 	}
 
+	if stats.TotalTokens > 0 {
+		result["total_tokens"] = stats.TotalTokens
+		result["prompt_tokens"] = stats.PromptTokens
+		result["completion_tokens"] = stats.CompletionTokens
+	}
+
 	if obs != nil && obs.config.Debug {
 		events := obs.GetEvents()
 		if len(events) > 0 {

package/go/rlm/openai.go

@@ -34,6 +34,20 @@ type chatResponse struct {
 	Error *struct {
 		Message string `json:"message"`
 	} `json:"error"`
+	Usage *TokenUsage `json:"usage,omitempty"`
+}
+
+// TokenUsage represents token consumption from an LLM API response.
+type TokenUsage struct {
+	PromptTokens     int `json:"prompt_tokens"`
+	CompletionTokens int `json:"completion_tokens"`
+	TotalTokens      int `json:"total_tokens"`
+}
+
+// ChatCompletionResult holds the content and token usage from an LLM call.
+type ChatCompletionResult struct {
+	Content string
+	Usage   *TokenUsage
 }
 
 var (
@@ -48,7 +62,7 @@ var (
 	}
 )
 
-func CallChatCompletion(request ChatRequest) (string, error) {
+func CallChatCompletion(request ChatRequest) (ChatCompletionResult, error) {
 	endpoint := buildEndpoint(request.APIBase)
 	payload := map[string]interface{}{
 		"model":    request.Model,
@@ -61,7 +75,7 @@ func CallChatCompletion(request ChatRequest) (string, error) {
 
 	body, err := json.Marshal(payload)
 	if err != nil {
-		return "", err
+		return ChatCompletionResult{}, err
 	}
 
 	// Use shared client with connection pooling
@@ -76,7 +90,7 @@ func CallChatCompletion(request ChatRequest) (string, error) {
 
 	req, err := http.NewRequest(http.MethodPost, endpoint, bytes.NewReader(body))
 	if err != nil {
-		return "", err
+		return ChatCompletionResult{}, err
 	}
 	req.Header.Set("Content-Type", "application/json")
 	if request.APIKey != "" {
@@ -85,7 +99,7 @@ func CallChatCompletion(request ChatRequest) (string, error) {
 
 	resp, err := client.Do(req)
 	if err != nil {
-		return "", err
+		return ChatCompletionResult{}, err
 	}
 	defer func() {
 		_ = resp.Body.Close()
@@ -93,27 +107,30 @@ func CallChatCompletion(request ChatRequest) (string, error) {
 
 	responseBody, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return "", err
+		return ChatCompletionResult{}, err
 	}
 
 	if resp.StatusCode >= http.StatusBadRequest {
-		return "", NewAPIError(resp.StatusCode, strings.TrimSpace(string(responseBody)))
+		return ChatCompletionResult{}, NewAPIError(resp.StatusCode, strings.TrimSpace(string(responseBody)))
 	}
 
 	var parsed chatResponse
 	if err := json.Unmarshal(responseBody, &parsed); err != nil {
-		return "", err
+		return ChatCompletionResult{}, err
 	}
 
 	if parsed.Error != nil && parsed.Error.Message != "" {
-		return "", errors.New(parsed.Error.Message)
+		return ChatCompletionResult{}, errors.New(parsed.Error.Message)
 	}
 
 	if len(parsed.Choices) == 0 {
-		return "", errors.New("no choices returned by LLM")
+		return ChatCompletionResult{}, errors.New("no choices returned by LLM")
 	}
 
-	return parsed.Choices[0].Message.Content, nil
+	return ChatCompletionResult{
+		Content: parsed.Choices[0].Message.Content,
+		Usage:   parsed.Usage,
+	}, nil
 }
 
 func buildEndpoint(apiBase string) string {

package/go/rlm/rlm.go

@@ -21,6 +21,7 @@ type RLM struct {
 	observer         *Observer
 	metaAgent        *MetaAgent
 	contextOverflow  *ContextOverflowConfig
+	lcmEngine        *LCMEngine // Lossless Context Management engine (optional)
 }
 
 func New(model string, config Config) *RLM {
@@ -37,6 +38,9 @@ func New(model string, config Config) *RLM {
 		obs = NewNoopObserver()
 	}
 
+	// Configure tokenizer for accurate token counting with this model
+	SetDefaultTokenizer(model)
+
 	r := &RLM{
 		model:            model,
 		recursiveModel:   recursiveModel,
@@ -52,7 +56,6 @@ func New(model string, config Config) *RLM {
 		stats:            RLMStats{},
 		observer:         obs,
 	}
-
 	// Setup meta-agent if enabled
 	if config.MetaAgent != nil && config.MetaAgent.Enabled {
 		r.metaAgent = NewMetaAgent(r, *config.MetaAgent, obs)
@@ -67,6 +70,19 @@ func New(model string, config Config) *RLM {
 		r.contextOverflow = &defaultConfig
 	}
 
+	// Setup LCM engine if enabled
+	if config.LCM != nil && config.LCM.Enabled {
+		store := NewLCMStore(fmt.Sprintf("session_%d", time.Now().UnixNano()))
+		summarizer := NewLCMSummarizer(model, config.APIBase, config.APIKey, config.TimeoutSeconds, config.ExtraParams, obs)
+		modelLimit := 0
+		if config.ContextOverflow != nil && config.ContextOverflow.MaxModelTokens > 0 {
+			modelLimit = config.ContextOverflow.MaxModelTokens
+		} else {
+			modelLimit = LookupModelTokenLimit(model)
+		}
+		r.lcmEngine = NewLCMEngine(*config.LCM, store, summarizer, obs, modelLimit)
+	}
+
 	return r
 }
 
@@ -100,6 +116,13 @@ func (r *RLM) Completion(query string, context string) (string, RLMStats, error)
 	r.stats.Depth = r.currentDepth
 	replEnv := r.buildREPLEnv(query, context)
 	systemPrompt := BuildSystemPrompt(len(context), r.currentDepth, query, r.useMetacognitive)
+
+	// ─── LCM-managed completion flow ────────────────────────────────────
+	if r.lcmEngine != nil && r.lcmEngine.IsEnabled() {
+		return r.completionWithLCM(query, systemPrompt, replEnv)
+	}
+
+	// ─── Legacy completion flow (no LCM) ────────────────────────────────
 	messages := []Message{
 		{Role: "system", Content: systemPrompt},
 		{Role: "user", Content: query},
@@ -175,6 +198,89 @@ func (r *RLM) Completion(query string, context string) (string, RLMStats, error)
 	return "", r.stats, NewMaxIterationsError(r.maxIterations)
 }
 
+// completionWithLCM runs the completion loop using the LCM engine for context management.
+// Messages flow through the LCM store: persisted verbatim in the immutable store,
+// active context assembled from recent messages + summary nodes, and compaction
+// triggered via the context control loop after each turn.
+func (r *RLM) completionWithLCM(query string, systemPrompt string, replEnv map[string]interface{}) (string, RLMStats, error) {
+	store := r.lcmEngine.GetStore()
+
+	// Persist system prompt and initial query in the immutable store
+	store.PersistMessage(RoleSystem, systemPrompt, nil)
+	store.PersistMessage(RoleUser, query, nil)
+
+	r.observer.Debug("rlm.lcm", "Starting LCM-managed completion, initial tokens: %d",
+		store.ActiveContextTokens())
+
+	for iteration := 0; iteration < r.maxIterations; iteration++ {
+		r.stats.Iterations = iteration + 1
+		r.observer.Debug("rlm.lcm", "Iteration %d/%d at depth %d (active tokens: %d)",
+			iteration+1, r.maxIterations, r.currentDepth, store.ActiveContextTokens())
+
+		// Run the LCM context control loop (may trigger async or blocking compaction)
+		if err := r.lcmEngine.OnNewItem(); err != nil {
+			r.observer.Error("rlm.lcm", "Context control loop error: %v", err)
+			// Non-fatal: continue with current context
+		}
+
+		// Build messages from the active context (includes summaries with IDs)
+		messages := store.BuildMessages()
+
+		response, err := r.callLLM(messages)
+		if err != nil {
+			// Check for context overflow — LCM should handle this via compaction,
+			// but fall back to blocking compaction if the API still rejects
+			if r.contextOverflow != nil && r.contextOverflow.Enabled {
+				if _, isOverflow := IsContextOverflow(err); isOverflow {
+					r.observer.Debug("rlm.lcm", "Context overflow despite LCM, forcing blocking compaction")
+					if compactErr := r.lcmEngine.blockingCompaction(); compactErr != nil {
+						r.observer.Error("rlm.lcm", "Emergency compaction failed: %v", compactErr)
+						return "", r.stats, err
+					}
+					// Also try condensing old summaries to free more space
+					_ = r.lcmEngine.CondenseOldSummaries()
+					iteration-- // Retry
+					continue
+				}
+			}
+			r.observer.Error("rlm.lcm", "LLM call failed on iteration %d: %v", iteration+1, err)
+			return "", r.stats, err
+		}
+
+		// Persist assistant response in the immutable store
+		store.PersistMessage(RoleAssistant, response, nil)
+
+		if IsFinal(response) {
+			answer, ok := ParseResponse(response, replEnv)
+			if ok {
+				r.observer.Debug("rlm.lcm", "FINAL answer on iteration %d (store: %d msgs, %d summaries)",
+					iteration+1, store.MessageCount(), store.Stats().TotalSummaries)
+				r.observer.Event("rlm.lcm.completion_success", map[string]string{
+					"iterations":        fmt.Sprintf("%d", iteration+1),
+					"llm_calls":         fmt.Sprintf("%d", r.stats.LlmCalls),
+					"total_messages":    fmt.Sprintf("%d", store.MessageCount()),
+					"total_summaries":   fmt.Sprintf("%d", store.Stats().TotalSummaries),
+					"compression_ratio": fmt.Sprintf("%.2f", store.Stats().CompressionRatio),
+				})
+				return answer, r.stats, nil
+			}
+		}
+
+		execResult, err := r.repl.Execute(response, replEnv)
+		if err != nil {
+			r.observer.Debug("rlm.lcm", "REPL execution error: %v", err)
+			execResult = fmt.Sprintf("Error: %s", err.Error())
+		} else {
+			r.observer.Debug("rlm.lcm", "REPL output: %s", truncateStr(execResult, 200))
+		}
+
+		// Persist REPL result as user message in the immutable store
+		store.PersistMessage(RoleUser, execResult, nil)
+	}
+
+	return "", r.stats, NewMaxIterationsError(r.maxIterations)
+}
+
 func (r *RLM) callLLM(messages []Message) (string, error) {
 	r.stats.LlmCalls++
 	defaultModel := r.model
@@ -198,14 +304,22 @@ func (r *RLM) callLLM(messages []Message) (string, error) {
 	result, err := CallChatCompletion(request)
 	duration := time.Since(start)
 
-	r.observer.LLMCall(defaultModel, len(messages), 0, duration, err)
+	tokensUsed := 0
+	if result.Usage != nil {
+		r.stats.PromptTokens += result.Usage.PromptTokens
+		r.stats.CompletionTokens += result.Usage.CompletionTokens
+		r.stats.TotalTokens += result.Usage.TotalTokens
+		tokensUsed = result.Usage.TotalTokens
+	}
+
+	r.observer.LLMCall(defaultModel, len(messages), tokensUsed, duration, err)
 
 	if err != nil {
 		return "", err
 	}
 
-	r.observer.Debug("llm", "Response received (%d chars) in %s", len(result), duration)
-	return result, nil
+	r.observer.Debug("llm", "Response received (%d chars, %d tokens) in %s", len(result.Content), tokensUsed, duration)
+	return result.Content, nil
 }
 
 func (r *RLM) buildREPLEnv(query string, context string) map[string]interface{} {
@@ -294,6 +408,23 @@ func (r *RLM) GetObserver() *Observer {
 	return r.observer
 }
 
+// GetLCMEngine returns the LCM engine if enabled, nil otherwise.
+func (r *RLM) GetLCMEngine() *LCMEngine {
+	return r.lcmEngine
+}
+
+// LLMMap executes an LLM-Map operation for parallel batch processing.
+func (r *RLM) LLMMap(config LLMMapConfig) (*LLMMapResult, error) {
+	mapper := NewLLMMapper(r.model, r.apiBase, r.apiKey, r.timeoutSeconds, r.extraParams, r.observer)
+	return mapper.Execute(config)
+}
+
+// AgenticMap executes an Agentic-Map operation with full sub-agent sessions per item.
+func (r *RLM) AgenticMap(config AgenticMapConfig) (*AgenticMapResult, error) {
+	mapper := NewAgenticMapper(r.model, r.apiBase, r.apiKey, r.timeoutSeconds, r.extraParams, r.observer)
+	return mapper.Execute(config)
+}
+
 // Shutdown gracefully shuts down the RLM engine and its observer.
 func (r *RLM) Shutdown() {
 	if r.observer != nil {

package/go/rlm/store_backend.go

@@ -0,0 +1,121 @@
+package rlm
+
+import "io"
+
+// ─── Store Backend Interface ─────────────────────────────────────────────────
+// Abstracts the persistence layer for LCM's dual-state memory.
+// The in-memory implementation (LCMStore) remains the default.
+// A SQLite implementation provides crash recovery, indexed full-text search,
+// and transactional writes as described in the LCM paper (Section 2.1).
+
+// StoreBackend defines the persistence operations for the LCM store.
+// Implementations must be safe for concurrent use.
+type StoreBackend interface {
+	// ─── Message Operations ──────────────────────────────────────────
+	// PersistMessage stores a message in the immutable store.
+	PersistMessage(msg *StoreMessage) error
+	// GetMessage retrieves a message by ID.
+	GetMessage(id string) (*StoreMessage, error)
+	// GetAllMessages returns all messages in chronological order.
+	GetAllMessages() ([]*StoreMessage, error)
+	// MessageCount returns the total number of persisted messages.
+	MessageCount() (int, error)
+
+	// ─── Summary Operations ─────────────────────────────────────────
+	// PersistSummary stores a summary node in the DAG.
+	PersistSummary(node *SummaryNode) error
+	// GetSummary retrieves a summary by ID.
+	GetSummary(id string) (*SummaryNode, error)
+	// GetAllSummaries returns all summary nodes.
+	GetAllSummaries() ([]*SummaryNode, error)
+	// UpdateSummaryParent sets the parent ID on a summary (for condensation).
+	UpdateSummaryParent(summaryID, parentID string) error
+
+	// ─── Search ─────────────────────────────────────────────────────
+	// GrepMessages searches message content with a regex pattern.
+	// Returns matching messages with optional summary scope filtering.
+	GrepMessages(pattern string, summaryScope *string, maxResults int) ([]*StoreMessage, error)
+
+	// ─── Lifecycle ──────────────────────────────────────────────────
+	io.Closer
+}
+
+// ─── In-Memory Backend ───────────────────────────────────────────────────────
+// MemoryBackend wraps the existing in-memory maps as a StoreBackend.
+// This is the default backend and requires no external dependencies.
+
+type MemoryBackend struct {
+	messages   map[string]*StoreMessage
+	messageSeq []*StoreMessage
+	summaries  map[string]*SummaryNode
+}
+
+// NewMemoryBackend creates a new in-memory backend.
+func NewMemoryBackend() *MemoryBackend {
+	return &MemoryBackend{
+		messages:   make(map[string]*StoreMessage),
+		messageSeq: make([]*StoreMessage, 0),
+		summaries:  make(map[string]*SummaryNode),
+	}
+}
+
+func (m *MemoryBackend) PersistMessage(msg *StoreMessage) error {
+	m.messages[msg.ID] = msg
+	m.messageSeq = append(m.messageSeq, msg)
+	return nil
+}
+
+func (m *MemoryBackend) GetMessage(id string) (*StoreMessage, error) {
+	msg, ok := m.messages[id]
+	if !ok {
+		return nil, nil
+	}
+	return msg, nil
+}
+
+func (m *MemoryBackend) GetAllMessages() ([]*StoreMessage, error) {
+	result := make([]*StoreMessage, len(m.messageSeq))
+	copy(result, m.messageSeq)
+	return result, nil
+}
+
+func (m *MemoryBackend) MessageCount() (int, error) {
+	return len(m.messageSeq), nil
+}
+
+func (m *MemoryBackend) PersistSummary(node *SummaryNode) error {
+	m.summaries[node.ID] = node
+	return nil
+}
+
+func (m *MemoryBackend) GetSummary(id string) (*SummaryNode, error) {
+	sum, ok := m.summaries[id]
+	if !ok {
+		return nil, nil
+	}
+	return sum, nil
+}
+
+func (m *MemoryBackend) GetAllSummaries() ([]*SummaryNode, error) {
+	result := make([]*SummaryNode, 0, len(m.summaries))
+	for _, s := range m.summaries {
+		result = append(result, s)
+	}
+	return result, nil
+}
+
+func (m *MemoryBackend) UpdateSummaryParent(summaryID, parentID string) error {
+	if sum, ok := m.summaries[summaryID]; ok {
+		sum.ParentID = parentID
+	}
+	return nil
+}
+
+func (m *MemoryBackend) GrepMessages(pattern string, summaryScope *string, maxResults int) ([]*StoreMessage, error) {
+	// For in-memory, just return all messages (filtering happens in LCMStore.Grep)
+	return m.messageSeq, nil
+}
+
+func (m *MemoryBackend) Close() error {
+	return nil
+}