lollms-client 1.5.9__tar.gz → 1.6.0__tar.gz

{lollms_client-1.5.9/src/lollms_client.egg-info → lollms_client-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lollms_client
-Version: 1.5.9
+Version: 1.6.0
 Summary: A client library for LoLLMs generate endpoint
 Author-email: ParisNeo <parisneoai@gmail.com>
 License:                                  Apache License

{lollms_client-1.5.9 → lollms_client-1.6.0}/src/lollms_client/__init__.py

@@ -8,7 +8,7 @@ from lollms_client.lollms_utilities import PromptReshaper # Keep general utiliti
 from lollms_client.lollms_mcp_binding import LollmsMCPBinding, LollmsMCPBindingManager
 from lollms_client.lollms_llm_binding import LollmsLLMBindingManager
 
-__version__ = "1.5.9" # Updated version
+__version__ = "1.6.0" # Updated version
 
 # Optionally, you could define __all__ if you want to be explicit about exports
 __all__ = [

{lollms_client-1.5.9 → lollms_client-1.6.0}/src/lollms_client/lollms_core.py

@@ -5732,357 +5732,317 @@ Provide the final aggregated answer in {output_format} format, directly addressi
         callback("Deep analysis complete.", MSG_TYPE.MSG_TYPE_STEP_END)
         return final_output
 
-    def long_context_processing(
-        self,
-        text_to_process: str,
-        contextual_prompt: Optional[str] = None,
-        system_prompt: str= None,
-        context_fill_percentage: float = 0.75,
-        overlap_tokens: int = 0,
-        expected_generation_tokens: int = 1500,
-        streaming_callback: Optional[Callable] = None,
-        return_scratchpad_only: bool = False,
-        debug: bool = True,
-        **kwargs
-    ) -> str:
-        """Enhanced long context processing with Moby Dick literary analysis optimization."""
+def long_context_processing(
+    self,
+    text_to_process: str,
+    contextual_prompt: Optional[str] = None,
+    system_prompt: str | None = None,
+    context_fill_percentage: float = 0.75,
+    overlap_tokens: int = 150,  # Added a default for better context continuity
+    expected_generation_tokens: int = 1500,
+    streaming_callback: Optional[Callable] = None,
+    return_scratchpad_only: bool = False,
+    debug: bool = True,
+    **kwargs
+) -> str:
+    """
+    Processes long text by breaking it down into chunks, analyzing each one incrementally,
+    and synthesizing the results into a comprehensive final response based on a user-defined objective.
+    """
+
+    if debug:
+        print(f"\n🔧 DEBUG: Starting processing with {len(text_to_process):,} characters")
+
+    # Validate context fill percentage
+    if not (0.1 <= context_fill_percentage <= 0.9):
+        raise ValueError(f"context_fill_percentage must be between 0.1 and 0.9, got {context_fill_percentage}")
+
+    # Get context size
+    try:
+        context_size = self.llm.get_context_size() or 8192 # Using a more modern default
+    except:
+        context_size = 8192
+
+    if debug:
+        print(f"🔧 DEBUG: Context size: {context_size}, Fill %: {context_fill_percentage}")
+
+    # Handle empty input
+    if not text_to_process:
+        return ""
+
+    # Use a simple word-based split for token estimation
+    tokens = text_to_process.split()
+    if debug:
+        print(f"🔧 DEBUG: Tokenized into {len(tokens):,} word tokens")
+
+    # Dynamic token budget calculation
+    def calculate_token_budgets(scratchpad_content: str = "", step_num: int = 0) -> dict:
+        # Generic prompt templates are more concise
+        base_system_tokens = 150
+        user_template_tokens = 250
+        scratchpad_tokens = len(scratchpad_content.split()) * 1.3 if scratchpad_content else 0
+
+        used_tokens = base_system_tokens + user_template_tokens + scratchpad_tokens + expected_generation_tokens
+        total_budget = int(context_size * context_fill_percentage)
+        available_for_chunk = max(500, int(total_budget - used_tokens)) # Ensure a reasonable minimum chunk size
+
+        budget_info = {
+            "total_budget": total_budget,
+            "chunk_budget": available_for_chunk,
+            "efficiency_ratio": available_for_chunk / total_budget if total_budget > 0 else 0,
+            "scratchpad_tokens": int(scratchpad_tokens),
+            "used_tokens": int(used_tokens)
+        }
 
         if debug:
-            print(f"\n🔧 DEBUG: Starting processing with {len(text_to_process):,} characters")
+            print(f"🔧 DEBUG Step {step_num}: Budget = {available_for_chunk}/{total_budget} tokens, "
+                  f"Scratchpad = {int(scratchpad_tokens)} tokens")
 
-        # Validate context fill percentage
-        if not (0.1 <= context_fill_percentage <= 0.9):
-            raise ValueError(f"context_fill_percentage must be between 0.1 and 0.9, got {context_fill_percentage}")
+        return budget_info
 
-        # Get context size
-        try:
-            context_size = self.llm.get_context_size() or 4096
-        except:
-            context_size = 4096
+    # Initial budget calculation
+    initial_budget = calculate_token_budgets()
+    chunk_size_tokens = initial_budget["chunk_budget"]
 
-        if debug:
-            print(f"🔧 DEBUG: Context size: {context_size}, Fill %: {context_fill_percentage}")
+    if debug:
+        print(f"🔧 DEBUG: Initial chunk size: {chunk_size_tokens} word tokens")
 
-        # Handle empty input
-        if not text_to_process:
-            return ""
+    if streaming_callback:
+        streaming_callback(
+            f"Context Budget: {initial_budget['chunk_budget']:,}/{initial_budget['total_budget']:,} tokens "
+            f"({initial_budget['efficiency_ratio']:.1%} efficiency)",
+            MSG_TYPE.MSG_TYPE_STEP,
+            {"budget_info": initial_budget}
+        )
 
-        # Use word-based tokenization
-        tokens = text_to_process.split()
+    # Single pass for short content
+    if len(tokens) <= chunk_size_tokens:
         if debug:
-            print(f"🔧 DEBUG: Tokenized into {len(tokens):,} word tokens")
-
-        # Dynamic token budget calculation
-        def calculate_token_budgets(scratchpad_content: str = "", step_num: int = 0) -> dict:
-            base_system_tokens = 250  # Increased for literary-specific prompts
-            user_template_tokens = 300  # Increased for detailed instructions
-            scratchpad_tokens = len(scratchpad_content.split()) * 1.3 if scratchpad_content else 0
-
-            used_tokens = base_system_tokens + user_template_tokens + scratchpad_tokens + expected_generation_tokens
-            total_budget = int(context_size * context_fill_percentage)
-            available_for_chunk = max(400, int(total_budget - used_tokens))  # Increased minimum for better context
-
-            budget_info = {
-                "total_budget": total_budget,
-                "chunk_budget": available_for_chunk,
-                "efficiency_ratio": available_for_chunk / total_budget,
-                "scratchpad_tokens": int(scratchpad_tokens),
-                "used_tokens": int(used_tokens)
-            }
+            print("🔧 DEBUG: Content is short enough for single-pass processing")
 
+        if streaming_callback:
+            streaming_callback("Content fits in a single pass", MSG_TYPE.MSG_TYPE_STEP, {})
+
+        # Generic single-pass system prompt
+        system_prompt = (
+            "You are an expert AI assistant for text analysis and summarization. "
+            "Your task is to carefully analyze the provided text and generate a comprehensive, "
+            "accurate, and well-structured response that directly addresses the user's objective. "
+            "Focus on extracting key information, identifying main themes, and synthesizing the content effectively."
+        )
+
+        prompt_objective = contextual_prompt or "Provide a comprehensive summary and analysis of the provided text."
+        final_prompt = f"Objective: {prompt_objective}\n\n--- Full Text Content ---\n{text_to_process}"
+
+        try:
+            result = self.remove_thinking_blocks(self.llm.generate_text(final_prompt, system_prompt=system_prompt, **kwargs))
             if debug:
-                print(f"🔧 DEBUG Step {step_num}: Budget = {available_for_chunk}/{total_budget} tokens, "
-                      f"Scratchpad = {int(scratchpad_tokens)} tokens")
+                print(f"🔧 DEBUG: Single-pass result: {len(result):,} characters")
+            return result
+        except Exception as e:
+            if debug:
+                print(f"🔧 DEBUG: Single-pass processing failed: {e}")
+            return f"Error in single-pass processing: {e}"
+
+    # Multi-chunk processing for long content
+    if debug:
+        print("🔧 DEBUG: Using multi-chunk processing for long content")
 
-            return budget_info
+    chunk_summaries = []
+    current_position = 0
+    step_number = 1
 
-        # Initial budget calculation
-        initial_budget = calculate_token_budgets()
-        chunk_size_tokens = initial_budget["chunk_budget"]
+    while current_position < len(tokens):
+        # Recalculate budget for each step for dynamic adaptation
+        current_scratchpad = "\n\n---\n\n".join(chunk_summaries)
+        current_budget = calculate_token_budgets(current_scratchpad, step_number)
+        adaptive_chunk_size = max(500, current_budget["chunk_budget"])
+
+        # Extract the next chunk of text
+        chunk_end = min(current_position + adaptive_chunk_size, len(tokens))
+        chunk_tokens = tokens[current_position:chunk_end]
+        chunk_text = " ".join(chunk_tokens)
 
         if debug:
-            print(f"🔧 DEBUG: Initial chunk size: {chunk_size_tokens} word tokens")
+            print(f"\n🔧 DEBUG Step {step_number}: Processing chunk from {current_position} to {chunk_end} "
+                  f"({len(chunk_tokens)} tokens)")
+
+        # Progress calculation
+        remaining_tokens = len(tokens) - current_position
+        estimated_remaining_steps = max(1, -(-remaining_tokens // adaptive_chunk_size)) # Ceiling division
+        total_estimated_steps = step_number + estimated_remaining_steps -1
+        progress = (current_position / len(tokens)) * 90 if len(tokens) > 0 else 0
 
         if streaming_callback:
             streaming_callback(
-                f"Context Budget: {initial_budget['chunk_budget']:,}/{initial_budget['total_budget']:,} tokens "
-                f"({initial_budget['efficiency_ratio']:.1%} efficiency)",
-                MSG_TYPE.MSG_TYPE_STEP,
-                {"budget_info": initial_budget}
+                f"Processing chunk {step_number}/{total_estimated_steps} - "
+                f"Budget: {adaptive_chunk_size:,} tokens",
+                MSG_TYPE.MSG_TYPE_STEP_START,
+                {"step": step_number, "progress": progress}
             )
 
-        # Single pass for short content
-        if len(tokens) <= chunk_size_tokens:
-            if debug:
-                print("🔧 DEBUG: Using single-pass processing")
-
-            if streaming_callback:
-                streaming_callback("Content fits in single pass", MSG_TYPE.MSG_TYPE_STEP, {})
-
-            # FIXED: Moby Dick-specific single-pass system prompt
+        try:
+            # Generic, state-aware system prompt
             system_prompt = (
-                "You are a literary analysis expert specializing in Herman Melville's works. "
-                "Analyze the provided Moby Dick text with deep understanding that Melville's "
-                "detailed realism, technical descriptions, and cultural documentation are "
-                "integral literary techniques. Focus on themes, narrative methods, symbolism, "
-                "and cultural commentary."
+                f"You are a component in a multi-step text processing pipeline. Your role is to analyze a chunk of text and extract key information relevant to a global objective.\n\n"
+                f"**Current Status:** You are on step {step_number} of approximately {total_estimated_steps} steps. Progress is at {progress:.1f}%.\n\n"
+                f"**Your Task:**\n"
+                f"Analyze the 'New Text Chunk' provided below. Extract and summarize any information, data points, or key ideas that are relevant to the 'Global Objective'.\n"
+                f"Review the 'Existing Scratchpad Content' to understand what has already been found. Your goal is to add *new* insights that are not already captured.\n\n"
+                f"**CRITICAL:** Do NOT repeat information already present in the scratchpad. Focus only on new, relevant details from the current chunk. If the chunk contains no new relevant information, respond with '[No new information found in this chunk.]'."
             )
 
-            prompt_objective = contextual_prompt or "Provide comprehensive Moby Dick literary analysis."
-            final_prompt = f"{prompt_objective}\n\n--- Moby Dick Content ---\n{text_to_process}"
-
-            try:
-                result = self.remove_thinking_blocks(self.llm.generate_text(final_prompt, system_prompt=system_prompt, **kwargs))
-                if debug:
-                    print(f"🔧 DEBUG: Single-pass result: {len(result):,} characters")
-                return result
-            except Exception as e:
-                if debug:
-                    print(f"🔧 DEBUG: Single-pass failed: {e}")
-                return f"Error in single-pass processing: {e}"
-
-        # Multi-chunk processing with FIXED prompts
-        if debug:
-            print("🔧 DEBUG: Using multi-chunk processing with Moby Dick-optimized prompts")
-
-        chunk_summaries = []
-        current_position = 0
-        step_number = 1
+            # Generic, context-aware user prompt
+            summarization_objective = contextual_prompt or "Create a comprehensive summary by extracting all key facts, concepts, and conclusions from the text."
+            scratchpad_status = "The analysis is just beginning; this is the first chunk." if not chunk_summaries else f"Building on existing analysis with {len(chunk_summaries)} sections already completed."
+
+            user_prompt = (
+                f"--- Global Objective ---\n{summarization_objective}\n\n"
+                f"--- Current Progress ---\n"
+                f"{scratchpad_status} (Step {step_number}/{total_estimated_steps})\n\n"
+                f"--- Existing Scratchpad Content (for context) ---\n{current_scratchpad}\n\n"
+                f"--- New Text Chunk to Analyze ---\n{chunk_text}\n\n"
+                f"--- Your Instructions ---\n"
+                f"Extract key information from the 'New Text Chunk' that aligns with the 'Global Objective'. "
+                f"Provide a concise summary of the new findings. Do not repeat what is already in the scratchpad. "
+                f"If no new relevant information is found, state that clearly."
+            )
 
-        while current_position < len(tokens):
-            # Recalculate budget
-            current_scratchpad = "\n\n---\n\n".join(chunk_summaries) if chunk_summaries else "[Empty]"
-            current_budget = calculate_token_budgets(current_scratchpad, step_number)
-            adaptive_chunk_size = max(400, current_budget["chunk_budget"])  # Increased minimum
+            if debug:
+                print(f"🔧 DEBUG: Sending {len(user_prompt)} char prompt to LLM")
 
-            # Extract chunk
-            chunk_end = min(current_position + adaptive_chunk_size, len(tokens))
-            chunk_tokens = tokens[current_position:chunk_end]
-            chunk_text = " ".join(chunk_tokens)
+            chunk_summary = self.remove_thinking_blocks(self.llm.generate_text(user_prompt, system_prompt=system_prompt, **kwargs))
 
             if debug:
-                print(f"\n🔧 DEBUG Step {step_number}: Processing chunk {current_position}:{chunk_end} "
-                      f"({len(chunk_tokens)} tokens, {len(chunk_text)} chars)")
-
-            # Progress calculation
-            remaining_tokens = len(tokens) - current_position
-            estimated_remaining_steps = max(1, remaining_tokens // adaptive_chunk_size)
-            total_estimated_steps = step_number + estimated_remaining_steps - 1
-            progress = (current_position / len(tokens)) * 90
+                print(f"🔧 DEBUG: Received {len(chunk_summary)} char response preview: {chunk_summary[:200]}...")
+
+            # Generic content filtering
+            filter_out = False
+            filter_reason = "content accepted"
+
+            # Check for explicit rejection signals
+            if (chunk_summary.strip().lower().startswith('[no new') or
+                chunk_summary.strip().lower().startswith('no new information')):
+                filter_out = True
+                filter_reason = "explicit rejection signal"
+            # Check for overly short or generic refusal responses
+            elif len(chunk_summary.strip()) < 25:
+                filter_out = True
+                filter_reason = "response too short to be useful"
+            # Check for common error phrases
+            elif any(error_phrase in chunk_summary.lower()[:150] for error_phrase in [
+                'error', 'failed', 'cannot provide', 'unable to analyze', 'not possible', 'insufficient information']):
+                filter_out = True
+                filter_reason = "error or refusal response detected"
+
+            if not filter_out:
+                chunk_summaries.append(chunk_summary.strip())
+                content_added = True
+                if debug:
+                    print(f"🔧 DEBUG: ✅ Content added to scratchpad (total sections: {len(chunk_summaries)})")
+            else:
+                content_added = False
+                if debug:
+                    print(f"🔧 DEBUG: ❌ Content filtered out - {filter_reason}: {chunk_summary[:100]}...")
 
+            # Update progress via callback
             if streaming_callback:
+                updated_scratchpad = "\n\n---\n\n".join(chunk_summaries)
                 streaming_callback(
-                    f"Processing chunk {step_number}/{total_estimated_steps} - "
-                    f"Budget: {adaptive_chunk_size:,} tokens",
-                    MSG_TYPE.MSG_TYPE_STEP_START,
-                    {"step": step_number, "progress": progress}
-                )
-
-            try:
-                # FIXED: Moby Dick-specific system prompt that prevents false filtering
-                system_prompt = (
-                    f"You are analyzing Herman Melville's \"Moby Dick\" - a complex literary work where EVERY passage contains literary value.\n\n"
-                    f"**Critical Understanding:**\n"
-                    f"- Melville's detailed descriptions of whaling culture ARE literary techniques\n"
-                    f"- Technical passages reveal themes about knowledge, obsession, and human industry\n"
-                    f"- Social customs and maritime protocols reflect broader themes of hierarchy and civilization\n"
-                    f"- Even seemingly mundane details contribute to Melville's encyclopedic narrative style\n\n"
-                    f"**Current Status:** Step {step_number} of ~{total_estimated_steps} | Progress: {progress:.1f}%\n\n"
-                    f"**Your Task:**\n"
-                    f"Extract literary insights from this text chunk, focusing on:\n"
-                    f"1. **Themes** (obsession, knowledge, nature vs civilization, social hierarchy)\n"
-                    f"2. **Narrative Technique** (Melville's encyclopedic style, detailed realism)\n"
-                    f"3. **Cultural Commentary** (maritime society, American industry, social structures)\n"
-                    f"4. **Character Insights** (authority, dignity, social roles)\n"
-                    f"5. **Symbolic Elements** (ships, sea, whaling practices as metaphors)\n\n"
-                    f"**CRITICAL:** The scratchpad shows '{current_scratchpad[:20]}...' - if it shows '[Empty]', you are analyzing early content and everything you find is 'new' information. "
-                    f"Do NOT say '[No new information]' unless the chunk is literally empty or corrupted.\n\n"
-                    f"Be specific and extract concrete insights. Melville's detailed realism IS his literary technique."
+                    updated_scratchpad,
+                    MSG_TYPE.MSG_TYPE_SCRATCHPAD,
+                    {"step": step_number, "sections": len(chunk_summaries), "content_added": content_added, "filter_reason": filter_reason}
                 )
-
-                # FIXED: Moby Dick-specific user prompt with clear instructions
-                summarization_objective = contextual_prompt or "Create comprehensive literary analysis of Moby-Dick focusing on themes, character development, narrative techniques, and symbolism"
-
-                # Determine scratchpad status for better context
-                scratchpad_status = "The analysis is just beginning - this is among the first substantial content to be processed." if current_scratchpad == "[Empty]" else f"Building on existing analysis with {len(chunk_summaries)} sections already completed."
-
-                user_prompt = (
-                    f"--- Global Objective ---\n{summarization_objective}\n\n"
-                    f"--- Current Progress ---\n"
-                    f"Step {step_number} of ~{total_estimated_steps} | Progress: {progress:.1f}% | Token Budget: {adaptive_chunk_size:,}\n\n"
-                    f"--- Current Analysis State ---\n{scratchpad_status}\n\n"
-                    f"--- Existing Scratchpad Content ---\n{current_scratchpad}\n\n"
-                    f"--- New Text Chunk from Moby Dick ---\n{chunk_text}\n\n"
-                    f"--- Analysis Instructions ---\n"
-                    f"This is Melville's \"Moby Dick\" - extract literary insights from this passage. Consider:\n\n"
-                    f"• **What themes** does this passage develop? (obsession with knowledge, social hierarchy, maritime culture)\n"
-                    f"• **What narrative techniques** does Melville use? (detailed realism, encyclopedic style, technical precision)\n"
-                    f"• **What cultural commentary** is present? (whaling society, American industry, social protocols)\n"
-                    f"• **What character insights** emerge? (authority, dignity, social roles and expectations)\n"
-                    f"• **What symbolic elements** appear? (ships, maritime customs, hierarchical structures)\n\n"
-                    f"**Remember:** In Moby Dick, even technical descriptions serve literary purposes. Melville's detailed realism and cultural documentation ARE his narrative techniques.\n\n"
-                    f"Provide specific, concrete analysis with examples from the text. Extract insights that are not already captured in the scratchpad above."
+                progress_after = ((current_position + len(chunk_tokens)) / len(tokens)) * 90 if len(tokens) > 0 else 90
+                streaming_callback(
+                    f"Step {step_number} completed - {'Content added' if content_added else f'Filtered: {filter_reason}'}",
+                    MSG_TYPE.MSG_TYPE_STEP_END,
+                    {"progress": progress_after}
                 )
 
-                if debug:
-                    print(f"🔧 DEBUG: Sending {len(user_prompt)} char prompt to LLM")
-                    print(f"🔧 DEBUG: Scratchpad status: {scratchpad_status}")
-
-                chunk_summary = self.remove_thinking_blocks(self.llm.generate_text(user_prompt, system_prompt=system_prompt, **kwargs))
-
-                if debug:
-                    print(f"🔧 DEBUG: Received {len(chunk_summary)} char response")
-                    print(f"🔧 DEBUG: Response preview: {chunk_summary[:200]}...")
-
-                # FIXED: More intelligent content filtering specifically for literary analysis
-                filter_out = False
-
-                # Check for explicit rejection signals
-                if (chunk_summary.strip().startswith('[No new insights]') or
-                    chunk_summary.strip().startswith('[No new information]') or 
-                    chunk_summary.strip().startswith('[No significant') or
-                    'cannot provide' in chunk_summary.lower()[:100] or
-                    'unable to analyze' in chunk_summary.lower()[:100]):
-                    filter_out = True
-                    filter_reason = "explicit rejection signal"
-
-                # Check for too short responses
-                elif len(chunk_summary.strip()) < 50:
-                    filter_out = True  
-                    filter_reason = "response too short"
-
-                # Check for error responses
-                elif any(error_phrase in chunk_summary.lower()[:150] for error_phrase in [
-                    'error', 'failed', 'cannot', 'unable', 'not possible', 'insufficient']):
-                    filter_out = True
-                    filter_reason = "error response detected"
-
-                else:
-                    filter_reason = "content accepted"
-
-                if not filter_out:
-                    chunk_summaries.append(chunk_summary.strip())
-                    content_added = True
-                    if debug:
-                        print(f"🔧 DEBUG: ✅ Content added to scratchpad (total sections: {len(chunk_summaries)})")
-                else:
-                    content_added = False
-                    if debug:
-                        print(f"🔧 DEBUG: ❌ Content filtered out - {filter_reason}: {chunk_summary[:100]}...")
-
-                # Update progress
-                if streaming_callback:
-                    updated_scratchpad = "\n\n---\n\n".join(chunk_summaries)
-                    streaming_callback(
-                        updated_scratchpad,
-                        MSG_TYPE.MSG_TYPE_SCRATCHPAD,
-                        {
-                            "step": step_number,
-                            "sections": len(chunk_summaries),
-                            "content_added": content_added,
-                            "filter_reason": filter_reason
-                        }
-                    )
-
-                progress_after = ((current_position + len(chunk_tokens)) / len(tokens)) * 90
-                if streaming_callback:
-                    streaming_callback(
-                        f"Step {step_number} completed - {'Content added' if content_added else f'Filtered: {filter_reason}'}",
-                        MSG_TYPE.MSG_TYPE_STEP_END,
-                        {"progress": progress_after}
-                    )
-
-            except Exception as e:
-                error_msg = f"Step {step_number} failed: {str(e)}"
-                if debug:
-                    print(f"🔧 DEBUG: ❌ {error_msg}")
-                self.trace_exception(e)
-                if streaming_callback:
-                    streaming_callback(error_msg, MSG_TYPE.MSG_TYPE_EXCEPTION)
-                chunk_summaries.append(f"[Error in step {step_number}: {str(e)[:100]}]")
-
-            # Move to next chunk
-            current_position += max(1, adaptive_chunk_size - overlap_tokens)
-            step_number += 1
-
-            # Safety break
-            if step_number > 50:
-                if debug:
-                    print(f"🔧 DEBUG: Breaking after {step_number-1} steps for safety")
-                break
-
-        if debug:
-            print(f"\n🔧 DEBUG: Completed chunking. Total sections: {len(chunk_summaries)}")
-
-        # Return scratchpad if requested
-        if return_scratchpad_only:
-            final_scratchpad = "\n\n---\n\n".join(chunk_summaries)
+        except Exception as e:
+            error_msg = f"Step {step_number} failed: {str(e)}"
+            if debug:
+                print(f"🔧 DEBUG: ❌ {error_msg}")
+            self.trace_exception(e)
             if streaming_callback:
-                streaming_callback("Returning scratchpad only", MSG_TYPE.MSG_TYPE_STEP, {})
-            return final_scratchpad.strip()
+                streaming_callback(error_msg, MSG_TYPE.MSG_TYPE_EXCEPTION)
+            chunk_summaries.append(f"[Error processing chunk at step {step_number}: {str(e)[:150]}]")
 
-        # Final synthesis
+        # Move to the next chunk, allowing for overlap
+        current_position += max(1, adaptive_chunk_size - overlap_tokens)
+        step_number += 1
+        
+        # Safety break for excessively long documents
+        if step_number > 200:
+            if debug: print(f"🔧 DEBUG: Safety break after {step_number-1} steps.")
+            chunk_summaries.append("[Processing halted due to exceeding maximum step limit.]")
+            break
+
+    if debug:
+        print(f"\n🔧 DEBUG: Chunk processing complete. Total sections gathered: {len(chunk_summaries)}")
+
+    # Return only the scratchpad content if requested
+    if return_scratchpad_only:
+        final_scratchpad = "\n\n---\n\n".join(chunk_summaries)
         if streaming_callback:
-            streaming_callback("Synthesizing final comprehensive analysis...", MSG_TYPE.MSG_TYPE_STEP_START, {"progress": 90})
-
-        if not chunk_summaries:
-            error_msg = "No content was successfully processed. The text may not contain recognizable literary elements, or there may be an issue with the processing."
-            if debug:
-                print(f"🔧 DEBUG: ❌ {error_msg}")
-            return error_msg
+            streaming_callback("Returning scratchpad content as final output.", MSG_TYPE.MSG_TYPE_STEP, {})
+        return final_scratchpad.strip()
 
-        combined_scratchpad = "\n\n---\n\n".join(chunk_summaries)
-        synthesis_objective = contextual_prompt or "Create comprehensive literary analysis of Moby-Dick."
+    # Final Synthesis Step
+    if streaming_callback:
+        streaming_callback("Synthesizing final comprehensive response...", MSG_TYPE.MSG_TYPE_STEP_START, {"progress": 95})
 
+    if not chunk_summaries:
+        error_msg = "No content was successfully processed or extracted from the document. The input might be empty or an issue occurred during processing."
         if debug:
-            print(f"🔧 DEBUG: Synthesizing from {len(combined_scratchpad):,} char scratchpad with {len(chunk_summaries)} sections")
-
-        # FIXED: Moby Dick-specific synthesis prompts
-        synthesis_system_prompt = (
-            "You are a literary analysis expert creating a final comprehensive analysis of Herman Melville's Moby Dick.\n"
-            "Synthesize all the insights from the analysis sections into a coherent, scholarly response.\n"
-            "Create clear sections with markdown headers, eliminate redundancy, and provide a thorough analysis.\n"
-            "Focus on Melville's major themes, narrative techniques, cultural commentary, and symbolic elements.\n"
-            "Use specific examples from the text and maintain academic rigor throughout."
-        )
+            print(f"🔧 DEBUG: ❌ {error_msg}")
+        return error_msg
+
+    combined_scratchpad = "\n\n---\n\n".join(chunk_summaries)
+    synthesis_objective = contextual_prompt or "Provide a comprehensive, well-structured summary and analysis of the provided text."
+
+    if debug:
+        print(f"🔧 DEBUG: Synthesizing from {len(combined_scratchpad):,} char scratchpad with {len(chunk_summaries)} sections.")
+
+    # Generic synthesis prompts
+    synthesis_system_prompt = (
+        "You are an expert AI assistant specializing in synthesizing information. "
+        "Your task is to consolidate a series of text analysis sections from a scratchpad into a single, coherent, and well-structured final response. "
+        "Eliminate redundancy, organize the content logically, and ensure the final output directly and comprehensively addresses the user's primary objective. "
+        "Use markdown for clear formatting (e.g., headers, lists, bold text)."
+    )
+
+    synthesis_user_prompt = (
+        f"--- Final Objective ---\n{synthesis_objective}\n\n"
+        f"--- Collected Analysis Sections (Scratchpad) ---\n{combined_scratchpad}\n\n"
+        f"--- Your Final Task ---\n"
+        f"Synthesize all the information from the 'Collected Analysis Sections' into a single, high-quality, and comprehensive response. "
+        f"Your response must directly address the 'Final Objective'. "
+        f"Organize your answer logically with clear sections using markdown headers. "
+        f"Ensure all key information is included, remove any repetitive statements, and produce a polished, final document."
+    )
+
+    try:
+        final_answer = self.remove_thinking_blocks(self.llm.generate_text(synthesis_user_prompt, system_prompt=synthesis_system_prompt, **kwargs))
+        if debug:
+            print(f"🔧 DEBUG: Final synthesis generated: {len(final_answer):,} characters")
+        if streaming_callback:
+            streaming_callback("Final synthesis complete.", MSG_TYPE.MSG_TYPE_STEP_END, {"progress": 100})
+        return final_answer.strip()
 
-        synthesis_user_prompt = (
-            f"--- Analysis Objective ---\n{synthesis_objective}\n\n"
-            f"--- Processing Summary ---\n"
-            f"Successfully analyzed {len(chunk_summaries)} sections of Moby Dick through incremental literary analysis.\n"
-            f"Total scratchpad content: {len(combined_scratchpad):,} characters of literary insights.\n\n"
-            f"--- Collected Literary Analysis Sections ---\n{combined_scratchpad}\n\n"
-            f"--- Final Synthesis Task ---\n"
-            f"Create a comprehensive, well-structured literary analysis of Moby Dick using ALL the insights above. "
-            f"Organize into clear sections with markdown headers (## Theme Analysis, ## Narrative Techniques, ## Cultural Commentary, ## Symbolism, etc.). "
-            f"Eliminate redundancy and create a coherent, scholarly analysis that demonstrates understanding of Melville's complex literary achievement. "
-            f"Include specific textual examples and maintain academic depth throughout."
+    except Exception as e:
+        error_msg = f"The final synthesis step failed: {str(e)}. Returning the organized scratchpad content as a fallback."
+        if debug: print(f"🔧 DEBUG: ❌ {error_msg}")
+        
+        # Fallback to returning the organized scratchpad
+        organized_scratchpad = (
+            f"# Analysis Summary\n\n"
+            f"*Note: The final synthesis process encountered an error. The raw, organized analysis sections are provided below.*\n\n"
+            f"## Collected Sections\n\n"
+            f"{combined_scratchpad}"
         )
-
-        try:
-            final_answer = self.remove_thinking_blocks(self.llm.generate_text(synthesis_user_prompt, system_prompt=synthesis_system_prompt, **kwargs)) 
-
-            if debug:
-                print(f"🔧 DEBUG: Final analysis: {len(final_answer):,} characters")
-
-            if streaming_callback:
-                streaming_callback(f"Final synthesis completed - {len(final_answer):,} characters generated", MSG_TYPE.MSG_TYPE_STEP_END, {"progress": 100})
-
-            return final_answer.strip()
-
-        except Exception as e:
-            error_msg = f"Synthesis failed: {str(e)}. Returning organized scratchpad content."
-            if debug:
-                print(f"🔧 DEBUG: ❌ {error_msg}")
-
-            # Return organized scratchpad as fallback
-            organized_scratchpad = (
-                f"# Literary Analysis of Moby Dick\n\n"
-                f"*Note: Synthesis process encountered issues, presenting organized analysis sections:*\n\n"
-                f"## Analysis Sections\n\n"
-                f"{combined_scratchpad}"
-            )
-            return organized_scratchpad
+        return organized_scratchpad
         
 
 def chunk_text(text, tokenizer, detokenizer, chunk_size, overlap, use_separators=True):

{lollms_client-1.5.9 → lollms_client-1.6.0/src/lollms_client.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lollms_client
-Version: 1.5.9
+Version: 1.6.0
 Summary: A client library for LoLLMs generate endpoint
 Author-email: ParisNeo <parisneoai@gmail.com>
 License:                                  Apache License