massgen 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

massgen/cli.py

@@ -488,8 +488,18 @@ def create_backend(backend_type: str, **kwargs) -> Any:
         raise ConfigurationError(f"Unsupported backend type: {backend_type}")
 
 
-def create_agents_from_config(config: Dict[str, Any], orchestrator_config: Optional[Dict[str, Any]] = None, config_path: Optional[str] = None) -> Dict[str, ConfigurableAgent]:
-    """Create agents from configuration."""
+def create_agents_from_config(
+    config: Dict[str, Any],
+    orchestrator_config: Optional[Dict[str, Any]] = None,
+    config_path: Optional[str] = None,
+    memory_session_id: Optional[str] = None,
+) -> Dict[str, ConfigurableAgent]:
+    """Create agents from configuration.
+
+    Args:
+        memory_session_id: Optional session ID to use for memory isolation.
+                          If provided, overrides session_name from YAML config.
+    """
     agents = {}
 
     agent_entries = [config["agent"]] if "agent" in config else config.get("agents", None)
@@ -497,6 +507,43 @@ def create_agents_from_config(config: Dict[str, Any], orchestrator_config: Optio
     if not agent_entries:
         raise ConfigurationError("Configuration must contain either 'agent' or 'agents' section")
 
+    # Create shared Qdrant client for all agents (avoids concurrent access errors)
+    # ONE client can be used by multiple mem0 instances safely
+    shared_qdrant_client = None
+    global_memory_config = config.get("memory", {})
+    if global_memory_config.get("enabled", False) and global_memory_config.get("persistent_memory", {}).get("enabled", False):
+        try:
+            from qdrant_client import QdrantClient
+
+            pm_config = global_memory_config.get("persistent_memory", {})
+
+            # Support both server mode and file-based mode
+            qdrant_config = pm_config.get("qdrant", {})
+            mode = qdrant_config.get("mode", "local")  # "local" or "server"
+
+            if mode == "server":
+                # Server mode (RECOMMENDED for multi-agent)
+                host = qdrant_config.get("host", "localhost")
+                port = qdrant_config.get("port", 6333)
+                shared_qdrant_client = QdrantClient(host=host, port=port)
+                logger.info(f"🗄️  Shared Qdrant client created (server mode: {host}:{port})")
+            else:
+                # Local file-based mode (single agent only)
+                # WARNING: Does NOT support concurrent access by multiple agents
+                qdrant_path = pm_config.get("path", ".massgen/qdrant")
+                shared_qdrant_client = QdrantClient(path=qdrant_path)
+                logger.info(f"🗄️  Shared Qdrant client created (local mode: {qdrant_path})")
+                if len(agent_entries) > 1:
+                    logger.warning(
+                        "⚠️  Multi-agent setup detected with local Qdrant mode. "
+                        "This may cause concurrent access errors. "
+                        "Consider using server mode: set memory.persistent_memory.qdrant.mode='server'",
+                    )
+        except Exception as e:
+            logger.warning(f"⚠️  Failed to create shared Qdrant client: {e}")
+            logger.warning("   Persistent memory will be disabled for all agents")
+            logger.warning("   For multi-agent setup, start Qdrant server: docker-compose -f docker-compose.qdrant.yml up -d")
+
     for i, agent_data in enumerate(agent_entries, start=1):
         backend_config = agent_data.get("backend", {})
 
@@ -579,7 +626,201 @@ def create_agents_from_config(config: Dict[str, Any], orchestrator_config: Optio
 
         # Timeout configuration will be applied to orchestrator instead of individual agents
 
-        agent = ConfigurableAgent(config=agent_config, backend=backend)
+        # Merge global and per-agent memory configuration
+        global_memory_config = config.get("memory", {})
+        agent_memory_config = agent_data.get("memory", {})
+
+        # Deep merge: agent config overrides global config
+        def merge_configs(global_cfg, agent_cfg):
+            """Recursively merge agent config into global config."""
+            merged = global_cfg.copy()
+            for key, value in agent_cfg.items():
+                if isinstance(value, dict) and key in merged and isinstance(merged[key], dict):
+                    merged[key] = merge_configs(merged[key], value)
+                else:
+                    merged[key] = value
+            return merged
+
+        memory_config = merge_configs(global_memory_config, agent_memory_config)
+
+        # Create context monitor if memory config is enabled
+        context_monitor = None
+        if memory_config.get("enabled", False):
+            from .memory._context_monitor import ContextWindowMonitor
+
+            compression_config = memory_config.get("compression", {})
+            trigger_threshold = compression_config.get("trigger_threshold", 0.75)
+            target_ratio = compression_config.get("target_ratio", 0.40)
+
+            # Get model name from backend config
+            model_name = backend_config.get("model", "unknown")
+
+            # Normalize provider name for monitor
+            provider_map = {
+                "openai": "openai",
+                "anthropic": "anthropic",
+                "claude": "anthropic",
+                "google": "google",
+                "gemini": "google",
+            }
+            provider = provider_map.get(backend_type_lower, backend_type_lower)
+
+            context_monitor = ContextWindowMonitor(
+                model_name=model_name,
+                provider=provider,
+                trigger_threshold=trigger_threshold,
+                target_ratio=target_ratio,
+                enabled=True,
+            )
+            logger.info(
+                f"📊 Context monitor created for {agent_config.agent_id}: " f"{context_monitor.context_window:,} tokens, " f"trigger={trigger_threshold*100:.0f}%, target={target_ratio*100:.0f}%",
+            )
+
+        # Create per-agent memory objects if memory is enabled
+        conversation_memory = None
+        persistent_memory = None
+
+        if memory_config.get("enabled", False):
+            from .memory import ConversationMemory
+
+            # Create conversation memory for this agent
+            if memory_config.get("conversation_memory", {}).get("enabled", True):
+                conversation_memory = ConversationMemory()
+                logger.info(f"💾 Conversation memory created for {agent_config.agent_id}")
+
+            # Create persistent memory for this agent (if enabled)
+            if memory_config.get("persistent_memory", {}).get("enabled", False):
+                from .memory import PersistentMemory
+
+                pm_config = memory_config.get("persistent_memory", {})
+
+                # Get persistent memory configuration
+                agent_name = pm_config.get("agent_name", agent_config.agent_id)
+
+                # Use unified session: memory_session_id (from CLI) > YAML session_name > None
+                session_name = memory_session_id or pm_config.get("session_name")
+
+                on_disk = pm_config.get("on_disk", True)
+                qdrant_path = pm_config.get("path", ".massgen/qdrant")  # Project dir, not /tmp
+
+                try:
+                    # Configure LLM for memory operations (fact extraction)
+                    # RECOMMENDED: Use mem0's native LLMs (no adapter overhead, no async complexity)
+                    llm_cfg = pm_config.get("llm", {})
+
+                    if not llm_cfg:
+                        # Default: gpt-4.1-nano-2025-04-14 (mem0's default, fast and cheap for memory ops)
+                        llm_cfg = {
+                            "provider": "openai",
+                            "model": "gpt-4.1-nano-2025-04-14",
+                        }
+
+                    # Add API key if not specified
+                    if "api_key" not in llm_cfg:
+                        llm_provider = llm_cfg.get("provider", "openai")
+                        if llm_provider == "openai":
+                            llm_cfg["api_key"] = os.getenv("OPENAI_API_KEY")
+                        elif llm_provider == "anthropic":
+                            llm_cfg["api_key"] = os.getenv("ANTHROPIC_API_KEY")
+                        elif llm_provider == "groq":
+                            llm_cfg["api_key"] = os.getenv("GROQ_API_KEY")
+                        # Add more providers as needed
+
+                    # Configure embedding for persistent memory
+                    # RECOMMENDED: Use mem0's native embedders (no adapter overhead)
+                    embedding_cfg = pm_config.get("embedding", {})
+
+                    if not embedding_cfg:
+                        # Default: OpenAI text-embedding-3-small
+                        embedding_cfg = {
+                            "provider": "openai",
+                            "model": "text-embedding-3-small",
+                        }
+
+                    # Add API key if not specified
+                    if "api_key" not in embedding_cfg:
+                        emb_provider = embedding_cfg.get("provider", "openai")
+                        if emb_provider == "openai":
+                            api_key = os.getenv("OPENAI_API_KEY")
+                            if not api_key:
+                                logger.warning("⚠️  OPENAI_API_KEY not found in environment - embedding will fail!")
+                            else:
+                                logger.debug(f"✅ Using OPENAI_API_KEY from environment (key starts with: {api_key[:7]}...)")
+                            embedding_cfg["api_key"] = api_key
+                        elif emb_provider == "together":
+                            embedding_cfg["api_key"] = os.getenv("TOGETHER_API_KEY")
+                        elif emb_provider == "azure_openai":
+                            embedding_cfg["api_key"] = os.getenv("AZURE_OPENAI_API_KEY")
+                        # Add more providers as needed
+
+                    # Use shared Qdrant client if available
+                    if shared_qdrant_client:
+                        persistent_memory = PersistentMemory(
+                            agent_name=agent_name,
+                            session_name=session_name,
+                            llm_config=llm_cfg,  # Use native mem0 LLM
+                            embedding_config=embedding_cfg,  # Use native mem0 embedder
+                            qdrant_client=shared_qdrant_client,  # Share ONE client from server
+                            on_disk=on_disk,
+                        )
+                        logger.info(
+                            f"💾 Persistent memory created for {agent_config.agent_id} "
+                            f"(agent_name={agent_name}, session={session_name or 'cross-session'}, "
+                            f"llm={llm_cfg.get('provider')}/{llm_cfg.get('model')}, "
+                            f"embedder={embedding_cfg.get('provider')}/{embedding_cfg.get('model')}, shared_qdrant=True)",
+                        )
+                    else:
+                        # Fallback: create individual vector store (for backward compatibility)
+                        # WARNING: File-based Qdrant doesn't support concurrent access
+                        from mem0.vector_stores.configs import VectorStoreConfig
+
+                        vector_store_config = VectorStoreConfig(
+                            config={
+                                "on_disk": on_disk,
+                                "path": qdrant_path,
+                            },
+                        )
+
+                        persistent_memory = PersistentMemory(
+                            agent_name=agent_name,
+                            session_name=session_name,
+                            llm_config=llm_cfg,  # Use native mem0 LLM
+                            embedding_config=embedding_cfg,  # Use native mem0 embedder
+                            vector_store_config=vector_store_config,
+                            on_disk=on_disk,
+                        )
+                        logger.info(
+                            f"💾 Persistent memory created for {agent_config.agent_id} "
+                            f"(agent_name={agent_name}, session={session_name or 'cross-session'}, "
+                            f"llm={llm_cfg.get('provider')}/{llm_cfg.get('model')}, "
+                            f"embedder={embedding_cfg.get('provider')}/{embedding_cfg.get('model')}, path={qdrant_path})",
+                        )
+                except Exception as e:
+                    logger.warning(
+                        f"⚠️  Failed to create persistent memory for {agent_config.agent_id}: {e}",
+                    )
+                    persistent_memory = None
+
+        # Create agent
+        agent = ConfigurableAgent(
+            config=agent_config,
+            backend=backend,
+            conversation_memory=conversation_memory,
+            persistent_memory=persistent_memory,
+            context_monitor=context_monitor,
+        )
+
+        # Configure retrieval settings from YAML (if memory is enabled)
+        if memory_config.get("enabled", False):
+            retrieval_config = memory_config.get("retrieval", {})
+            agent._retrieval_limit = retrieval_config.get("limit", 5)
+            agent._retrieval_exclude_recent = retrieval_config.get("exclude_recent", True)
+
+            if retrieval_config:  # Only log if custom config provided
+                logger.info(
+                    f"🔧 Retrieval configured for {agent_config.agent_id}: " f"limit={agent._retrieval_limit}, exclude_recent={agent._retrieval_exclude_recent}",
+                )
+
         agents[agent.config.agent_id] = agent
 
     return agents
@@ -696,21 +937,25 @@ def relocate_filesystem_paths(config: Dict[str, Any]) -> None:
             backend_config["cwd"] = str(massgen_dir / "workspaces" / user_cwd)
 
 
-def load_previous_turns(session_info: Dict[str, Any], session_storage: str) -> List[Dict[str, Any]]:
+def load_previous_turns(session_info: Dict[str, Any], session_storage: str) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
     """
-    Load previous turns from session storage.
+    Load previous turns and winning agents history from session storage.
 
     Returns:
-        List of previous turn metadata dicts
+        tuple: (previous_turns, winning_agents_history)
+            - previous_turns: List of previous turn metadata dicts
+            - winning_agents_history: List of winning agents for memory sharing
+                                     Format: [{"agent_id": "agent_b", "turn": 1}, ...]
     """
     session_id = session_info.get("session_id")
     if not session_id:
-        return []
+        return [], []
 
     session_dir = Path(session_storage) / session_id
     if not session_dir.exists():
-        return []
+        return [], []
 
+    # Load previous turns
     previous_turns = []
     turn_num = 1
 
@@ -735,7 +980,17 @@ def load_previous_turns(session_info: Dict[str, Any], session_storage: str) -> L
 
         turn_num += 1
 
-    return previous_turns
+    # Load winning agents history for memory sharing across turns
+    winning_agents_history = []
+    winning_agents_file = session_dir / "winning_agents_history.json"
+    if winning_agents_file.exists():
+        try:
+            winning_agents_history = json.loads(winning_agents_file.read_text(encoding="utf-8"))
+            logger.info(f"📚 Loaded {len(winning_agents_history)} winning agent(s) from session storage: {winning_agents_history}")
+        except Exception as e:
+            logger.warning(f"⚠️  Failed to load winning agents history: {e}")
+
+    return previous_turns, winning_agents_history
 
 
 async def handle_session_persistence(
@@ -795,6 +1050,16 @@ async def handle_session_persistence(
     metadata_file = turn_dir / "metadata.json"
     metadata_file.write_text(json.dumps(metadata, indent=2), encoding="utf-8")
 
+    # Save winning agents history for memory sharing across turns
+    # This allows the orchestrator to restore winner tracking when recreated
+    if final_result.get("winning_agents_history"):
+        winning_agents_file = session_dir / "winning_agents_history.json"
+        winning_agents_file.write_text(
+            json.dumps(final_result["winning_agents_history"], indent=2),
+            encoding="utf-8",
+        )
+        logger.info(f"📚 Saved {len(final_result['winning_agents_history'])} winning agent(s) to session storage")
+
     # Create/update session summary for easy viewing
     session_summary_file = session_dir / "SESSION_SUMMARY.txt"
     summary_lines = []
@@ -896,8 +1161,8 @@ async def run_question_with_history(
             max_orchestration_restarts=coord_cfg.get("max_orchestration_restarts", 0),
         )
 
-    # Load previous turns from session storage for multi-turn conversations
-    previous_turns = load_previous_turns(session_info, session_storage)
+    # Load previous turns and winning agents history from session storage for multi-turn conversations
+    previous_turns, winning_agents_history = load_previous_turns(session_info, session_storage)
 
     orchestrator = Orchestrator(
         agents=agents,
@@ -905,6 +1170,7 @@ async def run_question_with_history(
         snapshot_storage=snapshot_storage,
         agent_temporary_workspace=agent_temporary_workspace,
         previous_turns=previous_turns,
+        winning_agents_history=winning_agents_history,  # Restore for memory sharing
     )
     # Create a fresh UI instance for each question to ensure clean state
     ui = CoordinationUI(
@@ -1883,6 +2149,7 @@ async def run_interactive_mode(
     original_config: Dict[str, Any] = None,
     orchestrator_cfg: Dict[str, Any] = None,
     config_path: Optional[str] = None,
+    memory_session_id: Optional[str] = None,
     **kwargs,
 ):
     """Run MassGen in interactive mode with conversation history."""
@@ -1971,8 +2238,13 @@ async def run_interactive_mode(
     if original_config and orchestrator_cfg:
         config_modified = prompt_for_context_paths(original_config, orchestrator_cfg)
         if config_modified:
-            # Recreate agents with updated context paths
-            agents = create_agents_from_config(original_config, orchestrator_cfg, config_path=config_path)
+            # Recreate agents with updated context paths (use same session)
+            agents = create_agents_from_config(
+                original_config,
+                orchestrator_cfg,
+                config_path=config_path,
+                memory_session_id=memory_session_id,
+            )
             print(f"   {BRIGHT_GREEN}✓ Agents reloaded with updated context paths{RESET}", flush=True)
             print()
 
@@ -1982,7 +2254,8 @@ async def run_interactive_mode(
     conversation_history = []
 
     # Session management for multi-turn filesystem support
-    session_id = None
+    # Use memory_session_id (unified with memory system) if provided, otherwise create later
+    session_id = memory_session_id
     current_turn = 0
     session_storage = kwargs.get("orchestrator", {}).get("session_storage", "sessions")
 
@@ -2029,8 +2302,13 @@ async def run_interactive_mode(
                                 new_turn_config = {"path": str(latest_turn_workspace.resolve()), "permission": "read"}
                                 backend_config["context_paths"] = existing_context_paths + [new_turn_config]
 
-                        # Recreate agents from modified config
-                        agents = create_agents_from_config(modified_config, orchestrator_cfg, config_path=config_path)
+                        # Recreate agents from modified config (use same session)
+                        agents = create_agents_from_config(
+                            modified_config,
+                            orchestrator_cfg,
+                            config_path=config_path,
+                            memory_session_id=session_id,
+                        )
                         logger.info(f"[CLI] Successfully recreated {len(agents)} agents with turn {current_turn} path as read-only context")
 
                 question = input(f"\n{BRIGHT_BLUE}👤 User:{RESET} ").strip()
@@ -2322,7 +2600,28 @@ async def main(args):
                     '  agent_temporary_workspace: "your_temp_dir"  # Directory for temporary agent workspaces',
                 )
 
-        agents = create_agents_from_config(config, orchestrator_cfg, config_path=str(resolved_path) if resolved_path else None)
+        # Create unified session ID for memory system (before creating agents)
+        # This ensures memory is isolated per session and unifies orchestrator + memory sessions
+        memory_session_id = None
+        if args.question:
+            # Single question mode: Create temp session per run
+            from datetime import datetime
+
+            memory_session_id = f"temp_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+            logger.info(f"📝 Created temp session for single-question mode: {memory_session_id}")
+        else:
+            # Interactive mode: Create session now (will be reused by orchestrator)
+            from datetime import datetime
+
+            memory_session_id = f"session_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+            logger.info(f"📝 Created session for interactive mode: {memory_session_id}")
+
+        agents = create_agents_from_config(
+            config,
+            orchestrator_cfg,
+            config_path=str(resolved_path) if resolved_path else None,
+            memory_session_id=memory_session_id,
+        )
 
         if not agents:
             raise ConfigurationError("No agents configured")
@@ -2358,9 +2657,17 @@ async def main(args):
                 #     print(f"\n{BRIGHT_GREEN}Final Response:{RESET}", flush=True)
                 #     print(f"{response}", flush=True)
             else:
-                # Pass the config path to interactive mode
+                # Pass the config path and session_id to interactive mode
                 config_file_path = str(resolved_path) if args.config and resolved_path else None
-                await run_interactive_mode(agents, ui_config, original_config=config, orchestrator_cfg=orchestrator_cfg, config_path=config_file_path, **kwargs)
+                await run_interactive_mode(
+                    agents,
+                    ui_config,
+                    original_config=config,
+                    orchestrator_cfg=orchestrator_cfg,
+                    config_path=config_file_path,
+                    memory_session_id=memory_session_id,
+                    **kwargs,
+                )
         finally:
             # Cleanup all agents' filesystem managers (including Docker containers)
             for agent_id, agent in agents.items():

massgen/configs/README.md

@@ -227,53 +227,104 @@ Most configurations use environment variables for API keys:so
 
 ## Release History & Examples
 
-### v0.1.3 - Latest
-**New Features:** Post-Evaluation Workflow, Custom Multimodal Understanding Tools, Docker Sudo Mode
+### v0.1.5 - Latest
+**New Features:** Memory System with Semantic Retrieval
 
 **Configuration Files:**
-- `configs/tools/custom_tools/multimodal_tools/understand_image.yaml` - Image analysis configuration
-- `configs/tools/custom_tools/multimodal_tools/understand_audio.yaml` - Audio transcription configuration
-- `configs/tools/custom_tools/multimodal_tools/understand_video.yaml` - Video analysis configuration
-- `configs/tools/custom_tools/multimodal_tools/understand_file.yaml` - Document processing configuration
+- `gpt5mini_gemini_context_window_management.yaml` - Multi-agent with automatic context compression
+- `gpt5mini_gemini_research_to_implementation.yaml` - **Research-to-implementation workflow** (featured in case study)
+- `gpt5mini_high_reasoning_gemini.yaml` - High reasoning agents with memory integration
+- `gpt5mini_gemini_baseline_research_to_implementation.yaml` - Baseline research workflow
+- `single_agent_compression_test.yaml` - Testing context compression behavior
+
+**Documentation & Case Studies:**
+- `docs/source/user_guide/memory.rst` - Complete memory system user guide
+- `docs/source/examples/case_studies/multi-turn-persistent-memory.md` - **Memory case study with demo video**
+- Memory design decisions and architecture documentation
+- API reference for PersistentMemory, ConversationMemory, and ContextMonitor
 
-**Documentation:**
-- `massgen/tool/docs/multimodal_tools.md` - Complete 779-line multimodal tools guide
-- `docs/source/user_guide/multimodal.rst` - Updated multimodal documentation with custom tools
-- `docs/source/user_guide/code_execution.rst` - Enhanced with 98 lines documenting sudo mode
-- `massgen/docker/README.md` - Updated Docker documentation with sudo mode instructions
+**Key Features:**
+- **Long-Term Memory**: Semantic storage via mem0 with vector database integration
+- **Context Compression**: Automatic compression when approaching token limits
+- **Cross-Agent Sharing**: Agents learn from each other's experiences
+- **Session Management**: Memory persistence across conversations
 
-**Case Study:**
-- [Multimodal Video Understanding](../../docs/case_studies/multimodal-case-study-video-analysis.md)
+**Try it:**
+```bash
+# Install or upgrade
+pip install --upgrade massgen
+
+# Multi-agent collaboration with context compression
+massgen --config @examples/memory/gpt5mini_gemini_context_window_management \
+  "Analyze the MassGen codebase comprehensively. Create an architecture document that explains: (1) Core components and their responsibilities, (2) How different modules interact, (3) Key design patterns used, (4) Main entry points and request flows. Read > 30 files to build a complete understanding."
 
-**Example Resources:**
-- `configs/resources/v0.1.3-example/multimodality.jpg` - Image example
-- `configs/resources/v0.1.3-example/Sherlock_Holmes.mp3` - Audio example
-- `configs/resources/v0.1.3-example/oppenheimer_trailer_1920.mp4` - Video example
-- `configs/resources/v0.1.3-example/TUMIX.pdf` - PDF document example
+# Research-to-implementation workflow with memory persistence
+# Prerequisites: Start Qdrant and crawl4ai Docker containers
+docker run -d -p 6333:6333 -p 6334:6334 \
+  -v $(pwd)/.massgen/qdrant_storage:/qdrant/storage:z qdrant/qdrant
+docker run -d -p 11235:11235 --name crawl4ai --shm-size=1g unclecode/crawl4ai:latest
+
+# Session 1 - Research phase:
+massgen --config @examples/memory/gpt5mini_gemini_research_to_implementation \
+  "Use crawl4ai to research the latest multi-agent AI papers and techniques from 2025. Focus on: coordination mechanisms, voting strategies, tool-use patterns, and architectural innovations."
+
+# Session 2 - Implementation analysis (continue in same session):
+# "Based on the multi-agent research from earlier, which techniques should we implement in MassGen to make it more state-of-the-art? Consider MassGen's current architecture and what would be most impactful."
+
+→ See [Multi-Turn Persistent Memory Case Study](../../docs/source/examples/case_studies/multi-turn-persistent-memory.md) for detailed analysis
+
+# Test automatic context compression
+massgen --config @examples/memory/single_agent_compression_test \
+  "Analyze the MassGen codebase comprehensively. Create an architecture document that explains: (1) Core components and their responsibilities, (2) How different modules interact, (3) Key design patterns used, (4) Main entry points and request flows. Read > 30 files to build a complete understanding."
+```
+
+### v0.1.4
+**New Features:** Multimodal Generation Tools, Binary File Protection, Crawl4AI Integration
+
+**Configuration Files:**
+- `text_to_image_generation_single.yaml` / `text_to_image_generation_multi.yaml` - Image generation
+- `text_to_video_generation_single.yaml` / `text_to_video_generation_multi.yaml` - Video generation
+- `text_to_speech_generation_single.yaml` / `text_to_speech_generation_multi.yaml` - Audio generation
+- `text_to_file_generation_single.yaml` / `text_to_file_generation_multi.yaml` - Document generation
+- `crawl4ai_example.yaml` - Web scraping configuration
 
 **Key Features:**
-- **Post-Evaluation Tools**: Submit and restart capabilities for winning agents with confidence assessments
-- **Multimodal Understanding**: Analyze images, audio, video, and documents using GPT-4.1
-- **Docker Sudo Mode**: Execute privileged commands in containerized environments
-- **Config Builder**: Improved workflow with auto-detection and better provider handling
+- **Generation Tools**: Create images, videos, audio, and documents using OpenAI APIs
+- **Binary File Protection**: Automatic blocking prevents text tools from reading 40+ binary file types
+- **Web Scraping**: Crawl4AI integration for intelligent content extraction
+- **Enhanced Security**: Smart tool suggestions guide users to appropriate specialized tools
 
 **Try it:**
 ```bash
-# Install or upgrade
-pip install --upgrade massgen
+# Generate an image from text
+massgen --config @examples/tools/custom_tools/multimodal_tools/text_to_image_generation_single \
+  "Please generate an image of a cat in space."
+
+# Generate a video from text
+massgen --config @examples/tools/custom_tools/multimodal_tools/text_to_video_generation_single \
+  "Generate a 4 seconds video with neon-lit alley at night, light rain, slow push-in, cinematic."
+
+# Generate documents (PDF, DOCX, etc.)
+massgen --config @examples/tools/custom_tools/multimodal_tools/text_to_file_generation_single \
+  "Please generate a comprehensive technical report about the latest developments in Large Language Models (LLMs)."
+```
+
+### v0.1.3
+**New Features:** Post-Evaluation Workflow, Custom Multimodal Understanding Tools, Docker Sudo Mode
 
+**Configuration Files:**
+- `understand_image.yaml`, `understand_audio.yaml`, `understand_video.yaml`, `understand_file.yaml`
+
+**Key Features:**
+- **Post-Evaluation Tools**: Submit and restart capabilities for winning agents
+- **Multimodal Understanding**: Analyze images, audio, video, and documents
+- **Docker Sudo Mode**: Execute privileged commands in containers
+
+**Try it:**
+```bash
 # Try multimodal image understanding
-# (Requires OPENAI_API_KEY in .env)
 massgen --config @examples/tools/custom_tools/multimodal_tools/understand_image \
   "Please summarize the content in this image."
-
-# Try multimodal audio understanding
-massgen --config @examples/tools/custom_tools/multimodal_tools/understand_audio \
-  "Please summarize the content in this audio."
-
-# Try multimodal video understanding
-massgen --config @examples/tools/custom_tools/multimodal_tools/understand_video \
-  "What's happening in this video?"
 ```
 
 ### v0.1.2
@@ -284,7 +335,7 @@ massgen --config @examples/tools/custom_tools/multimodal_tools/understand_video
 - `configs/basic/multi/three_agents_default.yaml` - Updated with Grok-4-fast model
 
 **Documentation:**
-- `docs/case_studies/INTELLIGENT_PLANNING_MODE.md` - Complete intelligent planning mode guide
+- `docs/dev_notes/intelligent_planning_mode.md` - Complete intelligent planning mode guide
 
 **Key Features:**
 - **Intelligent Planning Mode**: Automatic analysis of question irreversibility for dynamic MCP tool blocking
@@ -392,7 +443,7 @@ massgen --config @examples/tools/code-execution/docker_with_resource_limits \
 - `massgen/configs/basic/single/single_gpt4o_video_generation.yaml` - Video generation with OpenAI Sora-2
 
 **Case Study:**
-- [Universal Code Execution via MCP](../../docs/case_studies/universal-code-execution-mcp.md)
+- [Universal Code Execution via MCP](../../docs/source/examples/case_studies/universal-code-execution-mcp.md)
 
 **Key Features:**
 - Universal `execute_command` tool works across Claude, Gemini, OpenAI (Response API), and Chat Completions providers (Grok, ZAI, etc.)
@@ -465,7 +516,7 @@ massgen --config @examples/tools/filesystem/cc_gpt5_gemini_filesystem \
 - New `FileOperationTracker` class for read-before-delete enforcement
 - Enhanced PathPermissionManager with operation tracking methods
 
-**Case Study:** [MCP Planning Mode](../../docs/case_studies/mcp-planning-mode.md)
+**Case Study:** [MCP Planning Mode](../../docs/source/examples/case_studies/mcp-planning-mode.md)
 
 **Try it:**
 ```bash
@@ -492,7 +543,7 @@ massgen --config @examples/tools/planning/five_agents_twitter_mcp_planning_mode
 - New `ExternalAgentBackend` class bridging MassGen with external frameworks
 - Multiple code executor types: LocalCommandLineCodeExecutor, DockerCommandLineCodeExecutor, JupyterCodeExecutor, YepCodeCodeExecutor
 
-**Case Study:** [AG2 Framework Integration](../../docs/case_studies/ag2-framework-integration.md)
+**Case Study:** [AG2 Framework Integration](../../docs/source/examples/case_studies/ag2-framework-integration.md)
 
 **Try it:**
 ```bash
@@ -561,7 +612,7 @@ massgen --config @examples/tools/filesystem/gemini_gpt5nano_file_context_path \
 - Automatic `.massgen` directory management for persistent conversation context
 - Enhanced path permissions with `will_be_writable` flag and smart exclusion patterns
 
-**Case Study:** [Multi-Turn Filesystem Support](../../docs/case_studies/multi-turn-filesystem-support.md)
+**Case Study:** [Multi-Turn Filesystem Support](../../docs/source/examples/case_studies/multi-turn-filesystem-support.md)
 ```bash
 # Turn 1 - Initial creation
 Turn 1: Make a website about Bob Dylan
@@ -599,7 +650,7 @@ massgen --config @examples/basic/multi/two_qwen_vllm \
 - All configs now organized by provider & use case (basic/, providers/, tools/, teams/)
 - Use same configs as v0.0.21 for compatibility, but now with improved performance
 
-**Case Study:** [Advanced Filesystem with User Context Path Support](../../docs/case_studies/v0.0.21-v0.0.22-filesystem-permissions.md)
+**Case Study:** [Advanced Filesystem with User Context Path Support](../../docs/source/examples/case_studies/v0.0.21-v0.0.22-filesystem-permissions.md)
 ```bash
 # Multi-agent collaboration with granular filesystem permissions
 massgen --config @examples/tools/filesystem/gpt5mini_cc_fs_context_path "Enhance the website in massgen/configs/resources with: 1) A dark/light theme toggle with smooth transitions, 2) An interactive feature that helps users engage with the blog content (your choice - could be search, filtering by topic, reading time estimates, social sharing, reactions, etc.), and 3) Visual polish with CSS animations or transitions that make the site feel more modern and responsive. Use vanilla JavaScript and be creative with the implementation details."
@@ -645,7 +696,7 @@ massgen --config @examples/tools/mcp/gpt5_nano_mcp_example \
 
 ### v0.0.16
 **New Features:** Unified Filesystem Support with MCP Integration
-**Case Study:** [Cross-Backend Collaboration with Gemini MCP Filesystem](../../docs/case_studies/unified-filesystem-mcp-integration.md)
+**Case Study:** [Cross-Backend Collaboration with Gemini MCP Filesystem](../../docs/source/examples/case_studies/unified-filesystem-mcp-integration.md)
 ```bash
 # Gemini and Claude Code agents with unified filesystem via MCP
 massgen --config @examples/tools/mcp/gemini_mcp_filesystem_test_with_claude_code "Create a presentation that teaches a reinforcement learning algorithm and output it in LaTeX Beamer format. No figures should be added."
@@ -658,7 +709,7 @@ massgen --config @examples/tools/mcp/gemini_mcp_filesystem_test_with_claude_code
 
 ### v0.0.12 - v0.0.14
 **New Features:** Enhanced Logging and Workspace Management
-**Case Study:** [Claude Code Workspace Management with Comprehensive Logging](../../docs/case_studies/claude-code-workspace-management.md)
+**Case Study:** [Claude Code Workspace Management with Comprehensive Logging](../../docs/source/examples/case_studies/claude-code-workspace-management.md)
 ```bash
 # Multi-agent Claude Code collaboration with enhanced workspace isolation
 massgen --config @examples/tools/filesystem/claude_code_context_sharing "Create a website about a diverse set of fun facts about LLMs, placing the output in one index.html file"