massgen 0.1.4__py3-none-any.whl → 0.1.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of massgen might be problematic. Click here for more details.

Files changed (84) hide show
  1. massgen/__init__.py +1 -1
  2. massgen/backend/base_with_custom_tool_and_mcp.py +453 -23
  3. massgen/backend/capabilities.py +39 -0
  4. massgen/backend/chat_completions.py +111 -197
  5. massgen/backend/claude.py +210 -181
  6. massgen/backend/gemini.py +1015 -1559
  7. massgen/backend/grok.py +3 -2
  8. massgen/backend/response.py +160 -220
  9. massgen/chat_agent.py +340 -20
  10. massgen/cli.py +399 -25
  11. massgen/config_builder.py +20 -54
  12. massgen/config_validator.py +931 -0
  13. massgen/configs/README.md +95 -10
  14. massgen/configs/memory/gpt5mini_gemini_baseline_research_to_implementation.yaml +94 -0
  15. massgen/configs/memory/gpt5mini_gemini_context_window_management.yaml +187 -0
  16. massgen/configs/memory/gpt5mini_gemini_research_to_implementation.yaml +127 -0
  17. massgen/configs/memory/gpt5mini_high_reasoning_gemini.yaml +107 -0
  18. massgen/configs/memory/single_agent_compression_test.yaml +64 -0
  19. massgen/configs/tools/custom_tools/claude_code_custom_tool_with_mcp_example.yaml +1 -0
  20. massgen/configs/tools/custom_tools/claude_custom_tool_example_no_path.yaml +1 -1
  21. massgen/configs/tools/custom_tools/claude_custom_tool_with_mcp_example.yaml +1 -0
  22. massgen/configs/tools/custom_tools/computer_use_browser_example.yaml +1 -1
  23. massgen/configs/tools/custom_tools/computer_use_docker_example.yaml +1 -1
  24. massgen/configs/tools/custom_tools/gemini_custom_tool_with_mcp_example.yaml +1 -0
  25. massgen/configs/tools/custom_tools/gpt5_nano_custom_tool_with_mcp_example.yaml +1 -0
  26. massgen/configs/tools/custom_tools/gpt_oss_custom_tool_with_mcp_example.yaml +1 -0
  27. massgen/configs/tools/custom_tools/grok3_mini_custom_tool_with_mcp_example.yaml +1 -0
  28. massgen/configs/tools/custom_tools/interop/ag2_and_langgraph_lesson_planner.yaml +65 -0
  29. massgen/configs/tools/custom_tools/interop/ag2_and_openai_assistant_lesson_planner.yaml +65 -0
  30. massgen/configs/tools/custom_tools/interop/ag2_lesson_planner_example.yaml +48 -0
  31. massgen/configs/tools/custom_tools/interop/agentscope_lesson_planner_example.yaml +48 -0
  32. massgen/configs/tools/custom_tools/interop/langgraph_lesson_planner_example.yaml +49 -0
  33. massgen/configs/tools/custom_tools/interop/openai_assistant_lesson_planner_example.yaml +50 -0
  34. massgen/configs/tools/custom_tools/interop/smolagent_lesson_planner_example.yaml +49 -0
  35. massgen/configs/tools/custom_tools/qwen_api_custom_tool_with_mcp_example.yaml +1 -0
  36. massgen/configs/tools/custom_tools/two_models_with_tools_example.yaml +44 -0
  37. massgen/formatter/_gemini_formatter.py +61 -15
  38. massgen/memory/README.md +277 -0
  39. massgen/memory/__init__.py +26 -0
  40. massgen/memory/_base.py +193 -0
  41. massgen/memory/_compression.py +237 -0
  42. massgen/memory/_context_monitor.py +211 -0
  43. massgen/memory/_conversation.py +255 -0
  44. massgen/memory/_fact_extraction_prompts.py +333 -0
  45. massgen/memory/_mem0_adapters.py +257 -0
  46. massgen/memory/_persistent.py +687 -0
  47. massgen/memory/docker-compose.qdrant.yml +36 -0
  48. massgen/memory/docs/DESIGN.md +388 -0
  49. massgen/memory/docs/QUICKSTART.md +409 -0
  50. massgen/memory/docs/SUMMARY.md +319 -0
  51. massgen/memory/docs/agent_use_memory.md +408 -0
  52. massgen/memory/docs/orchestrator_use_memory.md +586 -0
  53. massgen/memory/examples.py +237 -0
  54. massgen/orchestrator.py +207 -7
  55. massgen/tests/memory/test_agent_compression.py +174 -0
  56. massgen/tests/memory/test_context_window_management.py +286 -0
  57. massgen/tests/memory/test_force_compression.py +154 -0
  58. massgen/tests/memory/test_simple_compression.py +147 -0
  59. massgen/tests/test_ag2_lesson_planner.py +223 -0
  60. massgen/tests/test_agent_memory.py +534 -0
  61. massgen/tests/test_config_validator.py +1156 -0
  62. massgen/tests/test_conversation_memory.py +382 -0
  63. massgen/tests/test_langgraph_lesson_planner.py +223 -0
  64. massgen/tests/test_orchestrator_memory.py +620 -0
  65. massgen/tests/test_persistent_memory.py +435 -0
  66. massgen/token_manager/token_manager.py +6 -0
  67. massgen/tool/__init__.py +2 -9
  68. massgen/tool/_decorators.py +52 -0
  69. massgen/tool/_extraframework_agents/ag2_lesson_planner_tool.py +251 -0
  70. massgen/tool/_extraframework_agents/agentscope_lesson_planner_tool.py +303 -0
  71. massgen/tool/_extraframework_agents/langgraph_lesson_planner_tool.py +275 -0
  72. massgen/tool/_extraframework_agents/openai_assistant_lesson_planner_tool.py +247 -0
  73. massgen/tool/_extraframework_agents/smolagent_lesson_planner_tool.py +180 -0
  74. massgen/tool/_manager.py +102 -16
  75. massgen/tool/_registered_tool.py +3 -0
  76. massgen/tool/_result.py +3 -0
  77. {massgen-0.1.4.dist-info → massgen-0.1.6.dist-info}/METADATA +138 -77
  78. {massgen-0.1.4.dist-info → massgen-0.1.6.dist-info}/RECORD +82 -37
  79. massgen/backend/gemini_mcp_manager.py +0 -545
  80. massgen/backend/gemini_trackers.py +0 -344
  81. {massgen-0.1.4.dist-info → massgen-0.1.6.dist-info}/WHEEL +0 -0
  82. {massgen-0.1.4.dist-info → massgen-0.1.6.dist-info}/entry_points.txt +0 -0
  83. {massgen-0.1.4.dist-info → massgen-0.1.6.dist-info}/licenses/LICENSE +0 -0
  84. {massgen-0.1.4.dist-info → massgen-0.1.6.dist-info}/top_level.txt +0 -0
massgen/configs/README.md CHANGED
@@ -227,7 +227,101 @@ Most configurations use environment variables for API keys:so
227
227
 
228
228
  ## Release History & Examples
229
229
 
230
- ### v0.1.4 - Latest
230
+ ### v0.1.6 - Latest
231
+ **New Features:** Framework Interoperability & Backend Refactoring
232
+
233
+ **Configuration Files:**
234
+ - `ag2_lesson_planner_example.yaml` - AG2 nested chat as custom tool (supports streaming)
235
+ - `langgraph_lesson_planner_example.yaml` - LangGraph workflows integrated as tools
236
+ - `agentscope_lesson_planner_example.yaml` - AgentScope agent system integration
237
+ - `openai_assistant_lesson_planner_example.yaml` - OpenAI Assistants as tools
238
+ - `smolagent_lesson_planner_example.yaml` - HuggingFace SmoLAgent integration
239
+ - `ag2_and_langgraph_lesson_planner.yaml` - Multi-framework collaboration (AG2 + LangGraph)
240
+ - `ag2_and_openai_assistant_lesson_planner.yaml` - AG2 + OpenAI Assistants combination
241
+ - `two_models_with_tools_example.yaml` - Multiple models with custom tools
242
+
243
+ **Key Features:**
244
+ - **Framework Interoperability**: Use agents from external frameworks (AG2, LangGraph, AgentScope, OpenAI Assistants, SmoLAgent) as MassGen tools
245
+ - **Streaming Support**: AG2 supports streaming; other frameworks return complete results
246
+ - **Configuration Validator**: Pre-flight YAML validation with detailed error messages
247
+ - **Unified Tool Execution**: ToolExecutionConfig dataclass for consistent tool handling
248
+ - **Gemini Simplification**: Major backend cleanup reducing codebase by 1,598 lines
249
+
250
+ **Try It:**
251
+ ```bash
252
+ # Use AG2 agents for lesson planning (supports streaming)
253
+ # Requirements: pip install pyautogen, OPENAI_API_KEY must be set
254
+ massgen --config massgen/configs/tools/custom_tools/ag2_lesson_planner_example.yaml "Create a lesson plan for photosynthesis"
255
+
256
+ # Use LangGraph workflows as tools
257
+ # Requirements: pip install langgraph langchain-openai langchain-core, OPENAI_API_KEY must be set
258
+ massgen --config massgen/configs/tools/custom_tools/langgraph_lesson_planner_example.yaml "Create a lesson plan for photosynthesis"
259
+
260
+ # Use AgentScope multi-agent framework as tools
261
+ # Requirements: pip install agentscope, OPENAI_API_KEY must be set
262
+ massgen --config massgen/configs/tools/custom_tools/agentscope_lesson_planner_example.yaml "Create a lesson plan for photosynthesis"
263
+
264
+ # Use OpenAI Assistants API as tools
265
+ # Requirements: pip install openai, OPENAI_API_KEY must be set
266
+ massgen --config massgen/configs/tools/custom_tools/openai_assistant_lesson_planner_example.yaml "Create a lesson plan for photosynthesis"
267
+
268
+ # Use SmolAgent (HuggingFace) as tools
269
+ # Requirements: pip install smolagents, OPENAI_API_KEY must be set
270
+ massgen --config massgen/configs/tools/custom_tools/smolagent_lesson_planner_example.yaml "Create a lesson plan for photosynthesis"
271
+
272
+ # Combine multiple frameworks
273
+ # Requirements: pip install pyautogen langgraph langchain-openai langchain-core, OPENAI_API_KEY must be set
274
+ massgen --config massgen/configs/tools/custom_tools/ag2_and_langgraph_lesson_planner.yaml "Create a lesson plan for photosynthesis"
275
+ ```
276
+
277
+ ### v0.1.5
278
+ **New Features:** Memory System with Semantic Retrieval
279
+
280
+ **Configuration Files:**
281
+ - `gpt5mini_gemini_context_window_management.yaml` - Multi-agent with automatic context compression
282
+ - `gpt5mini_gemini_research_to_implementation.yaml` - Research-to-implementation workflow (featured in case study)
283
+ - `gpt5mini_high_reasoning_gemini.yaml` - High reasoning agents with memory integration
284
+ - `gpt5mini_gemini_baseline_research_to_implementation.yaml` - Baseline research workflow
285
+ - `single_agent_compression_test.yaml` - Testing context compression behavior
286
+
287
+ **Key Features:**
288
+ - **Long-Term Memory**: Semantic storage via mem0 with vector database integration
289
+ - **Context Compression**: Automatic compression when approaching token limits
290
+ - **Cross-Agent Sharing**: Agents learn from each other's experiences
291
+ - **Session Management**: Memory persistence across conversations
292
+
293
+ **Try it:**
294
+ ```bash
295
+ # Install or upgrade
296
+ pip install --upgrade massgen
297
+
298
+ # Multi-agent collaboration with context compression
299
+ massgen --config @examples/memory/gpt5mini_gemini_context_window_management \
300
+ "Analyze the MassGen codebase comprehensively. Create an architecture document that explains: (1) Core components and their responsibilities, (2) How different modules interact, (3) Key design patterns used, (4) Main entry points and request flows. Read > 30 files to build a complete understanding."
301
+
302
+ # Research-to-implementation workflow with memory persistence
303
+ # Prerequisites: Start Qdrant and crawl4ai Docker containers
304
+ docker run -d -p 6333:6333 -p 6334:6334 \
305
+ -v $(pwd)/.massgen/qdrant_storage:/qdrant/storage:z qdrant/qdrant
306
+ docker run -d -p 11235:11235 --name crawl4ai --shm-size=1g unclecode/crawl4ai:latest
307
+
308
+ # Session 1 - Research phase:
309
+ massgen --config @examples/memory/gpt5mini_gemini_research_to_implementation \
310
+ "Use crawl4ai to research the latest multi-agent AI papers and techniques from 2025. Focus on: coordination mechanisms, voting strategies, tool-use patterns, and architectural innovations."
311
+
312
+ # Session 2 - Implementation analysis (continue in same session):
313
+ # "Based on the multi-agent research from earlier, which techniques should we implement in MassGen to make it more state-of-the-art? Consider MassGen's current architecture and what would be most impactful."
314
+ ```
315
+
316
+ → See [Multi-Turn Persistent Memory Case Study](../../docs/source/examples/case_studies/multi-turn-persistent-memory.md) for detailed analysis
317
+
318
+ ```bash
319
+ # Test automatic context compression
320
+ massgen --config @examples/memory/single_agent_compression_test \
321
+ "Analyze the MassGen codebase comprehensively. Create an architecture document that explains: (1) Core components and their responsibilities, (2) How different modules interact, (3) Key design patterns used, (4) Main entry points and request flows. Read > 30 files to build a complete understanding."
322
+ ```
323
+
324
+ ### v0.1.4
231
325
  **New Features:** Multimodal Generation Tools, Binary File Protection, Crawl4AI Integration
232
326
 
233
327
  **Configuration Files:**
@@ -237,12 +331,6 @@ Most configurations use environment variables for API keys:so
237
331
  - `text_to_file_generation_single.yaml` / `text_to_file_generation_multi.yaml` - Document generation
238
332
  - `crawl4ai_example.yaml` - Web scraping configuration
239
333
 
240
- **Documentation:**
241
- - `README_PYPI.md` - Standalone PyPI package documentation
242
- - `docs/dev_notes/release_checklist.md` - Release workflow guide
243
- - `docs/source/user_guide/protected_paths.rst` - Binary file protection documentation
244
- - `.github/workflows/docs-automation.yml` - Documentation CI/CD automation
245
-
246
334
  **Key Features:**
247
335
  - **Generation Tools**: Create images, videos, audio, and documents using OpenAI APIs
248
336
  - **Binary File Protection**: Automatic blocking prevents text tools from reading 40+ binary file types
@@ -251,9 +339,6 @@ Most configurations use environment variables for API keys:so
251
339
 
252
340
  **Try it:**
253
341
  ```bash
254
- # Install or upgrade
255
- pip install --upgrade massgen
256
-
257
342
  # Generate an image from text
258
343
  massgen --config @examples/tools/custom_tools/multimodal_tools/text_to_image_generation_single \
259
344
  "Please generate an image of a cat in space."
@@ -0,0 +1,94 @@
1
+ # Example Configuration: Memory-Enhanced Research-to-Implementation Workflow
2
+ #
3
+ # Use Case: Demonstrates how memory enables strategic self-improvement
4
+ #
5
+ # This configuration demonstrates MassGen's self-evolution capabilities:
6
+ # - Session 1: Research multi-agent AI papers using crawl4ai
7
+ # - Session 2: Apply research findings to improve MassGen itself
8
+ # - Memory: Bridges research phase to implementation analysis
9
+ # - Self-improvement: Agents use external research to enhance their own architecture
10
+ #
11
+ # Prerequisites:
12
+ # 1. Start Qdrant server:
13
+ # docker run -d -p 6333:6333 -p 6334:6334 -v $(pwd)/.massgen/qdrant_storage:/qdrant/storage:z qdrant/qdrant
14
+ #
15
+ # 2. Start crawl4ai Docker container:
16
+ # docker pull unclecode/crawl4ai:latest
17
+ # docker run -d -p 11235:11235 --name crawl4ai --shm-size=1g unclecode/crawl4ai:latest
18
+ #
19
+ # 3. Verify both containers are running:
20
+ # docker ps | grep -E "qdrant|crawl4ai"
21
+ #
22
+ # Run Session 1 (Research Phase):
23
+ # uv run massgen --config @examples/memory/gpt5mini_gemini_baseline_research_to_implementation.yaml "Use crawl4ai to research the latest multi-agent AI papers and techniques from 2025. Focus on: coordination mechanisms, voting strategies, tool-use patterns, and architectural innovations."
24
+ #
25
+ # Run Session 2 (Implementation Analysis) - Same session:
26
+ # uv run massgen --config @examples/memory/gpt5mini_gemini_baseline_research_to_implementation.yaml "Based on the multi-agent research from earlier, which techniques should we implement in MassGen to make it more state-of-the-art? Consider MassGen's current architecture and what would be most impactful."
27
+ #
28
+ # Or in a new session (still accesses memories from Session 1):
29
+ # Change session_name in config or run interactively to continue
30
+
31
+ # ====================
32
+ # AGENT DEFINITIONS
33
+ # ====================
34
+ agents:
35
+ - id: "agent_a"
36
+ backend:
37
+ type: "openai"
38
+ model: "gpt-5-mini"
39
+ text:
40
+ verbosity: "medium"
41
+ reasoning:
42
+ effort: "medium"
43
+ summary: "auto"
44
+ cwd: "workspace1"
45
+
46
+ # Register crawl4ai custom tools for web scraping
47
+ custom_tools:
48
+ - name: ["crawl4ai_md", "crawl4ai_html", "crawl4ai_screenshot", "crawl4ai_pdf", "crawl4ai_execute_js", "crawl4ai_crawl"]
49
+ category: "web_scraping"
50
+ path: "massgen/tool/_web_tools/crawl4ai_tool.py"
51
+ function: ["crawl4ai_md", "crawl4ai_html", "crawl4ai_screenshot", "crawl4ai_pdf", "crawl4ai_execute_js", "crawl4ai_crawl"]
52
+
53
+ - id: "agent_b"
54
+ backend:
55
+ type: "gemini"
56
+ model: "gemini-2.5-flash"
57
+ cwd: "workspace2"
58
+
59
+ # Register crawl4ai custom tools for web scraping
60
+ custom_tools:
61
+ - name: ["crawl4ai_md", "crawl4ai_html", "crawl4ai_screenshot", "crawl4ai_pdf", "crawl4ai_execute_js", "crawl4ai_crawl"]
62
+ category: "web_scraping"
63
+ path: "massgen/tool/_web_tools/crawl4ai_tool.py"
64
+ function: ["crawl4ai_md", "crawl4ai_html", "crawl4ai_screenshot", "crawl4ai_pdf", "crawl4ai_execute_js", "crawl4ai_crawl"]
65
+
66
+
67
+ # ====================
68
+ # MEMORY CONFIGURATION
69
+ # ====================
70
+ memory:
71
+ enabled: false
72
+
73
+ # ====================
74
+ # ORCHESTRATOR CONFIGURATION
75
+ # ====================
76
+ orchestrator:
77
+ # Multi-turn mode for interactive research sessions
78
+ session_storage: "research_sessions"
79
+ agent_temporary_workspace: "research_workspaces"
80
+ snapshot_storage: "research_snapshots"
81
+
82
+ # Give agents read access to MassGen codebase for Phase 2
83
+ context_paths:
84
+ - path: "massgen"
85
+ permission: "read"
86
+ - path: "docs"
87
+ permission: "read"
88
+
89
+ # ====================
90
+ # UI CONFIGURATION
91
+ # ====================
92
+ ui:
93
+ display_type: "rich_terminal"
94
+ logging_enabled: true
@@ -0,0 +1,187 @@
1
+ # Example Configuration: Context Window Management with Memory
2
+ #
3
+ # Use Case: Demonstrates automatic context compression when approaching token limits
4
+ #
5
+ # This configuration demonstrates:
6
+ # - Automatic context window monitoring and compression
7
+ # - Token-aware conversation management (75% threshold, 40% target)
8
+ # - Persistent memory integration for long-term knowledge retention
9
+ # - Graceful handling when context window fills up
10
+ # - Multi-agent collaboration with shared context management
11
+ #
12
+ # Run with:
13
+ # uv run massgen --config @examples/memory/gpt5mini_gemini_context_window_management.yaml "Analyze the MassGen codebase comprehensively. Create an architecture document that explains: (1) Core components and their responsibilities, (2) How different modules interact, (3) Key design patterns used, (4) Main entry points and request flows. Read > 30 files to build a complete understanding."
14
+
15
+ # ====================
16
+ # AGENT DEFINITIONS
17
+ # ====================
18
+ agents:
19
+ - id: "agent_a"
20
+ backend:
21
+ # Use GPT-5-mini with medium reasoning
22
+ type: "openai"
23
+ model: "gpt-5-mini"
24
+ text:
25
+ verbosity: "medium"
26
+ reasoning:
27
+ effort: "medium"
28
+ summary: "auto"
29
+ cwd: "workspace1"
30
+
31
+ - id: "agent_b"
32
+ backend:
33
+ # Use Gemini 2.5 Flash for cost-effective testing
34
+ type: "gemini"
35
+ model: "gemini-2.5-flash"
36
+ cwd: "workspace2"
37
+
38
+ # ====================
39
+ # MEMORY CONFIGURATION
40
+ # ====================
41
+ memory:
42
+ # Enable/disable persistent memory (default: true)
43
+ enabled: true
44
+
45
+ # Memory configuration
46
+ conversation_memory:
47
+ enabled: true # Short-term conversation tracking (recommended: always true)
48
+
49
+ persistent_memory:
50
+ enabled: true # Long-term knowledge storage (set to false to disable)
51
+ on_disk: true # Persist across restarts
52
+ # session_name: "test_session" # Optional - if not specified, auto-generates unique ID
53
+ # Format: agent_storyteller_20251023_143022_a1b2c3
54
+ # Specify to continue a specific session
55
+
56
+ # Vector store backend (default: qdrant)
57
+ vector_store: "qdrant"
58
+
59
+ # LLM configuration for memory operations (fact extraction)
60
+ # RECOMMENDED: Use mem0's native LLMs (no adapter overhead, no async complexity)
61
+ llm:
62
+ provider: "openai" # Options: openai, anthropic, groq, together, etc.
63
+ model: "gpt-4.1-nano-2025-04-14" # Fast and cheap model for memory ops (mem0's default)
64
+
65
+ # Embedding configuration (uses mem0's native embedders)
66
+ # RECOMMENDED: Specify provider and model for clarity
67
+ embedding:
68
+ provider: "openai" # Options: openai, together, azure_openai, gemini, huggingface, etc.
69
+ model: "text-embedding-3-small" # OpenAI's efficient embedding model
70
+
71
+ # Qdrant client configuration
72
+ # IMPORTANT: For multi-agent setups, use server mode to avoid concurrent access errors
73
+ qdrant:
74
+ mode: "server" # Options: "server" (recommended for multi-agent) or "local" (single agent only)
75
+ host: "localhost" # Qdrant server host (default: localhost)
76
+ port: 6333 # Qdrant server port (default: 6333)
77
+ # For local mode (single agent only):
78
+ # mode: "local"
79
+ # path: ".massgen/qdrant" # Local storage path
80
+
81
+ # Context window management thresholds
82
+ compression:
83
+ trigger_threshold: 0.25 # Compress when context usage exceeds 25%
84
+ target_ratio: 0.10 # Target 10% of context after compression
85
+
86
+ # Memory retrieval configuration
87
+ retrieval:
88
+ limit: 5 # Number of memory facts to retrieve from mem0 (default: 5)
89
+ exclude_recent: true # Only retrieve after compression to avoid duplicates (default: true)
90
+
91
+ # Memory system behavior when enabled:
92
+ # - ConversationMemory: Tracks short-term conversation history (verbatim messages)
93
+ # - PersistentMemory: Stores long-term knowledge in vector database (extracted facts via mem0)
94
+ # - Automatic compression: Triggers at threshold, removes old messages from conversation_memory
95
+ # - Smart retrieval: Only retrieves from persistent_memory AFTER compression
96
+ # - Before compression: All context in conversation_memory, no retrieval (avoids duplicates)
97
+ # - After compression: Retrieves relevant facts from compressed messages
98
+ # - Each agent gets separate memory: agent_name defaults to agent ID (agent_a, agent_b)
99
+ #
100
+ # How mem0 works:
101
+ # - When recording: mem0's LLM extracts key facts from conversations
102
+ # - When retrieving: Returns extracted facts (e.g., "User explored Mars", not full conversation)
103
+ # - retrieval.limit controls how many facts to retrieve (each fact is ~1 sentence)
104
+ #
105
+ # Session management (UNIFIED):
106
+ # - Each agent gets separate memory (agent_name = agent ID: agent_a, agent_b)
107
+ # - Session ID is unified between orchestrator and memory system:
108
+ # - Interactive mode: session_YYYYMMDD_HHMMSS (created at start, shared by all turns)
109
+ # - Single question: temp_YYYYMMDD_HHMMSS (created per run, isolated)
110
+ # - Memories are isolated per session: agent_a in session_1 can't access session_2 memories
111
+ # - To continue a previous session: Specify session_name in YAML (overrides auto-generation)
112
+ # - For cross-session memory: Remove session_name from YAML or set to null
113
+ # - Qdrant database: Shared at .massgen/qdrant, filtered by agent_id + session_id
114
+ #
115
+ # To disable persistent memory for testing, set:
116
+ # memory.persistent_memory.enabled: false
117
+ #
118
+ # See massgen/memory/docs/ for detailed documentation.
119
+
120
+ # ====================
121
+ # ORCHESTRATOR CONFIGURATION
122
+ # ====================
123
+ orchestrator:
124
+ # Multi-turn mode to enable interactive storytelling
125
+ session_storage: "memory_test_sessions"
126
+
127
+ # Agent workspace for any file operations
128
+ agent_temporary_workspace: "memory_test_workspaces"
129
+ snapshot_storage: "memory_test_snapshots"
130
+
131
+ # Additional context paths
132
+ context_paths:
133
+ - path: "massgen"
134
+ permission: "read"
135
+
136
+ # ====================
137
+ # UI CONFIGURATION
138
+ # ====================
139
+ ui:
140
+ display_type: "rich_terminal"
141
+ logging_enabled: true
142
+
143
+ # ====================
144
+ # EXECUTION FLOW
145
+ # ====================
146
+ # What happens:
147
+ # 1. User starts an interactive story with the agent
148
+ # 2. Agent responds with detailed narrative (400-600 words per turn)
149
+ # 3. As conversation continues, token usage is monitored automatically
150
+ # 4. When context usage reaches 75% of model's limit:
151
+ # - System logs: "📊 Context usage: X / Y tokens (Z%) - compressing old context"
152
+ # - Old messages are compressed into persistent memory (if configured)
153
+ # - Recent messages (fitting in 40% of context window) are kept
154
+ # - Compression details logged: "📦 Compressed N messages (X tokens) into long-term memory"
155
+ # 5. Agent continues seamlessly with compressed context
156
+ # 6. Story maintains consistency by referencing persistent memories
157
+ # 7. Process repeats as needed for very long conversations
158
+ #
159
+ # Expected logs with persistent memory:
160
+ #
161
+ # Turn 1-10 (Before compression):
162
+ # 📊 Context Window (Turn 5): 45,000 / 128,000 tokens (35%)
163
+ # ⏭️ Skipping retrieval (no compression yet, all context in conversation_memory)
164
+ #
165
+ # Turn 11 (Compression triggers):
166
+ # ⚠️ Context Window (Turn 11): 96,000 / 128,000 tokens (75%) - Approaching limit!
167
+ # 🔄 Attempting compression (96,000 → 51,200 tokens)
168
+ # 📦 Context compressed: Removed 15 old messages (44,800 tokens).
169
+ # Kept 8 recent messages (51,200 tokens).
170
+ # Old messages remain accessible via semantic search.
171
+ # ✅ Conversation history updated after compression: 8 messages
172
+ #
173
+ # Turn 12+ (After compression):
174
+ # 🔍 Retrieving compressed memories (limit=5, compressed=True)...
175
+ # 💭 Retrieved 3 memory fact(s) from mem0
176
+ # [Agent sees: retrieved facts + recent 8 messages - no duplication!]
177
+ #
178
+ # Expected output WITHOUT persistent memory:
179
+ # 📦 Context compressed: Removed 15 messages (44,800 tokens).
180
+ # No persistent memory - old messages NOT retrievable.
181
+ #
182
+ # Token Budget Allocation (after compression):
183
+ # - Conversation history: 40% (kept in active context)
184
+ # - Retrieved memories: ~5 facts (~100-250 tokens)
185
+ # - New user messages: varies
186
+ # - System prompt overhead: varies
187
+ # - Response generation: varies
@@ -0,0 +1,127 @@
1
+ # Example Configuration: Memory-Enhanced Research-to-Implementation Workflow
2
+ #
3
+ # Use Case: Demonstrates how memory enables strategic self-improvement
4
+ #
5
+ # This configuration demonstrates MassGen's self-evolution capabilities:
6
+ # - Session 1: Research multi-agent AI papers using crawl4ai
7
+ # - Session 2: Apply research findings to say how to improve MassGen itself
8
+ # - Memory: Bridges research phase to implementation analysis
9
+ # - Self-improvement: Agents use external research to enhance their own architecture
10
+ #
11
+ # Prerequisites:
12
+ # 1. Start Qdrant server:
13
+ # docker run -d -p 6333:6333 -p 6334:6334 -v $(pwd)/.massgen/qdrant_storage:/qdrant/storage:z qdrant/qdrant
14
+ #
15
+ # 2. Start crawl4ai Docker container:
16
+ # docker pull unclecode/crawl4ai:latest
17
+ # docker run -d -p 11235:11235 --name crawl4ai --shm-size=1g unclecode/crawl4ai:latest
18
+ #
19
+ # 3. Verify both containers are running:
20
+ # docker ps | grep -E "qdrant|crawl4ai"
21
+ #
22
+ # Run Session 1 (Research Phase):
23
+ # uv run massgen --config @examples/memory/gpt5mini_gemini_research_to_implementation.yaml "Use crawl4ai to research the latest multi-agent AI papers and techniques from 2025. Focus on: coordination mechanisms, voting strategies, tool-use patterns, and architectural innovations."
24
+ #
25
+ # Run Session 2 (Implementation Analysis) - Same session:
26
+ # "Based on the multi-agent research from earlier, which techniques should we implement in MassGen to make it more state-of-the-art? Consider MassGen's current architecture and what would be most impactful."
27
+ #
28
+
29
+ # ====================
30
+ # AGENT DEFINITIONS
31
+ # ====================
32
+ agents:
33
+ - id: "agent_a"
34
+ backend:
35
+ type: "openai"
36
+ model: "gpt-5-mini"
37
+ text:
38
+ verbosity: "medium"
39
+ reasoning:
40
+ effort: "medium"
41
+ summary: "auto"
42
+ cwd: "workspace1"
43
+
44
+ # Register crawl4ai custom tools for web scraping
45
+ custom_tools:
46
+ - name: ["crawl4ai_md", "crawl4ai_html", "crawl4ai_screenshot", "crawl4ai_pdf", "crawl4ai_execute_js", "crawl4ai_crawl"]
47
+ category: "web_scraping"
48
+ path: "massgen/tool/_web_tools/crawl4ai_tool.py"
49
+ function: ["crawl4ai_md", "crawl4ai_html", "crawl4ai_screenshot", "crawl4ai_pdf", "crawl4ai_execute_js", "crawl4ai_crawl"]
50
+
51
+ - id: "agent_b"
52
+ backend:
53
+ type: "gemini"
54
+ model: "gemini-2.5-flash"
55
+ cwd: "workspace2"
56
+
57
+ # Register crawl4ai custom tools for web scraping
58
+ custom_tools:
59
+ - name: ["crawl4ai_md", "crawl4ai_html", "crawl4ai_screenshot", "crawl4ai_pdf", "crawl4ai_execute_js", "crawl4ai_crawl"]
60
+ category: "web_scraping"
61
+ path: "massgen/tool/_web_tools/crawl4ai_tool.py"
62
+ function: ["crawl4ai_md", "crawl4ai_html", "crawl4ai_screenshot", "crawl4ai_pdf", "crawl4ai_execute_js", "crawl4ai_crawl"]
63
+
64
+
65
+ # ====================
66
+ # MEMORY CONFIGURATION
67
+ # ====================
68
+ memory:
69
+ enabled: true
70
+
71
+ conversation_memory:
72
+ enabled: true
73
+
74
+ persistent_memory:
75
+ enabled: true
76
+ on_disk: true
77
+ session_name: "research_to_implementation" # Same session for both phases
78
+ vector_store: "qdrant"
79
+
80
+ # LLM configuration for memory operations
81
+ llm:
82
+ provider: "openai"
83
+ model: "gpt-4.1-nano-2025-04-14"
84
+
85
+ # Embedding configuration
86
+ embedding:
87
+ provider: "openai"
88
+ model: "text-embedding-3-small"
89
+
90
+ # Qdrant server configuration (required for multi-agent)
91
+ qdrant:
92
+ mode: "server"
93
+ host: "localhost"
94
+ port: 6333
95
+
96
+ # Context window management
97
+ compression:
98
+ trigger_threshold: 0.75 # Compress at 75%
99
+ target_ratio: 0.40 # Keep 40% after compression
100
+
101
+ # Memory retrieval configuration
102
+ retrieval:
103
+ limit: 10 # Get more facts for cross-phase synthesis
104
+ exclude_recent: true # Only retrieve after compression
105
+
106
+ # ====================
107
+ # ORCHESTRATOR CONFIGURATION
108
+ # ====================
109
+ orchestrator:
110
+ # Multi-turn mode for interactive research sessions
111
+ session_storage: "research_sessions"
112
+ agent_temporary_workspace: "research_workspaces"
113
+ snapshot_storage: "research_snapshots"
114
+
115
+ # Give agents read access to MassGen codebase for Phase 2
116
+ context_paths:
117
+ - path: "massgen"
118
+ permission: "read"
119
+ - path: "docs"
120
+ permission: "read"
121
+
122
+ # ====================
123
+ # UI CONFIGURATION
124
+ # ====================
125
+ ui:
126
+ display_type: "rich_terminal"
127
+ logging_enabled: true
@@ -0,0 +1,107 @@
1
+ # Example Configuration: Context Window Management with Memory
2
+ #
3
+ # Use Case: Demonstrates automatic context compression when approaching token limits
4
+ #
5
+ # This configuration demonstrates:
6
+ # - Automatic context window monitoring and compression
7
+ # - Token-aware conversation management (75% threshold, 40% target)
8
+ # - Persistent memory integration for long-term knowledge retention
9
+ # - Graceful handling when context window fills up
10
+ # - Multi-agent collaboration with shared context management
11
+ #
12
+ # Run with:
13
+ # uv run massgen --config @examples/memory/gpt5mini_high_reasoning_gemini.yaml "Analyze the pros and cons of using LLMs in commercial applications."
14
+
15
+ # ====================
16
+ # AGENT DEFINITIONS
17
+ # ====================
18
+ agents:
19
+ - id: "agent_a"
20
+ backend:
21
+ # Use GPT-5-mini with medium reasoning
22
+ type: "openai"
23
+ model: "gpt-5-mini"
24
+ text:
25
+ verbosity: "medium"
26
+ reasoning:
27
+ effort: "high"
28
+ summary: "auto"
29
+ enable_web_search: true
30
+
31
+ - id: "agent_b"
32
+ backend:
33
+ # Use Gemini 2.5 Flash for cost-effective testing
34
+ type: "gemini"
35
+ model: "gemini-2.5-flash"
36
+ enable_web_search: true
37
+
38
+ # ====================
39
+ # MEMORY CONFIGURATION
40
+ # ====================
41
+ memory:
42
+ # Enable/disable persistent memory (default: true)
43
+ enabled: true
44
+
45
+ # Memory configuration
46
+ conversation_memory:
47
+ enabled: true # Short-term conversation tracking (recommended: always true)
48
+
49
+ persistent_memory:
50
+ enabled: true # Long-term knowledge storage (set to false to disable)
51
+ on_disk: true # Persist across restarts
52
+ # session_name: "test_session" # Optional - if not specified, auto-generates unique ID
53
+ # Format: agent_storyteller_20251023_143022_a1b2c3
54
+ # Specify to continue a specific session
55
+
56
+ # Vector store backend (default: qdrant)
57
+ vector_store: "qdrant"
58
+
59
+ # LLM configuration for memory operations (fact extraction)
60
+ # RECOMMENDED: Use mem0's native LLMs (no adapter overhead, no async complexity)
61
+ llm:
62
+ provider: "openai" # Options: openai, anthropic, groq, together, etc.
63
+ model: "gpt-4.1-nano-2025-04-14" # Fast and cheap model for memory ops (mem0's default)
64
+
65
+ # Embedding configuration (uses mem0's native embedders)
66
+ # RECOMMENDED: Specify provider and model for clarity
67
+ embedding:
68
+ provider: "openai" # Options: openai, together, azure_openai, gemini, huggingface, etc.
69
+ model: "text-embedding-3-small" # OpenAI's efficient embedding model
70
+
71
+ # Qdrant client configuration
72
+ # IMPORTANT: For multi-agent setups, use server mode to avoid concurrent access errors
73
+ qdrant:
74
+ mode: "server" # Options: "server" (recommended for multi-agent) or "local" (single agent only)
75
+ host: "localhost" # Qdrant server host (default: localhost)
76
+ port: 6333 # Qdrant server port (default: 6333)
77
+ # For local mode (single agent only):
78
+ # mode: "local"
79
+ # path: ".massgen/qdrant" # Local storage path
80
+
81
+ # Context window management thresholds
82
+ compression:
83
+ trigger_threshold: 0.25 # Compress when context usage exceeds 25%
84
+ target_ratio: 0.10 # Target 10% of context after compression
85
+
86
+ # Memory retrieval configuration
87
+ retrieval:
88
+ limit: 5 # Number of memory facts to retrieve from mem0 (default: 5)
89
+ exclude_recent: true # Only retrieve after compression to avoid duplicates (default: true)
90
+
91
+ # ====================
92
+ # ORCHESTRATOR CONFIGURATION
93
+ # ====================
94
+ orchestrator:
95
+ # Multi-turn mode to enable interactive storytelling
96
+ session_storage: "memory_test_sessions"
97
+
98
+ # Agent workspace for any file operations
99
+ agent_temporary_workspace: "memory_test_workspaces"
100
+ snapshot_storage: "memory_test_snapshots"
101
+
102
+ # ====================
103
+ # UI CONFIGURATION
104
+ # ====================
105
+ ui:
106
+ display_type: "rich_terminal"
107
+ logging_enabled: true