amd-gaia 0.15.0__py3-none-any.whl → 0.15.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. {amd_gaia-0.15.0.dist-info → amd_gaia-0.15.2.dist-info}/METADATA +222 -223
  2. amd_gaia-0.15.2.dist-info/RECORD +182 -0
  3. {amd_gaia-0.15.0.dist-info → amd_gaia-0.15.2.dist-info}/WHEEL +1 -1
  4. {amd_gaia-0.15.0.dist-info → amd_gaia-0.15.2.dist-info}/entry_points.txt +1 -0
  5. {amd_gaia-0.15.0.dist-info → amd_gaia-0.15.2.dist-info}/licenses/LICENSE.md +20 -20
  6. gaia/__init__.py +29 -29
  7. gaia/agents/__init__.py +19 -19
  8. gaia/agents/base/__init__.py +9 -9
  9. gaia/agents/base/agent.py +2132 -2177
  10. gaia/agents/base/api_agent.py +119 -120
  11. gaia/agents/base/console.py +1967 -1841
  12. gaia/agents/base/errors.py +237 -237
  13. gaia/agents/base/mcp_agent.py +86 -86
  14. gaia/agents/base/tools.py +88 -83
  15. gaia/agents/blender/__init__.py +7 -0
  16. gaia/agents/blender/agent.py +553 -556
  17. gaia/agents/blender/agent_simple.py +133 -135
  18. gaia/agents/blender/app.py +211 -211
  19. gaia/agents/blender/app_simple.py +41 -41
  20. gaia/agents/blender/core/__init__.py +16 -16
  21. gaia/agents/blender/core/materials.py +506 -506
  22. gaia/agents/blender/core/objects.py +316 -316
  23. gaia/agents/blender/core/rendering.py +225 -225
  24. gaia/agents/blender/core/scene.py +220 -220
  25. gaia/agents/blender/core/view.py +146 -146
  26. gaia/agents/chat/__init__.py +9 -9
  27. gaia/agents/chat/agent.py +809 -835
  28. gaia/agents/chat/app.py +1065 -1058
  29. gaia/agents/chat/session.py +508 -508
  30. gaia/agents/chat/tools/__init__.py +15 -15
  31. gaia/agents/chat/tools/file_tools.py +96 -96
  32. gaia/agents/chat/tools/rag_tools.py +1744 -1729
  33. gaia/agents/chat/tools/shell_tools.py +437 -436
  34. gaia/agents/code/__init__.py +7 -7
  35. gaia/agents/code/agent.py +549 -549
  36. gaia/agents/code/cli.py +377 -0
  37. gaia/agents/code/models.py +135 -135
  38. gaia/agents/code/orchestration/__init__.py +24 -24
  39. gaia/agents/code/orchestration/checklist_executor.py +1763 -1763
  40. gaia/agents/code/orchestration/checklist_generator.py +713 -713
  41. gaia/agents/code/orchestration/factories/__init__.py +9 -9
  42. gaia/agents/code/orchestration/factories/base.py +63 -63
  43. gaia/agents/code/orchestration/factories/nextjs_factory.py +118 -118
  44. gaia/agents/code/orchestration/factories/python_factory.py +106 -106
  45. gaia/agents/code/orchestration/orchestrator.py +841 -841
  46. gaia/agents/code/orchestration/project_analyzer.py +391 -391
  47. gaia/agents/code/orchestration/steps/__init__.py +67 -67
  48. gaia/agents/code/orchestration/steps/base.py +188 -188
  49. gaia/agents/code/orchestration/steps/error_handler.py +314 -314
  50. gaia/agents/code/orchestration/steps/nextjs.py +828 -828
  51. gaia/agents/code/orchestration/steps/python.py +307 -307
  52. gaia/agents/code/orchestration/template_catalog.py +469 -469
  53. gaia/agents/code/orchestration/workflows/__init__.py +14 -14
  54. gaia/agents/code/orchestration/workflows/base.py +80 -80
  55. gaia/agents/code/orchestration/workflows/nextjs.py +186 -186
  56. gaia/agents/code/orchestration/workflows/python.py +94 -94
  57. gaia/agents/code/prompts/__init__.py +11 -11
  58. gaia/agents/code/prompts/base_prompt.py +77 -77
  59. gaia/agents/code/prompts/code_patterns.py +2034 -2036
  60. gaia/agents/code/prompts/nextjs_prompt.py +40 -40
  61. gaia/agents/code/prompts/python_prompt.py +109 -109
  62. gaia/agents/code/schema_inference.py +365 -365
  63. gaia/agents/code/system_prompt.py +41 -41
  64. gaia/agents/code/tools/__init__.py +42 -42
  65. gaia/agents/code/tools/cli_tools.py +1138 -1138
  66. gaia/agents/code/tools/code_formatting.py +319 -319
  67. gaia/agents/code/tools/code_tools.py +769 -769
  68. gaia/agents/code/tools/error_fixing.py +1347 -1347
  69. gaia/agents/code/tools/external_tools.py +180 -180
  70. gaia/agents/code/tools/file_io.py +845 -845
  71. gaia/agents/code/tools/prisma_tools.py +190 -190
  72. gaia/agents/code/tools/project_management.py +1016 -1016
  73. gaia/agents/code/tools/testing.py +321 -321
  74. gaia/agents/code/tools/typescript_tools.py +122 -122
  75. gaia/agents/code/tools/validation_parsing.py +461 -461
  76. gaia/agents/code/tools/validation_tools.py +806 -806
  77. gaia/agents/code/tools/web_dev_tools.py +1758 -1758
  78. gaia/agents/code/validators/__init__.py +16 -16
  79. gaia/agents/code/validators/antipattern_checker.py +241 -241
  80. gaia/agents/code/validators/ast_analyzer.py +197 -197
  81. gaia/agents/code/validators/requirements_validator.py +145 -145
  82. gaia/agents/code/validators/syntax_validator.py +171 -171
  83. gaia/agents/docker/__init__.py +7 -7
  84. gaia/agents/docker/agent.py +643 -642
  85. gaia/agents/emr/__init__.py +8 -8
  86. gaia/agents/emr/agent.py +1504 -1506
  87. gaia/agents/emr/cli.py +1322 -1322
  88. gaia/agents/emr/constants.py +475 -475
  89. gaia/agents/emr/dashboard/__init__.py +4 -4
  90. gaia/agents/emr/dashboard/server.py +1972 -1974
  91. gaia/agents/jira/__init__.py +11 -11
  92. gaia/agents/jira/agent.py +894 -894
  93. gaia/agents/jira/jql_templates.py +299 -299
  94. gaia/agents/routing/__init__.py +7 -7
  95. gaia/agents/routing/agent.py +567 -570
  96. gaia/agents/routing/system_prompt.py +75 -75
  97. gaia/agents/summarize/__init__.py +11 -0
  98. gaia/agents/summarize/agent.py +885 -0
  99. gaia/agents/summarize/prompts.py +129 -0
  100. gaia/api/__init__.py +23 -23
  101. gaia/api/agent_registry.py +238 -238
  102. gaia/api/app.py +305 -305
  103. gaia/api/openai_server.py +575 -575
  104. gaia/api/schemas.py +186 -186
  105. gaia/api/sse_handler.py +373 -373
  106. gaia/apps/__init__.py +4 -4
  107. gaia/apps/llm/__init__.py +6 -6
  108. gaia/apps/llm/app.py +184 -169
  109. gaia/apps/summarize/app.py +116 -633
  110. gaia/apps/summarize/html_viewer.py +133 -133
  111. gaia/apps/summarize/pdf_formatter.py +284 -284
  112. gaia/audio/__init__.py +2 -2
  113. gaia/audio/audio_client.py +439 -439
  114. gaia/audio/audio_recorder.py +269 -269
  115. gaia/audio/kokoro_tts.py +599 -599
  116. gaia/audio/whisper_asr.py +432 -432
  117. gaia/chat/__init__.py +16 -16
  118. gaia/chat/app.py +428 -430
  119. gaia/chat/prompts.py +522 -522
  120. gaia/chat/sdk.py +1228 -1225
  121. gaia/cli.py +5659 -5632
  122. gaia/database/__init__.py +10 -10
  123. gaia/database/agent.py +176 -176
  124. gaia/database/mixin.py +290 -290
  125. gaia/database/testing.py +64 -64
  126. gaia/eval/batch_experiment.py +2332 -2332
  127. gaia/eval/claude.py +542 -542
  128. gaia/eval/config.py +37 -37
  129. gaia/eval/email_generator.py +512 -512
  130. gaia/eval/eval.py +3179 -3179
  131. gaia/eval/groundtruth.py +1130 -1130
  132. gaia/eval/transcript_generator.py +582 -582
  133. gaia/eval/webapp/README.md +167 -167
  134. gaia/eval/webapp/package-lock.json +875 -875
  135. gaia/eval/webapp/package.json +20 -20
  136. gaia/eval/webapp/public/app.js +3402 -3402
  137. gaia/eval/webapp/public/index.html +87 -87
  138. gaia/eval/webapp/public/styles.css +3661 -3661
  139. gaia/eval/webapp/server.js +415 -415
  140. gaia/eval/webapp/test-setup.js +72 -72
  141. gaia/installer/__init__.py +23 -0
  142. gaia/installer/init_command.py +1275 -0
  143. gaia/installer/lemonade_installer.py +619 -0
  144. gaia/llm/__init__.py +10 -2
  145. gaia/llm/base_client.py +60 -0
  146. gaia/llm/exceptions.py +12 -0
  147. gaia/llm/factory.py +70 -0
  148. gaia/llm/lemonade_client.py +3421 -3221
  149. gaia/llm/lemonade_manager.py +294 -294
  150. gaia/llm/providers/__init__.py +9 -0
  151. gaia/llm/providers/claude.py +108 -0
  152. gaia/llm/providers/lemonade.py +118 -0
  153. gaia/llm/providers/openai_provider.py +79 -0
  154. gaia/llm/vlm_client.py +382 -382
  155. gaia/logger.py +189 -189
  156. gaia/mcp/agent_mcp_server.py +245 -245
  157. gaia/mcp/blender_mcp_client.py +138 -138
  158. gaia/mcp/blender_mcp_server.py +648 -648
  159. gaia/mcp/context7_cache.py +332 -332
  160. gaia/mcp/external_services.py +518 -518
  161. gaia/mcp/mcp_bridge.py +811 -550
  162. gaia/mcp/servers/__init__.py +6 -6
  163. gaia/mcp/servers/docker_mcp.py +83 -83
  164. gaia/perf_analysis.py +361 -0
  165. gaia/rag/__init__.py +10 -10
  166. gaia/rag/app.py +293 -293
  167. gaia/rag/demo.py +304 -304
  168. gaia/rag/pdf_utils.py +235 -235
  169. gaia/rag/sdk.py +2194 -2194
  170. gaia/security.py +183 -163
  171. gaia/talk/app.py +287 -289
  172. gaia/talk/sdk.py +538 -538
  173. gaia/testing/__init__.py +87 -87
  174. gaia/testing/assertions.py +330 -330
  175. gaia/testing/fixtures.py +333 -333
  176. gaia/testing/mocks.py +493 -493
  177. gaia/util.py +46 -46
  178. gaia/utils/__init__.py +33 -33
  179. gaia/utils/file_watcher.py +675 -675
  180. gaia/utils/parsing.py +223 -223
  181. gaia/version.py +100 -100
  182. amd_gaia-0.15.0.dist-info/RECORD +0 -168
  183. gaia/agents/code/app.py +0 -266
  184. gaia/llm/llm_client.py +0 -723
  185. {amd_gaia-0.15.0.dist-info → amd_gaia-0.15.2.dist-info}/top_level.txt +0 -0
gaia/agents/chat/app.py CHANGED
@@ -1,1058 +1,1065 @@
1
- # Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved.
2
- # SPDX-License-Identifier: MIT
3
- """
4
- Chat Agent Application - Interactive chat with RAG and file search.
5
- """
6
-
7
- import argparse
8
- import os
9
- import sys
10
- from pathlib import Path
11
-
12
- from gaia.agents.chat.agent import ChatAgent, ChatAgentConfig
13
- from gaia.logger import get_logger
14
-
15
- logger = get_logger(__name__)
16
-
17
-
18
- def parse_args():
19
- """Parse command line arguments."""
20
- parser = argparse.ArgumentParser(
21
- description="Chat Agent with RAG and file search capabilities"
22
- )
23
-
24
- # LLM backend options
25
- parser.add_argument(
26
- "--use-claude", action="store_true", help="Use Claude API instead of local LLM"
27
- )
28
- parser.add_argument(
29
- "--use-chatgpt",
30
- action="store_true",
31
- help="Use ChatGPT/OpenAI API instead of local LLM",
32
- )
33
- parser.add_argument(
34
- "--claude-model",
35
- type=str,
36
- default="claude-sonnet-4-20250514",
37
- help="Claude model to use (default: claude-sonnet-4-20250514)",
38
- )
39
- parser.add_argument(
40
- "--model-id",
41
- type=str,
42
- default=None,
43
- help="Model ID for local LLM (default: Qwen3-Coder-30B-A3B-Instruct-GGUF)",
44
- )
45
-
46
- # Agent configuration
47
- parser.add_argument(
48
- "--max-steps",
49
- type=int,
50
- default=10,
51
- help="Maximum conversation steps (default: 10)",
52
- )
53
- parser.add_argument(
54
- "--streaming", action="store_true", help="Enable streaming responses"
55
- )
56
- parser.add_argument(
57
- "--show-stats", action="store_true", help="Show performance statistics"
58
- )
59
- parser.add_argument(
60
- "--show-prompts", action="store_true", help="Display prompts sent to LLM"
61
- )
62
- parser.add_argument("--debug", action="store_true", help="Enable debug output")
63
- parser.add_argument(
64
- "--silent",
65
- action="store_true",
66
- help="Suppress agent reasoning output (silent mode)",
67
- )
68
-
69
- # RAG configuration
70
- parser.add_argument(
71
- "--index",
72
- "-i",
73
- type=str,
74
- metavar="PATH",
75
- help="Index a document before running (combine with --query for one-shot usage)",
76
- )
77
- parser.add_argument(
78
- "--documents", "-d", nargs="+", help="Documents to index for RAG"
79
- )
80
- parser.add_argument(
81
- "--watch", "-w", nargs="+", help="Directories to monitor for new documents"
82
- )
83
- parser.add_argument(
84
- "--chunk-size", type=int, default=500, help="Document chunk size (default: 500)"
85
- )
86
- parser.add_argument(
87
- "--chunk-overlap",
88
- type=int,
89
- default=100,
90
- help="Chunk overlap in characters (default: 100)",
91
- )
92
- parser.add_argument(
93
- "--max-chunks",
94
- type=int,
95
- default=5,
96
- help="Maximum chunks to retrieve (default: 5)",
97
- )
98
- parser.add_argument(
99
- "--use-llm-chunking",
100
- action="store_true",
101
- help="Use LLM-based semantic chunking for better context preservation (slower but more accurate)",
102
- )
103
- parser.add_argument(
104
- "--allowed-paths",
105
- nargs="+",
106
- help="Allowed directory paths for file operations (default: current directory)",
107
- )
108
-
109
- # Input/output
110
- parser.add_argument(
111
- "--query", "-q", type=str, help="Single query to execute (non-interactive mode)"
112
- )
113
- parser.add_argument("--output-dir", type=str, help="Directory for output files")
114
- parser.add_argument(
115
- "--list-tools", action="store_true", help="List available tools and exit"
116
- )
117
-
118
- return parser.parse_args()
119
-
120
-
121
- def interactive_mode(agent: ChatAgent):
122
- """Run agent in interactive chat mode."""
123
- print("=" * 60)
124
- print("Chat Agent with RAG - Interactive Mode")
125
- print("=" * 60)
126
-
127
- # Display model information
128
- model_name = getattr(agent, "model_display_name", "Unknown")
129
- print(f"\n🤖 Model: {model_name}")
130
-
131
- print("\nCapabilities:")
132
- print(" • Document Q&A using RAG")
133
- print(" - Documents: PDF, TXT, MD, CSV, JSON")
134
- print(" - Backend: Python, Java, C/C++, Go, Rust, Ruby, PHP, Swift, etc.")
135
- print(" - Web: JS/TS, HTML, CSS/SCSS/SASS, Vue, Svelte, React (JSX/TSX)")
136
- print(" - Config: YAML, XML, TOML, INI, ENV")
137
- print(" • Document summarization with multiple styles")
138
- print(" • Code retrieval and search (no code generation)")
139
- print(" • File search and operations")
140
- print(" • Shell command execution")
141
- print(" • Auto-indexing when files change")
142
- print(" • Session persistence with auto-save")
143
- print("\nSession Commands:")
144
- print(" /resume [id] - Resume a previous session (or list if no id)")
145
- print(" /save - Save current session")
146
- print(" /sessions - List all available sessions")
147
- print(" /reset - Clear conversation and start fresh")
148
- print("\nDocument Commands:")
149
- print(" /index <path> - Index a document or directory")
150
- print(" /watch <dir> - Watch a directory for changes")
151
- print(" /list - List indexed documents")
152
- print(" /status - Show system status")
153
- print("\nDebug Commands:")
154
- print(" /chunks <file> - Show all chunks for a document")
155
- print(" /chunk <n> - Show specific chunk content")
156
- print(" /test <query> - Test query retrieval with scores")
157
- print(
158
- " /dump <file|#> - Export document + chunks to markdown (use number from /list)"
159
- )
160
- print(" /clear-cache - Clear RAG cache (force re-indexing)")
161
- print(" /search-debug - Toggle RAG debug mode")
162
- print("\nOther Commands:")
163
- print(" /help or /? - Show this help")
164
- print(" /quit - Exit")
165
- print("\nOr just type your question to chat with indexed documents.")
166
- print("=" * 60)
167
-
168
- while True:
169
- try:
170
- user_input = input("\nYou: ").strip()
171
-
172
- if not user_input:
173
- continue
174
-
175
- # Handle commands
176
- if user_input.startswith("/"):
177
- parts = user_input.split(maxsplit=1)
178
- command = parts[0].lower()
179
- arg = parts[1] if len(parts) > 1 else None
180
-
181
- if command == "/quit":
182
- # Auto-save before quitting
183
- if agent.current_session:
184
- print("\n💾 Saving current session...")
185
- agent.save_current_session()
186
- print("Goodbye!")
187
- break
188
-
189
- elif command in ["/help", "/?"]:
190
- print("\n" + "=" * 60)
191
- print("Chat Agent - Available Commands & Capabilities")
192
- print("=" * 60)
193
- print("\n🎯 CAPABILITIES:")
194
- print(" • Document Q&A using RAG")
195
- print(" - Documents: PDF, TXT, MD, CSV, JSON")
196
- print(
197
- " - Backend: Python, Java, C/C++, Go, Rust, Ruby, PHP, Swift, Kotlin, Scala"
198
- )
199
- print(
200
- " - Web: JS/TS, HTML, CSS/SCSS/SASS/LESS, Vue, Svelte, Astro, React"
201
- )
202
- print(" - Config: YAML, XML, TOML, INI, ENV, Properties")
203
- print(" - Build: Gradle, CMake, Makefiles")
204
- print(" - Database: SQL")
205
- print(
206
- " • Document summarization (brief, detailed, bullets, executive, etc.)"
207
- )
208
- print(" • Code retrieval and search (no code generation)")
209
- print(" • File operations and search")
210
- print(" • Shell command execution (ls, grep, find, etc.)")
211
- print(" • Auto-indexing when files change")
212
- print(" • Session persistence with auto-save")
213
- print("\n📋 SESSION MANAGEMENT:")
214
- print(" /resume [id] - Resume session (or list if no id)")
215
- print(" /save - Save current session")
216
- print(" /sessions - List all sessions")
217
- print(" /reset - Clear and start fresh")
218
- print("\n📚 DOCUMENT MANAGEMENT:")
219
- print(" /index <path> - Index file or directory")
220
- print(" /watch <dir> - Watch directory for changes")
221
- print(" /list - List indexed documents")
222
- print(" /status - Show system status")
223
- print("\n🔍 DEBUG & OBSERVABILITY:")
224
- print(" /chunks <file> - Show all chunks for a document")
225
- print(" /chunk <n> - Show specific chunk content")
226
- print(" /test <query> - Test query retrieval with scores")
227
- print(
228
- " /dump <file|#> - Export document + chunks to markdown (use number from /list)"
229
- )
230
- print(" /clear-cache - Clear RAG cache (force re-indexing)")
231
- print(" /search-debug - Toggle RAG debug mode")
232
- print("\n💬 EXAMPLE QUERIES:")
233
- print(" Documents:")
234
- print(" 'What does the document say about X?'")
235
- print(" 'Summarize report.pdf in bullet points'")
236
- print(" Backend Code:")
237
- print(" 'Where is the UserAuth class defined?'")
238
- print(" 'Find all functions that use database connections'")
239
- print(" 'What does the authenticate() function do?'")
240
- print(" Web Development:")
241
- print(" 'Find all CSS classes with hover effects'")
242
- print(" 'Where are the API endpoints defined?'")
243
- print(" 'Show me all Vue components that use props'")
244
- print(" General:")
245
- print(" 'List all Python files in src/'")
246
- print(" 'Search for TODO comments in my code'")
247
- print(" 'Find all files that import React'")
248
- print("\n⚙️ OTHER COMMANDS:")
249
- print(" /help or /? - Show this help")
250
- print(" /quit - Exit (auto-saves)")
251
- print("=" * 60)
252
-
253
- elif command == "/resume":
254
- if not arg:
255
- # List available sessions
256
- sessions = agent.session_manager.list_sessions()
257
- if not sessions:
258
- print("\n📂 No saved sessions found.")
259
- else:
260
- print("\n" + "=" * 60)
261
- print("Available Sessions")
262
- print("=" * 60)
263
- for sess in sessions:
264
- print(f"\n ID: {sess['session_id']}")
265
- print(f" Created: {sess['created_at']}")
266
- print(f" Updated: {sess['updated_at']}")
267
- print(f" Documents: {sess['num_documents']}")
268
- print(f" Messages: {sess['num_messages']}")
269
- print("\n" + "=" * 60)
270
- print("\nUse: /resume <session_id> to load a session")
271
- else:
272
- # Resume specific session
273
- print(f"\n📂 Resuming session: {arg}")
274
- if agent.load_session(arg):
275
- session = agent.current_session
276
- print("✅ Loaded session with:")
277
- print(f" - {len(session.indexed_documents)} documents")
278
- print(
279
- f" - {len(session.watched_directories)} watched directories"
280
- )
281
- print(f" - {len(session.chat_history)} chat messages")
282
- else:
283
- print(f"❌ Failed to load session: {arg}")
284
-
285
- elif command == "/save":
286
- print("\n💾 Saving current session...")
287
- if agent.save_current_session():
288
- session_id = (
289
- agent.current_session.session_id
290
- if agent.current_session
291
- else "unknown"
292
- )
293
- print(f"✅ Session saved: {session_id}")
294
- else:
295
- print("❌ Failed to save session")
296
-
297
- elif command == "/sessions":
298
- sessions = agent.session_manager.list_sessions()
299
- if not sessions:
300
- print("\n📂 No saved sessions found.")
301
- else:
302
- print("\n" + "=" * 60)
303
- print(f"Found {len(sessions)} Session(s)")
304
- print("=" * 60)
305
- for i, sess in enumerate(sessions, 1):
306
- print(f"\n{i}. {sess['session_id']}")
307
- print(f" Created: {sess['created_at']}")
308
- print(
309
- f" Documents: {sess['num_documents']}, Messages: {sess['num_messages']}"
310
- )
311
- print("\n" + "=" * 60)
312
-
313
- elif command == "/reset":
314
- print("\n🔄 Resetting conversation...")
315
- # Save current session first
316
- if agent.current_session:
317
- agent.save_current_session()
318
- # Create new session
319
- agent.current_session = agent.session_manager.create_session()
320
- # Clear chat history (if agent tracks it)
321
- if hasattr(agent, "chat_history"):
322
- agent.chat_history = []
323
- print("✅ Conversation reset. Previous session saved.")
324
- print(f" New session: {agent.current_session.session_id}")
325
-
326
- elif command == "/index":
327
- if not arg:
328
- print("Usage: /index <file_or_directory_path>")
329
- print("Example: /index /path/to/document.pdf")
330
- print(" /index /path/to/documents/")
331
- continue
332
-
333
- # Check if it's a directory or file
334
- path = Path(arg)
335
-
336
- if path.is_dir():
337
- print(f"\n📁 Indexing all documents in directory: {arg}")
338
- # Find all supported document types
339
- doc_patterns = ["*.pdf", "*.txt", "*.md", "*.csv", "*.json"]
340
- doc_files = []
341
- for pattern in doc_patterns:
342
- doc_files.extend(path.glob(pattern))
343
-
344
- if not doc_files:
345
- print("❌ No supported documents found in directory")
346
- print(" Supported types: PDF, TXT, MD, CSV, JSON")
347
- continue
348
-
349
- print(f"Found {len(doc_files)} document(s)\n")
350
- success_count = 0
351
- for doc_file in doc_files:
352
- print(f" 📄 Indexing: {doc_file.name}...")
353
- # Directly call the RAG index method
354
- try:
355
- result = agent.rag.index_document(
356
- str(doc_file.absolute())
357
- )
358
- if result.get("success"):
359
- print(
360
- f" ✅ Success: {result.get('num_chunks', 0)} chunks created"
361
- )
362
- success_count += 1
363
- else:
364
- error = result.get("error", "Unknown error")
365
- print(f" ❌ Failed: {error}")
366
- except Exception as e:
367
- print(f" ❌ Error: {e}")
368
-
369
- print(
370
- f"\n📊 Summary: {success_count}/{len(doc_files)} documents indexed successfully"
371
- )
372
-
373
- else:
374
- # Single file
375
- if not os.path.exists(arg):
376
- print(f"\n❌ File not found: {arg}")
377
- print(" Please check the file path and try again")
378
- continue
379
-
380
- print(f"\n📄 Indexing: {path.name}")
381
- print("=" * 60)
382
-
383
- try:
384
- # Directly call the RAG index method
385
- result = agent.rag.index_document(str(path.absolute()))
386
-
387
- if result.get("success"):
388
- # Display success with detailed stats
389
- print("✅ INDEXING SUCCESSFUL")
390
- print("=" * 60)
391
- print(f"📁 File: {result.get('file_name', path.name)}")
392
- print(f"📄 Type: {result.get('file_type', 'Unknown')}")
393
- print(
394
- f"💾 Size: {result.get('file_size_mb', 0):.2f} MB"
395
- )
396
-
397
- # Show num_pages for PDFs
398
- if result.get("num_pages"):
399
- print(f"📖 Pages: {result['num_pages']}")
400
-
401
- print(
402
- f"📦 Chunks Created: {result.get('num_chunks', 0)}"
403
- )
404
-
405
- # Show cache/reindex status
406
- if result.get("from_cache"):
407
- print("⚡ Loaded from cache (fast)")
408
- elif result.get("already_indexed"):
409
- print("ℹ️ Already indexed (skipped)")
410
- elif result.get("reindexed"):
411
- print("🔄 Reindexed (updated)")
412
-
413
- print("\n📊 GLOBAL STATISTICS")
414
- print("=" * 60)
415
- print(
416
- f"Total Documents Indexed: {result.get('total_indexed_files', 0)}"
417
- )
418
- print(f"Total Chunks: {result.get('total_chunks', 0)}")
419
- print("=" * 60)
420
- else:
421
- # Display error
422
- print(" INDEXING FAILED")
423
- print("=" * 60)
424
- error = result.get("error", "Unknown error")
425
- print(f"Error: {error}")
426
- if result.get("file_name"):
427
- print(f"File: {result['file_name']}")
428
- print("=" * 60)
429
-
430
- except Exception as e:
431
- print(" INDEXING FAILED")
432
- print("=" * 60)
433
- print(f"Error: {e}")
434
- print("=" * 60)
435
-
436
- elif command == "/watch":
437
- if not arg:
438
- print("Usage: /watch <directory_path>")
439
- print("Example: /watch /path/to/documents")
440
- continue
441
- result = agent.process_query(f"Watch the directory: {arg}")
442
- print(f"\n{result['result']}")
443
-
444
- elif command == "/list":
445
- # Directly access indexed files from RAG system
446
- print("\n" + "=" * 60)
447
- print("📚 INDEXED DOCUMENTS")
448
- print("=" * 60)
449
-
450
- if not agent.rag.indexed_files:
451
- print("No documents indexed yet.")
452
- print("\nUse /index <path> to index a document")
453
- else:
454
- print(f"Total: {len(agent.rag.indexed_files)} document(s)\n")
455
- for i, file_path in enumerate(
456
- sorted(agent.rag.indexed_files), 1
457
- ):
458
- file_name = Path(file_path).name
459
- file_type = Path(file_path).suffix
460
-
461
- # Get chunk count for this file if available
462
- num_chunks = 0
463
- if file_path in agent.rag.file_to_chunk_indices:
464
- num_chunks = len(
465
- agent.rag.file_to_chunk_indices[file_path]
466
- )
467
-
468
- print(f"{i}. {file_name}")
469
- print(f" Type: {file_type}")
470
- print(f" Chunks: {num_chunks}")
471
- print(f" Path: {file_path}")
472
- print()
473
-
474
- print("=" * 60)
475
-
476
- elif command == "/status":
477
- # Directly access RAG status
478
- print("\n" + "=" * 60)
479
- print("📊 RAG SYSTEM STATUS")
480
- print("=" * 60)
481
-
482
- status = agent.rag.get_status()
483
-
484
- print("\n📚 Documents:")
485
- print(f" Indexed Files: {status['indexed_files']}")
486
- print(f" Total Chunks: {status['total_chunks']}")
487
-
488
- print("\n⚙️ Configuration:")
489
- print(f" Chunk Size: {status['config']['chunk_size']} tokens")
490
- print(
491
- f" Chunk Overlap: {status['config']['chunk_overlap']} tokens"
492
- )
493
- print(f" Max Chunks per Query: {status['config']['max_chunks']}")
494
-
495
- print("\n💾 Storage:")
496
- print(f" Cache Directory: {status['cache_dir']}")
497
- print(f" Embedding Model: {status['embedding_model']}")
498
-
499
- # Show watched directories
500
- if agent.watch_directories:
501
- print("\n👀 Watched Directories:")
502
- for i, dir_path in enumerate(agent.watch_directories, 1):
503
- print(f" {i}. {dir_path}")
504
-
505
- # Show session info
506
- if agent.current_session:
507
- print("\n📋 Current Session:")
508
- print(f" ID: {agent.current_session.session_id}")
509
- print(f" Created: {agent.current_session.created_at}")
510
- print(f" Updated: {agent.current_session.updated_at}")
511
- print(
512
- f" Documents: {len(agent.current_session.indexed_documents)}"
513
- )
514
- print(f" Messages: {len(agent.current_session.chat_history)}")
515
-
516
- print("\n" + "=" * 60)
517
-
518
- elif command == "/chunks":
519
- if not arg:
520
- print("Usage: /chunks <filename>")
521
- print("Example: /chunks document.pdf")
522
- continue
523
-
524
- # Find file in indexed files
525
- matching_files = [
526
- f
527
- for f in agent.rag.indexed_files
528
- if Path(f).name == arg or f == arg
529
- ]
530
-
531
- if not matching_files:
532
- print(f"\n❌ File not indexed: {arg}")
533
- print("\nIndexed files:")
534
- for f in agent.rag.indexed_files:
535
- print(f" - {Path(f).name}")
536
- continue
537
-
538
- file_path = matching_files[0]
539
-
540
- # Get chunks for this file
541
- if file_path not in agent.rag.file_to_chunk_indices:
542
- print(f"\n❌ No chunks found for: {arg}")
543
- continue
544
-
545
- chunk_indices = agent.rag.file_to_chunk_indices[file_path]
546
-
547
- print("\n" + "=" * 60)
548
- print(f"DOCUMENT CHUNKS: {Path(file_path).name}")
549
- print("=" * 60)
550
- print(f"Total Chunks: {len(chunk_indices)}")
551
- print(f"Chunk Size: {agent.rag.config.chunk_size} tokens")
552
- print(f"Overlap: {agent.rag.config.chunk_overlap} tokens\n")
553
-
554
- for i, chunk_idx in enumerate(chunk_indices, 1):
555
- if chunk_idx < len(agent.rag.chunks):
556
- chunk = agent.rag.chunks[chunk_idx]
557
- token_count = len(chunk) // 4 # Rough estimate
558
-
559
- print(f"CHUNK {i} ({token_count} tokens):")
560
- print("┌" + "─" * 58 + "┐")
561
-
562
- # Show first 200 chars of chunk
563
- preview = chunk[:200].replace("\n", " ")
564
- if len(chunk) > 200:
565
- preview += "..."
566
-
567
- # Word wrap the preview
568
- import textwrap
569
-
570
- wrapped = textwrap.wrap(preview, width=56)
571
- for line in wrapped:
572
- print(f"│ {line:<56} ")
573
-
574
- print("└" + "─" * 58 + "┘")
575
- print()
576
-
577
- print("=" * 60)
578
- print("\nUse '/chunk <n>' to see full content of a specific chunk")
579
-
580
- elif command == "/chunk":
581
- if not arg or not arg.isdigit():
582
- print("Usage: /chunk <number>")
583
- print("Example: /chunk 5")
584
- continue
585
-
586
- chunk_num = int(arg) - 1 # Convert to 0-indexed
587
-
588
- if chunk_num < 0 or chunk_num >= len(agent.rag.chunks):
589
- print(
590
- f"\n❌ Invalid chunk number. Valid range: 1-{len(agent.rag.chunks)}"
591
- )
592
- continue
593
-
594
- chunk = agent.rag.chunks[chunk_num]
595
-
596
- # Find which file this chunk belongs to
597
- source_file = agent.rag.chunk_to_file.get(chunk_num, "Unknown")
598
-
599
- print("\n" + "=" * 60)
600
- print(f"CHUNK {arg}")
601
- print("=" * 60)
602
- print(
603
- f"Source: {Path(source_file).name if source_file != 'Unknown' else 'Unknown'}"
604
- )
605
- print(f"Token count: {len(chunk) // 4} (estimated)")
606
- print(f"Character count: {len(chunk):,}")
607
- print("\nCONTENT:")
608
- print("" + "─" * 58 + "┐")
609
-
610
- # Word wrap content
611
- import textwrap
612
-
613
- wrapped = textwrap.wrap(chunk, width=56)
614
- for line in wrapped:
615
- print(f" {line:<56} ")
616
-
617
- print("└" + "─" * 58 + "┘")
618
- print("=" * 60)
619
-
620
- elif command == "/test":
621
- if not arg:
622
- print("Usage: /test <query>")
623
- print("Example: /test what is the vision?")
624
- continue
625
-
626
- if not agent.rag.indexed_files:
627
- print("\n❌ No documents indexed. Use /index <path> first.")
628
- continue
629
-
630
- print("\n" + "=" * 60)
631
- print(f'QUERY TEST: "{arg}"')
632
- print("=" * 60)
633
-
634
- # Test retrieval
635
- try:
636
- # pylint: disable=protected-access
637
- chunks, scores = agent.rag._retrieve_chunks(arg)
638
-
639
- print(f"\nTop {len(chunks)} matching chunks:\n")
640
-
641
- for i, (chunk, score) in enumerate(zip(chunks, scores), 1):
642
- # Find chunk index
643
- chunk_idx = (
644
- agent.rag.chunks.index(chunk)
645
- if chunk in agent.rag.chunks
646
- else -1
647
- )
648
- source_file = agent.rag.chunk_to_file.get(
649
- chunk_idx, "Unknown"
650
- )
651
-
652
- # Color code by score
653
- if score > 0.75:
654
- emoji = "⭐"
655
- elif score > 0.60:
656
- emoji = ""
657
- else:
658
- emoji = "⚠️"
659
-
660
- print(
661
- f"{i}. {emoji} CHUNK {chunk_idx + 1} (score: {score:.3f})"
662
- )
663
- print(
664
- f" Source: {Path(source_file).name if source_file != 'Unknown' else 'Unknown'}"
665
- )
666
-
667
- # Preview
668
- preview = chunk[:150].replace("\n", " ")
669
- if len(chunk) > 150:
670
- preview += "..."
671
- print(f" Preview: {preview}")
672
- print()
673
-
674
- print("=" * 60)
675
- print("\nScore Legend:")
676
- print(" ⭐ Excellent (>0.75) - Highly relevant")
677
- print(" ✅ Good (0.60-0.75) - Probably relevant")
678
- print(" ⚠️ Poor (<0.60) - May not be relevant")
679
-
680
- except Exception as e:
681
- print(f"\n❌ Query test failed: {e}")
682
-
683
- elif command == "/dump":
684
- if not arg:
685
- print("Usage: /dump <filename_or_number>")
686
- print("Example: /dump document.pdf")
687
- print("Example: /dump 1")
688
- continue
689
-
690
- # Find file in indexed files
691
- from datetime import datetime
692
-
693
- file_path = None
694
-
695
- # Check if arg is a number (index)
696
- if arg.isdigit():
697
- index = int(arg) - 1 # Convert to 0-based index
698
- indexed_list = list(agent.rag.indexed_files)
699
- if 0 <= index < len(indexed_list):
700
- file_path = indexed_list[index]
701
- else:
702
- print(f"\n❌ Invalid index: {arg}")
703
- print(f"Valid range: 1-{len(indexed_list)}")
704
- continue
705
- else:
706
- # Try to match by filename
707
- matching_files = [
708
- f
709
- for f in agent.rag.indexed_files
710
- if Path(f).name == arg or f == arg
711
- ]
712
-
713
- if not matching_files:
714
- print(f"\n❌ File not indexed: {arg}")
715
- print("\nIndexed files:")
716
- for i, f in enumerate(agent.rag.indexed_files, 1):
717
- print(f" {i}. {Path(f).name}")
718
- continue
719
-
720
- file_path = matching_files[0]
721
-
722
- # Get chunks for this file
723
- if file_path not in agent.rag.file_to_chunk_indices:
724
- print(f"\n❌ No chunks found for: {arg}")
725
- continue
726
-
727
- chunk_indices = agent.rag.file_to_chunk_indices[file_path]
728
-
729
- print(f"\n📝 Generating markdown dump for: {Path(file_path).name}")
730
-
731
- # Try to get cached metadata first (faster!)
732
- cached_metadata = agent.rag.file_metadata.get(file_path, {})
733
- extracted_text = cached_metadata.get("full_text")
734
- num_pages = cached_metadata.get("num_pages")
735
- vlm_pages = cached_metadata.get("vlm_pages", 0)
736
- total_images = cached_metadata.get("total_images", 0)
737
-
738
- # Fall back to re-extraction only if cache is missing
739
- if not extracted_text:
740
- print(" ⚠️ No cached text found, re-extracting from file...")
741
- try:
742
- # pylint: disable=protected-access
743
- extracted_text, metadata = (
744
- agent.rag._extract_text_from_file(file_path)
745
- )
746
- num_pages = metadata.get("num_pages")
747
- vlm_pages = metadata.get("vlm_pages", 0)
748
- total_images = metadata.get("total_images", 0)
749
- except Exception as e:
750
- print(f"❌ Failed to extract text: {e}")
751
- continue
752
- else:
753
- print(
754
- " ✅ Using cached text and metadata (no re-extraction needed)"
755
- )
756
-
757
- # Create markdown content
758
- markdown_lines = []
759
- markdown_lines.append(f"# RAG Debug Dump: {Path(file_path).name}")
760
- markdown_lines.append(
761
- f"\nGenerated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n"
762
- )
763
-
764
- # Metadata
765
- markdown_lines.append("## Document Metadata\n")
766
- markdown_lines.append(f"- **File:** {Path(file_path).name}")
767
- markdown_lines.append(f"- **Path:** {file_path}")
768
- file_size = os.path.getsize(file_path) / (1024 * 1024)
769
- markdown_lines.append(f"- **Size:** {file_size:.2f} MB")
770
- if num_pages:
771
- markdown_lines.append(f"- **Pages:** {num_pages}")
772
- if vlm_pages and vlm_pages > 0:
773
- markdown_lines.append(
774
- f"- **VLM Enhanced Pages:** {vlm_pages} (images processed)"
775
- )
776
- if total_images and total_images > 0:
777
- markdown_lines.append(
778
- f"- **Total Images Extracted:** {total_images}"
779
- )
780
- markdown_lines.append(f"- **Characters:** {len(extracted_text):,}")
781
- markdown_lines.append(f"- **Chunks:** {len(chunk_indices)}")
782
- markdown_lines.append(
783
- f"- **Chunk Size:** {agent.rag.config.chunk_size} tokens"
784
- )
785
- markdown_lines.append(
786
- f"- **Chunk Overlap:** {agent.rag.config.chunk_overlap} tokens"
787
- )
788
- markdown_lines.append("")
789
-
790
- # Full document text
791
- markdown_lines.append("---\n")
792
- markdown_lines.append("## Full Document Text (As Extracted)")
793
- markdown_lines.append("\n```")
794
- markdown_lines.append(extracted_text)
795
- markdown_lines.append("```\n")
796
-
797
- # Chunked version with boundaries
798
- markdown_lines.append("---\n")
799
- markdown_lines.append("## Document with Chunk Boundaries\n")
800
-
801
- for i, chunk_idx in enumerate(chunk_indices, 1):
802
- if chunk_idx < len(agent.rag.chunks):
803
- chunk = agent.rag.chunks[chunk_idx]
804
-
805
- # Extract page number from chunk (with lookback)
806
- from gaia.agents.chat.tools.rag_tools import (
807
- extract_page_from_chunk,
808
- )
809
-
810
- page_num = extract_page_from_chunk(
811
- chunk, chunk_idx, agent.rag.chunks
812
- )
813
- page_info = (
814
- f"Page {page_num}" if page_num else "Page Unknown"
815
- )
816
-
817
- markdown_lines.append(
818
- f"### 🔷 CHUNK {i} ({page_info}, Index: {chunk_idx})\n"
819
- )
820
- markdown_lines.append(
821
- f"- **Token Count:** {len(chunk) // 4} (estimated)"
822
- )
823
- markdown_lines.append(
824
- f"- **Character Count:** {len(chunk):,}"
825
- )
826
- markdown_lines.append("")
827
- markdown_lines.append("```")
828
- markdown_lines.append(chunk)
829
- markdown_lines.append("```\n")
830
-
831
- markdown_lines.append("---\n")
832
- markdown_lines.append("## Chunk Summary\n")
833
- markdown_lines.append("| Chunk | Tokens | Characters | Preview |")
834
- markdown_lines.append("|-------|--------|------------|---------|")
835
-
836
- for i, chunk_idx in enumerate(chunk_indices, 1):
837
- if chunk_idx < len(agent.rag.chunks):
838
- chunk = agent.rag.chunks[chunk_idx]
839
- tokens = len(chunk) // 4
840
- chars = len(chunk)
841
- preview = chunk[:50].replace("\n", " ").replace("|", "\\|")
842
- markdown_lines.append(
843
- f"| {i} | {tokens} | {chars:,} | {preview}... |"
844
- )
845
-
846
- # Write to file
847
- output_filename = f"{Path(file_path).stem}_rag_dump.md"
848
- output_path = Path.cwd() / output_filename
849
-
850
- with open(output_path, "w", encoding="utf-8") as f:
851
- f.write("\n".join(markdown_lines))
852
-
853
- # Calculate output size
854
- output_content = "\n".join(markdown_lines)
855
- output_size = len(output_content)
856
-
857
- print(f" Markdown dump created: {output_filename}")
858
- print(f" Path: {output_path}")
859
- print("\n📊 Statistics:")
860
- print(f" Document: {len(extracted_text):,} characters")
861
- print(f" Chunks: {len(chunk_indices)}")
862
- print(f" Output: {output_size:,} characters")
863
- print("\nYou can now:")
864
- print(" 1. Review chunking quality")
865
- print(" 2. Verify PDF extraction accuracy")
866
- print(" 3. Search for specific content (Ctrl+F)")
867
- print(" 4. Share with others for troubleshooting")
868
-
869
- elif command == "/clear-cache":
870
- print("\n⚠️ This will clear the RAG cache for all documents.")
871
- print(
872
- " Cached chunks will be deleted and documents will need to be re-indexed."
873
- )
874
- response = input("\nAre you sure? (yes/no): ").strip().lower()
875
-
876
- if response == "yes":
877
- import shutil
878
-
879
- cache_dir = agent.rag.config.cache_dir
880
- if os.path.exists(cache_dir):
881
- shutil.rmtree(cache_dir)
882
- os.makedirs(cache_dir, exist_ok=True)
883
- print(f"\n✅ Cache cleared: {cache_dir}")
884
-
885
- # Clear in-memory state as well
886
- agent.rag.indexed_files.clear()
887
- agent.rag.chunks.clear()
888
- agent.rag.chunk_to_file.clear()
889
- agent.rag.file_to_chunk_indices.clear()
890
- agent.rag.file_metadata.clear()
891
- agent.rag.index = None
892
- agent.indexed_files.clear()
893
-
894
- print(
895
- "\nAll documents will be re-indexed from scratch on next access."
896
- )
897
- else:
898
- print(f"\nℹ️ Cache directory doesn't exist: {cache_dir}")
899
- else:
900
- print("\n❌ Cache clear cancelled")
901
-
902
- elif command == "/search-debug":
903
- # Toggle RAG debug mode
904
- current_debug = getattr(agent.rag.config, "show_stats", False)
905
- agent.rag.config.show_stats = not current_debug
906
-
907
- if agent.rag.config.show_stats:
908
- print("\n🔧 RAG Debug Mode: ENABLED")
909
- print("\nAll queries will now show:")
910
- print(" - Embedding generation details")
911
- print(" - Search execution info")
912
- print(" - Similarity scores")
913
- print(" - Chunk selection reasoning")
914
- else:
915
- print("\n🔧 RAG Debug Mode: DISABLED")
916
- print("\nQueries will run in normal mode")
917
-
918
- else:
919
- print(f"Unknown command: {command}. Type /help for help.")
920
-
921
- continue
922
-
923
- # Process regular query
924
- result = agent.process_query(user_input)
925
- # The answer is already streamed by the agent, no need to print it again
926
-
927
- # Update conversation history for session persistence
928
- if hasattr(agent, "conversation_history"):
929
- agent.conversation_history.append(
930
- {"role": "user", "content": user_input}
931
- )
932
- if result.get("result"):
933
- agent.conversation_history.append(
934
- {"role": "assistant", "content": result["result"]}
935
- )
936
-
937
- if result.get("error_count", 0) > 0:
938
- print(f"\n⚠️ {result['error_count']} error(s) occurred")
939
-
940
- except KeyboardInterrupt:
941
- print("\n\nGoodbye!")
942
- break
943
- except EOFError:
944
- print("\n\nGoodbye!")
945
- break
946
- except Exception as e:
947
- logger.error(f"Error: {e}")
948
- print(f"\n❌ Error: {e}")
949
-
950
-
951
- def main():
952
- """Main entry point."""
953
- args = parse_args()
954
-
955
- try:
956
- # Keep console output visible by default so users can see agent reasoning
957
- # Silent mode can be enabled with --silent flag or for single-query mode
958
- if args.silent:
959
- use_silent_mode = True
960
- elif args.query is not None:
961
- use_silent_mode = True # Auto-silent for non-interactive queries
962
- else:
963
- use_silent_mode = False # Show output in interactive mode
964
-
965
- # Create agent config
966
- config = ChatAgentConfig(
967
- use_claude=args.use_claude,
968
- use_chatgpt=args.use_chatgpt,
969
- claude_model=args.claude_model,
970
- model_id=args.model_id,
971
- max_steps=args.max_steps,
972
- show_prompts=args.show_prompts,
973
- output_dir=args.output_dir,
974
- streaming=args.streaming,
975
- show_stats=args.show_stats,
976
- silent_mode=use_silent_mode,
977
- debug=args.debug,
978
- rag_documents=args.documents,
979
- watch_directories=args.watch,
980
- chunk_size=args.chunk_size,
981
- chunk_overlap=args.chunk_overlap,
982
- max_chunks=args.max_chunks,
983
- use_llm_chunking=args.use_llm_chunking,
984
- allowed_paths=args.allowed_paths,
985
- )
986
-
987
- # Create agent with config
988
- agent = ChatAgent(config)
989
-
990
- # Create initial session if not loading one
991
- if not agent.current_session:
992
- agent.current_session = agent.session_manager.create_session()
993
- logger.debug(f"Created new session: {agent.current_session.session_id}")
994
-
995
- # Index document if --index flag provided
996
- if args.index:
997
- index_path = args.index
998
- if not os.path.exists(index_path):
999
- print(f"❌ File not found: {index_path}")
1000
- return 1
1001
-
1002
- print(f"📄 Indexing: {Path(index_path).name}")
1003
- print("=" * 60)
1004
-
1005
- result = agent.rag.index_document(str(Path(index_path).absolute()))
1006
-
1007
- if result.get("success"):
1008
- print("✅ INDEXING SUCCESSFUL")
1009
- print(f"📁 File: {result.get('file_name')}")
1010
- print(f"📄 Type: {result.get('file_type')}")
1011
- print(f"💾 Size: {result.get('file_size_mb', 0):.2f} MB")
1012
- if result.get("num_pages"):
1013
- print(f"📖 Pages: {result['num_pages']}")
1014
- print(f"📦 Chunks: {result.get('num_chunks', 0)}")
1015
- print("=" * 60)
1016
- else:
1017
- print(f" Indexing failed: {result.get('error')}")
1018
- return 1
1019
-
1020
- # List tools if requested
1021
- if args.list_tools:
1022
- agent.list_tools(verbose=True)
1023
- return 0
1024
-
1025
- # Single query mode
1026
- if args.query:
1027
- result = agent.process_query(args.query)
1028
- print(f"\n{result['result']}")
1029
-
1030
- if args.show_stats and result.get("duration"):
1031
- print("\n📊 Stats:")
1032
- print(f" Duration: {result['duration']:.2f}s")
1033
- print(f" Steps: {result['steps_taken']}")
1034
- print(f" Tokens: {result.get('total_tokens', 0):,}")
1035
-
1036
- return 0 if result["status"] == "success" else 1
1037
-
1038
- # Interactive mode
1039
- interactive_mode(agent)
1040
- return 0
1041
-
1042
- except KeyboardInterrupt:
1043
- print("\n\nInterrupted by user")
1044
- return 130
1045
- except Exception as e:
1046
- logger.error(f"Error: {e}", exc_info=True)
1047
- print(f"\n❌ Error: {e}")
1048
- return 1
1049
- finally:
1050
- # Cleanup
1051
- try:
1052
- agent.stop_watching()
1053
- except Exception: # pylint: disable=broad-except
1054
- pass
1055
-
1056
-
1057
- if __name__ == "__main__":
1058
- sys.exit(main())
1
+ # Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved.
2
+ # SPDX-License-Identifier: MIT
3
+ """
4
+ Chat Agent Application - Interactive chat with RAG and file search.
5
+ """
6
+
7
+ import argparse
8
+ import os
9
+ import sys
10
+ from pathlib import Path
11
+
12
+ from gaia.agents.chat.agent import ChatAgent, ChatAgentConfig
13
+ from gaia.logger import get_logger
14
+
15
+ logger = get_logger(__name__)
16
+
17
+
18
+ def parse_args():
19
+ """Parse command line arguments."""
20
+ parser = argparse.ArgumentParser(
21
+ description="Chat Agent with RAG and file search capabilities"
22
+ )
23
+
24
+ # LLM backend options
25
+ parser.add_argument(
26
+ "--use-claude", action="store_true", help="Use Claude API instead of local LLM"
27
+ )
28
+ parser.add_argument(
29
+ "--use-chatgpt",
30
+ action="store_true",
31
+ help="Use ChatGPT/OpenAI API instead of local LLM",
32
+ )
33
+ parser.add_argument(
34
+ "--claude-model",
35
+ type=str,
36
+ default="claude-sonnet-4-20250514",
37
+ help="Claude model to use (default: claude-sonnet-4-20250514)",
38
+ )
39
+ parser.add_argument(
40
+ "--model-id",
41
+ type=str,
42
+ default=None,
43
+ help="Model ID for local LLM (default: Qwen3-Coder-30B-A3B-Instruct-GGUF)",
44
+ )
45
+
46
+ # Agent configuration
47
+ parser.add_argument(
48
+ "--max-steps",
49
+ type=int,
50
+ default=10,
51
+ help="Maximum conversation steps (default: 10)",
52
+ )
53
+ parser.add_argument(
54
+ "--streaming", action="store_true", help="Enable streaming responses"
55
+ )
56
+ parser.add_argument(
57
+ "--show-stats", action="store_true", help="Show performance statistics"
58
+ )
59
+ parser.add_argument(
60
+ "--show-prompts", action="store_true", help="Display prompts sent to LLM"
61
+ )
62
+ parser.add_argument("--debug", action="store_true", help="Enable debug output")
63
+ parser.add_argument(
64
+ "--silent",
65
+ action="store_true",
66
+ help="Suppress agent reasoning output (silent mode)",
67
+ )
68
+
69
+ # RAG configuration
70
+ parser.add_argument(
71
+ "--index",
72
+ "-i",
73
+ type=str,
74
+ metavar="PATH",
75
+ help="Index a document before running (combine with --query for one-shot usage)",
76
+ )
77
+ parser.add_argument(
78
+ "--documents", "-d", nargs="+", help="Documents to index for RAG"
79
+ )
80
+ parser.add_argument(
81
+ "--watch", "-w", nargs="+", help="Directories to monitor for new documents"
82
+ )
83
+ parser.add_argument(
84
+ "--chunk-size", type=int, default=500, help="Document chunk size (default: 500)"
85
+ )
86
+ parser.add_argument(
87
+ "--chunk-overlap",
88
+ type=int,
89
+ default=100,
90
+ help="Chunk overlap in characters (default: 100)",
91
+ )
92
+ parser.add_argument(
93
+ "--max-chunks",
94
+ type=int,
95
+ default=5,
96
+ help="Maximum chunks to retrieve (default: 5)",
97
+ )
98
+ parser.add_argument(
99
+ "--use-llm-chunking",
100
+ action="store_true",
101
+ help="Use LLM-based semantic chunking for better context preservation (slower but more accurate)",
102
+ )
103
+ parser.add_argument(
104
+ "--allowed-paths",
105
+ nargs="+",
106
+ help="Allowed directory paths for file operations (default: current directory)",
107
+ )
108
+
109
+ # Input/output
110
+ parser.add_argument(
111
+ "--query", "-q", type=str, help="Single query to execute (non-interactive mode)"
112
+ )
113
+ parser.add_argument("--output-dir", type=str, help="Directory for output files")
114
+ parser.add_argument(
115
+ "--list-tools", action="store_true", help="List available tools and exit"
116
+ )
117
+
118
+ return parser.parse_args()
119
+
120
+
121
+ def interactive_mode(agent: ChatAgent):
122
+ """Run agent in interactive chat mode."""
123
+ print("=" * 60)
124
+ print("Chat Agent with RAG - Interactive Mode")
125
+ print("=" * 60)
126
+
127
+ # Display model information
128
+ model_name = getattr(agent, "model_display_name", "Unknown")
129
+ print(f"\n🤖 Model: {model_name}")
130
+
131
+ print("\nCapabilities:")
132
+ print(" • Document Q&A using RAG")
133
+ print(" - Documents: PDF, TXT, MD, CSV, JSON")
134
+ print(" - Backend: Python, Java, C/C++, Go, Rust, Ruby, PHP, Swift, etc.")
135
+ print(" - Web: JS/TS, HTML, CSS/SCSS/SASS, Vue, Svelte, React (JSX/TSX)")
136
+ print(" - Config: YAML, XML, TOML, INI, ENV")
137
+ print(" • Document summarization with multiple styles")
138
+ print(" • Code retrieval and search (no code generation)")
139
+ print(" • File search and operations")
140
+ print(" • Shell command execution")
141
+ print(" • Auto-indexing when files change")
142
+ print(" • Session persistence with auto-save")
143
+ print("\nSession Commands:")
144
+ print(" /resume [id] - Resume a previous session (or list if no id)")
145
+ print(" /save - Save current session")
146
+ print(" /sessions - List all available sessions")
147
+ print(" /reset - Clear conversation and start fresh")
148
+ print("\nDocument Commands:")
149
+ print(" /index <path> - Index a document or directory")
150
+ print(" /watch <dir> - Watch a directory for changes")
151
+ print(" /list - List indexed documents")
152
+ print(" /status - Show system status")
153
+ print("\nDebug Commands:")
154
+ print(" /chunks <file> - Show all chunks for a document")
155
+ print(" /chunk <n> - Show specific chunk content")
156
+ print(" /test <query> - Test query retrieval with scores")
157
+ print(
158
+ " /dump <file|#> - Export document + chunks to markdown (use number from /list)"
159
+ )
160
+ print(" /clear-cache - Clear RAG cache (force re-indexing)")
161
+ print(" /search-debug - Toggle RAG debug mode")
162
+ print("\nOther Commands:")
163
+ print(" /help or /? - Show this help")
164
+ print(" /quit - Exit")
165
+ print("\nOr just type your question to chat with indexed documents.")
166
+ print("=" * 60)
167
+
168
+ while True:
169
+ try:
170
+ user_input = input("\nYou: ").strip()
171
+
172
+ if not user_input:
173
+ continue
174
+
175
+ # Handle commands
176
+ if user_input.startswith("/"):
177
+ parts = user_input.split(maxsplit=1)
178
+ command = parts[0].lower()
179
+ arg = parts[1] if len(parts) > 1 else None
180
+
181
+ if command == "/quit":
182
+ # Auto-save before quitting
183
+ if agent.current_session:
184
+ print("\n💾 Saving current session...")
185
+ agent.save_current_session()
186
+ print("Goodbye!")
187
+ break
188
+
189
+ elif command in ["/help", "/?"]:
190
+ print("\n" + "=" * 60)
191
+ print("Chat Agent - Available Commands & Capabilities")
192
+ print("=" * 60)
193
+ print("\n🎯 CAPABILITIES:")
194
+ print(" • Document Q&A using RAG")
195
+ print(" - Documents: PDF, TXT, MD, CSV, JSON")
196
+ print(
197
+ " - Backend: Python, Java, C/C++, Go, Rust, Ruby, PHP, Swift, Kotlin, Scala"
198
+ )
199
+ print(
200
+ " - Web: JS/TS, HTML, CSS/SCSS/SASS/LESS, Vue, Svelte, Astro, React"
201
+ )
202
+ print(" - Config: YAML, XML, TOML, INI, ENV, Properties")
203
+ print(" - Build: Gradle, CMake, Makefiles")
204
+ print(" - Database: SQL")
205
+ print(
206
+ " • Document summarization (brief, detailed, bullets, executive, etc.)"
207
+ )
208
+ print(" • Code retrieval and search (no code generation)")
209
+ print(" • File operations and search")
210
+ print(" • Shell command execution (ls, grep, find, etc.)")
211
+ print(" • Auto-indexing when files change")
212
+ print(" • Session persistence with auto-save")
213
+ print("\n📋 SESSION MANAGEMENT:")
214
+ print(" /resume [id] - Resume session (or list if no id)")
215
+ print(" /save - Save current session")
216
+ print(" /sessions - List all sessions")
217
+ print(" /reset - Clear and start fresh")
218
+ print("\n📚 DOCUMENT MANAGEMENT:")
219
+ print(" /index <path> - Index file or directory")
220
+ print(" /watch <dir> - Watch directory for changes")
221
+ print(" /list - List indexed documents")
222
+ print(" /status - Show system status")
223
+ print("\n🔍 DEBUG & OBSERVABILITY:")
224
+ print(" /chunks <file> - Show all chunks for a document")
225
+ print(" /chunk <n> - Show specific chunk content")
226
+ print(" /test <query> - Test query retrieval with scores")
227
+ print(
228
+ " /dump <file|#> - Export document + chunks to markdown (use number from /list)"
229
+ )
230
+ print(" /clear-cache - Clear RAG cache (force re-indexing)")
231
+ print(" /search-debug - Toggle RAG debug mode")
232
+ print("\n💬 EXAMPLE QUERIES:")
233
+ print(" Documents:")
234
+ print(" 'What does the document say about X?'")
235
+ print(" 'Summarize report.pdf in bullet points'")
236
+ print(" Backend Code:")
237
+ print(" 'Where is the UserAuth class defined?'")
238
+ print(" 'Find all functions that use database connections'")
239
+ print(" 'What does the authenticate() function do?'")
240
+ print(" Web Development:")
241
+ print(" 'Find all CSS classes with hover effects'")
242
+ print(" 'Where are the API endpoints defined?'")
243
+ print(" 'Show me all Vue components that use props'")
244
+ print(" General:")
245
+ print(" 'List all Python files in src/'")
246
+ print(" 'Search for TODO comments in my code'")
247
+ print(" 'Find all files that import React'")
248
+ print("\n⚙️ OTHER COMMANDS:")
249
+ print(" /help or /? - Show this help")
250
+ print(" /quit - Exit (auto-saves)")
251
+ print("=" * 60)
252
+
253
+ elif command == "/resume":
254
+ if not arg:
255
+ # List available sessions
256
+ sessions = agent.session_manager.list_sessions()
257
+ if not sessions:
258
+ print("\n📂 No saved sessions found.")
259
+ else:
260
+ print("\n" + "=" * 60)
261
+ print("Available Sessions")
262
+ print("=" * 60)
263
+ for sess in sessions:
264
+ print(f"\n ID: {sess['session_id']}")
265
+ print(f" Created: {sess['created_at']}")
266
+ print(f" Updated: {sess['updated_at']}")
267
+ print(f" Documents: {sess['num_documents']}")
268
+ print(f" Messages: {sess['num_messages']}")
269
+ print("\n" + "=" * 60)
270
+ print("\nUse: /resume <session_id> to load a session")
271
+ else:
272
+ # Resume specific session
273
+ print(f"\n📂 Resuming session: {arg}")
274
+ if agent.load_session(arg):
275
+ session = agent.current_session
276
+ print("✅ Loaded session with:")
277
+ print(f" - {len(session.indexed_documents)} documents")
278
+ print(
279
+ f" - {len(session.watched_directories)} watched directories"
280
+ )
281
+ print(f" - {len(session.chat_history)} chat messages")
282
+ else:
283
+ print(f"❌ Failed to load session: {arg}")
284
+
285
+ elif command == "/save":
286
+ print("\n💾 Saving current session...")
287
+ if agent.save_current_session():
288
+ session_id = (
289
+ agent.current_session.session_id
290
+ if agent.current_session
291
+ else "unknown"
292
+ )
293
+ print(f"✅ Session saved: {session_id}")
294
+ else:
295
+ print("❌ Failed to save session")
296
+
297
+ elif command == "/sessions":
298
+ sessions = agent.session_manager.list_sessions()
299
+ if not sessions:
300
+ print("\n📂 No saved sessions found.")
301
+ else:
302
+ print("\n" + "=" * 60)
303
+ print(f"Found {len(sessions)} Session(s)")
304
+ print("=" * 60)
305
+ for i, sess in enumerate(sessions, 1):
306
+ print(f"\n{i}. {sess['session_id']}")
307
+ print(f" Created: {sess['created_at']}")
308
+ print(
309
+ f" Documents: {sess['num_documents']}, Messages: {sess['num_messages']}"
310
+ )
311
+ print("\n" + "=" * 60)
312
+
313
+ elif command == "/reset":
314
+ print("\n🔄 Resetting conversation...")
315
+ # Save current session first
316
+ if agent.current_session:
317
+ agent.save_current_session()
318
+ # Create new session
319
+ agent.current_session = agent.session_manager.create_session()
320
+ # Clear chat history (if agent tracks it)
321
+ if hasattr(agent, "chat_history"):
322
+ agent.chat_history = []
323
+ print("✅ Conversation reset. Previous session saved.")
324
+ print(f" New session: {agent.current_session.session_id}")
325
+
326
+ elif command == "/index":
327
+ if not arg:
328
+ print("Usage: /index <file_or_directory_path>")
329
+ print("Example: /index /path/to/document.pdf")
330
+ print(" /index /path/to/documents/")
331
+ continue
332
+
333
+ # Check if it's a directory or file
334
+ path = Path(arg)
335
+
336
+ if path.is_dir():
337
+ print(f"\n📁 Indexing all documents in directory: {arg}")
338
+ # Find all supported document types
339
+ doc_patterns = ["*.pdf", "*.txt", "*.md", "*.csv", "*.json"]
340
+ doc_files = []
341
+ for pattern in doc_patterns:
342
+ doc_files.extend(path.glob(pattern))
343
+
344
+ if not doc_files:
345
+ print("❌ No supported documents found in directory")
346
+ print(" Supported types: PDF, TXT, MD, CSV, JSON")
347
+ continue
348
+
349
+ print(f"Found {len(doc_files)} document(s)\n")
350
+ success_count = 0
351
+ for doc_file in doc_files:
352
+ print(f" 📄 Indexing: {doc_file.name}...")
353
+ # Directly call the RAG index method
354
+ try:
355
+ result = agent.rag.index_document(
356
+ str(doc_file.absolute())
357
+ )
358
+ if result.get("success"):
359
+ print(
360
+ f" ✅ Success: {result.get('num_chunks', 0)} chunks created"
361
+ )
362
+ success_count += 1
363
+ else:
364
+ error = result.get("error", "Unknown error")
365
+ print(f" ❌ Failed: {error}")
366
+ except Exception as e:
367
+ print(f" ❌ Error: {e}")
368
+
369
+ print(
370
+ f"\n📊 Summary: {success_count}/{len(doc_files)} documents indexed successfully"
371
+ )
372
+
373
+ # Update system prompt to include newly indexed documents
374
+ if success_count > 0:
375
+ agent.update_system_prompt()
376
+
377
+ else:
378
+ # Single file
379
+ if not os.path.exists(arg):
380
+ print(f"\n File not found: {arg}")
381
+ print(" Please check the file path and try again")
382
+ continue
383
+
384
+ print(f"\n📄 Indexing: {path.name}")
385
+ print("=" * 60)
386
+
387
+ try:
388
+ # Directly call the RAG index method
389
+ result = agent.rag.index_document(str(path.absolute()))
390
+
391
+ if result.get("success"):
392
+ # Display success with detailed stats
393
+ print("✅ INDEXING SUCCESSFUL")
394
+ print("=" * 60)
395
+ print(f"📁 File: {result.get('file_name', path.name)}")
396
+ print(f"📄 Type: {result.get('file_type', 'Unknown')}")
397
+ print(
398
+ f"💾 Size: {result.get('file_size_mb', 0):.2f} MB"
399
+ )
400
+
401
+ # Show num_pages for PDFs
402
+ if result.get("num_pages"):
403
+ print(f"📖 Pages: {result['num_pages']}")
404
+
405
+ print(
406
+ f"📦 Chunks Created: {result.get('num_chunks', 0)}"
407
+ )
408
+
409
+ # Show cache/reindex status
410
+ if result.get("from_cache"):
411
+ print(" Loaded from cache (fast)")
412
+ elif result.get("already_indexed"):
413
+ print("ℹ️ Already indexed (skipped)")
414
+ elif result.get("reindexed"):
415
+ print("🔄 Reindexed (updated)")
416
+
417
+ print("\n📊 GLOBAL STATISTICS")
418
+ print("=" * 60)
419
+ print(
420
+ f"Total Documents Indexed: {result.get('total_indexed_files', 0)}"
421
+ )
422
+ print(f"Total Chunks: {result.get('total_chunks', 0)}")
423
+ print("=" * 60)
424
+
425
+ # Update system prompt to include newly indexed document
426
+ agent.update_system_prompt()
427
+ else:
428
+ # Display error
429
+ print("❌ INDEXING FAILED")
430
+ print("=" * 60)
431
+ error = result.get("error", "Unknown error")
432
+ print(f"Error: {error}")
433
+ if result.get("file_name"):
434
+ print(f"File: {result['file_name']}")
435
+ print("=" * 60)
436
+
437
+ except Exception as e:
438
+ print(" INDEXING FAILED")
439
+ print("=" * 60)
440
+ print(f"Error: {e}")
441
+ print("=" * 60)
442
+
443
+ elif command == "/watch":
444
+ if not arg:
445
+ print("Usage: /watch <directory_path>")
446
+ print("Example: /watch /path/to/documents")
447
+ continue
448
+ result = agent.process_query(f"Watch the directory: {arg}")
449
+ print(f"\n{result['result']}")
450
+
451
+ elif command == "/list":
452
+ # Directly access indexed files from RAG system
453
+ print("\n" + "=" * 60)
454
+ print("📚 INDEXED DOCUMENTS")
455
+ print("=" * 60)
456
+
457
+ if not agent.rag.indexed_files:
458
+ print("No documents indexed yet.")
459
+ print("\nUse /index <path> to index a document")
460
+ else:
461
+ print(f"Total: {len(agent.rag.indexed_files)} document(s)\n")
462
+ for i, file_path in enumerate(
463
+ sorted(agent.rag.indexed_files), 1
464
+ ):
465
+ file_name = Path(file_path).name
466
+ file_type = Path(file_path).suffix
467
+
468
+ # Get chunk count for this file if available
469
+ num_chunks = 0
470
+ if file_path in agent.rag.file_to_chunk_indices:
471
+ num_chunks = len(
472
+ agent.rag.file_to_chunk_indices[file_path]
473
+ )
474
+
475
+ print(f"{i}. {file_name}")
476
+ print(f" Type: {file_type}")
477
+ print(f" Chunks: {num_chunks}")
478
+ print(f" Path: {file_path}")
479
+ print()
480
+
481
+ print("=" * 60)
482
+
483
+ elif command == "/status":
484
+ # Directly access RAG status
485
+ print("\n" + "=" * 60)
486
+ print("📊 RAG SYSTEM STATUS")
487
+ print("=" * 60)
488
+
489
+ status = agent.rag.get_status()
490
+
491
+ print("\n📚 Documents:")
492
+ print(f" Indexed Files: {status['indexed_files']}")
493
+ print(f" Total Chunks: {status['total_chunks']}")
494
+
495
+ print("\n⚙️ Configuration:")
496
+ print(f" Chunk Size: {status['config']['chunk_size']} tokens")
497
+ print(
498
+ f" Chunk Overlap: {status['config']['chunk_overlap']} tokens"
499
+ )
500
+ print(f" Max Chunks per Query: {status['config']['max_chunks']}")
501
+
502
+ print("\n💾 Storage:")
503
+ print(f" Cache Directory: {status['cache_dir']}")
504
+ print(f" Embedding Model: {status['embedding_model']}")
505
+
506
+ # Show watched directories
507
+ if agent.watch_directories:
508
+ print("\n👀 Watched Directories:")
509
+ for i, dir_path in enumerate(agent.watch_directories, 1):
510
+ print(f" {i}. {dir_path}")
511
+
512
+ # Show session info
513
+ if agent.current_session:
514
+ print("\n📋 Current Session:")
515
+ print(f" ID: {agent.current_session.session_id}")
516
+ print(f" Created: {agent.current_session.created_at}")
517
+ print(f" Updated: {agent.current_session.updated_at}")
518
+ print(
519
+ f" Documents: {len(agent.current_session.indexed_documents)}"
520
+ )
521
+ print(f" Messages: {len(agent.current_session.chat_history)}")
522
+
523
+ print("\n" + "=" * 60)
524
+
525
+ elif command == "/chunks":
526
+ if not arg:
527
+ print("Usage: /chunks <filename>")
528
+ print("Example: /chunks document.pdf")
529
+ continue
530
+
531
+ # Find file in indexed files
532
+ matching_files = [
533
+ f
534
+ for f in agent.rag.indexed_files
535
+ if Path(f).name == arg or f == arg
536
+ ]
537
+
538
+ if not matching_files:
539
+ print(f"\n❌ File not indexed: {arg}")
540
+ print("\nIndexed files:")
541
+ for f in agent.rag.indexed_files:
542
+ print(f" - {Path(f).name}")
543
+ continue
544
+
545
+ file_path = matching_files[0]
546
+
547
+ # Get chunks for this file
548
+ if file_path not in agent.rag.file_to_chunk_indices:
549
+ print(f"\n❌ No chunks found for: {arg}")
550
+ continue
551
+
552
+ chunk_indices = agent.rag.file_to_chunk_indices[file_path]
553
+
554
+ print("\n" + "=" * 60)
555
+ print(f"DOCUMENT CHUNKS: {Path(file_path).name}")
556
+ print("=" * 60)
557
+ print(f"Total Chunks: {len(chunk_indices)}")
558
+ print(f"Chunk Size: {agent.rag.config.chunk_size} tokens")
559
+ print(f"Overlap: {agent.rag.config.chunk_overlap} tokens\n")
560
+
561
+ for i, chunk_idx in enumerate(chunk_indices, 1):
562
+ if chunk_idx < len(agent.rag.chunks):
563
+ chunk = agent.rag.chunks[chunk_idx]
564
+ token_count = len(chunk) // 4 # Rough estimate
565
+
566
+ print(f"CHUNK {i} ({token_count} tokens):")
567
+ print("┌" + "─" * 58 + "┐")
568
+
569
+ # Show first 200 chars of chunk
570
+ preview = chunk[:200].replace("\n", " ")
571
+ if len(chunk) > 200:
572
+ preview += "..."
573
+
574
+ # Word wrap the preview
575
+ import textwrap
576
+
577
+ wrapped = textwrap.wrap(preview, width=56)
578
+ for line in wrapped:
579
+ print(f"│ {line:<56} │")
580
+
581
+ print("└" + "─" * 58 + "┘")
582
+ print()
583
+
584
+ print("=" * 60)
585
+ print("\nUse '/chunk <n>' to see full content of a specific chunk")
586
+
587
+ elif command == "/chunk":
588
+ if not arg or not arg.isdigit():
589
+ print("Usage: /chunk <number>")
590
+ print("Example: /chunk 5")
591
+ continue
592
+
593
+ chunk_num = int(arg) - 1 # Convert to 0-indexed
594
+
595
+ if chunk_num < 0 or chunk_num >= len(agent.rag.chunks):
596
+ print(
597
+ f"\n❌ Invalid chunk number. Valid range: 1-{len(agent.rag.chunks)}"
598
+ )
599
+ continue
600
+
601
+ chunk = agent.rag.chunks[chunk_num]
602
+
603
+ # Find which file this chunk belongs to
604
+ source_file = agent.rag.chunk_to_file.get(chunk_num, "Unknown")
605
+
606
+ print("\n" + "=" * 60)
607
+ print(f"CHUNK {arg}")
608
+ print("=" * 60)
609
+ print(
610
+ f"Source: {Path(source_file).name if source_file != 'Unknown' else 'Unknown'}"
611
+ )
612
+ print(f"Token count: {len(chunk) // 4} (estimated)")
613
+ print(f"Character count: {len(chunk):,}")
614
+ print("\nCONTENT:")
615
+ print("┌" + "─" * 58 + "┐")
616
+
617
+ # Word wrap content
618
+ import textwrap
619
+
620
+ wrapped = textwrap.wrap(chunk, width=56)
621
+ for line in wrapped:
622
+ print(f" {line:<56} ")
623
+
624
+ print("└" + "─" * 58 + "┘")
625
+ print("=" * 60)
626
+
627
+ elif command == "/test":
628
+ if not arg:
629
+ print("Usage: /test <query>")
630
+ print("Example: /test what is the vision?")
631
+ continue
632
+
633
+ if not agent.rag.indexed_files:
634
+ print("\n❌ No documents indexed. Use /index <path> first.")
635
+ continue
636
+
637
+ print("\n" + "=" * 60)
638
+ print(f'QUERY TEST: "{arg}"')
639
+ print("=" * 60)
640
+
641
+ # Test retrieval
642
+ try:
643
+ # pylint: disable=protected-access
644
+ chunks, scores = agent.rag._retrieve_chunks(arg)
645
+
646
+ print(f"\nTop {len(chunks)} matching chunks:\n")
647
+
648
+ for i, (chunk, score) in enumerate(zip(chunks, scores), 1):
649
+ # Find chunk index
650
+ chunk_idx = (
651
+ agent.rag.chunks.index(chunk)
652
+ if chunk in agent.rag.chunks
653
+ else -1
654
+ )
655
+ source_file = agent.rag.chunk_to_file.get(
656
+ chunk_idx, "Unknown"
657
+ )
658
+
659
+ # Color code by score
660
+ if score > 0.75:
661
+ emoji = "⭐"
662
+ elif score > 0.60:
663
+ emoji = "✅"
664
+ else:
665
+ emoji = "⚠️"
666
+
667
+ print(
668
+ f"{i}. {emoji} CHUNK {chunk_idx + 1} (score: {score:.3f})"
669
+ )
670
+ print(
671
+ f" Source: {Path(source_file).name if source_file != 'Unknown' else 'Unknown'}"
672
+ )
673
+
674
+ # Preview
675
+ preview = chunk[:150].replace("\n", " ")
676
+ if len(chunk) > 150:
677
+ preview += "..."
678
+ print(f" Preview: {preview}")
679
+ print()
680
+
681
+ print("=" * 60)
682
+ print("\nScore Legend:")
683
+ print(" ⭐ Excellent (>0.75) - Highly relevant")
684
+ print(" ✅ Good (0.60-0.75) - Probably relevant")
685
+ print(" ⚠️ Poor (<0.60) - May not be relevant")
686
+
687
+ except Exception as e:
688
+ print(f"\n❌ Query test failed: {e}")
689
+
690
+ elif command == "/dump":
691
+ if not arg:
692
+ print("Usage: /dump <filename_or_number>")
693
+ print("Example: /dump document.pdf")
694
+ print("Example: /dump 1")
695
+ continue
696
+
697
+ # Find file in indexed files
698
+ from datetime import datetime
699
+
700
+ file_path = None
701
+
702
+ # Check if arg is a number (index)
703
+ if arg.isdigit():
704
+ index = int(arg) - 1 # Convert to 0-based index
705
+ indexed_list = list(agent.rag.indexed_files)
706
+ if 0 <= index < len(indexed_list):
707
+ file_path = indexed_list[index]
708
+ else:
709
+ print(f"\n❌ Invalid index: {arg}")
710
+ print(f"Valid range: 1-{len(indexed_list)}")
711
+ continue
712
+ else:
713
+ # Try to match by filename
714
+ matching_files = [
715
+ f
716
+ for f in agent.rag.indexed_files
717
+ if Path(f).name == arg or f == arg
718
+ ]
719
+
720
+ if not matching_files:
721
+ print(f"\n❌ File not indexed: {arg}")
722
+ print("\nIndexed files:")
723
+ for i, f in enumerate(agent.rag.indexed_files, 1):
724
+ print(f" {i}. {Path(f).name}")
725
+ continue
726
+
727
+ file_path = matching_files[0]
728
+
729
+ # Get chunks for this file
730
+ if file_path not in agent.rag.file_to_chunk_indices:
731
+ print(f"\n❌ No chunks found for: {arg}")
732
+ continue
733
+
734
+ chunk_indices = agent.rag.file_to_chunk_indices[file_path]
735
+
736
+ print(f"\n📝 Generating markdown dump for: {Path(file_path).name}")
737
+
738
+ # Try to get cached metadata first (faster!)
739
+ cached_metadata = agent.rag.file_metadata.get(file_path, {})
740
+ extracted_text = cached_metadata.get("full_text")
741
+ num_pages = cached_metadata.get("num_pages")
742
+ vlm_pages = cached_metadata.get("vlm_pages", 0)
743
+ total_images = cached_metadata.get("total_images", 0)
744
+
745
+ # Fall back to re-extraction only if cache is missing
746
+ if not extracted_text:
747
+ print(" ⚠️ No cached text found, re-extracting from file...")
748
+ try:
749
+ # pylint: disable=protected-access
750
+ extracted_text, metadata = (
751
+ agent.rag._extract_text_from_file(file_path)
752
+ )
753
+ num_pages = metadata.get("num_pages")
754
+ vlm_pages = metadata.get("vlm_pages", 0)
755
+ total_images = metadata.get("total_images", 0)
756
+ except Exception as e:
757
+ print(f"❌ Failed to extract text: {e}")
758
+ continue
759
+ else:
760
+ print(
761
+ " Using cached text and metadata (no re-extraction needed)"
762
+ )
763
+
764
+ # Create markdown content
765
+ markdown_lines = []
766
+ markdown_lines.append(f"# RAG Debug Dump: {Path(file_path).name}")
767
+ markdown_lines.append(
768
+ f"\nGenerated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n"
769
+ )
770
+
771
+ # Metadata
772
+ markdown_lines.append("## Document Metadata\n")
773
+ markdown_lines.append(f"- **File:** {Path(file_path).name}")
774
+ markdown_lines.append(f"- **Path:** {file_path}")
775
+ file_size = os.path.getsize(file_path) / (1024 * 1024)
776
+ markdown_lines.append(f"- **Size:** {file_size:.2f} MB")
777
+ if num_pages:
778
+ markdown_lines.append(f"- **Pages:** {num_pages}")
779
+ if vlm_pages and vlm_pages > 0:
780
+ markdown_lines.append(
781
+ f"- **VLM Enhanced Pages:** {vlm_pages} (images processed)"
782
+ )
783
+ if total_images and total_images > 0:
784
+ markdown_lines.append(
785
+ f"- **Total Images Extracted:** {total_images}"
786
+ )
787
+ markdown_lines.append(f"- **Characters:** {len(extracted_text):,}")
788
+ markdown_lines.append(f"- **Chunks:** {len(chunk_indices)}")
789
+ markdown_lines.append(
790
+ f"- **Chunk Size:** {agent.rag.config.chunk_size} tokens"
791
+ )
792
+ markdown_lines.append(
793
+ f"- **Chunk Overlap:** {agent.rag.config.chunk_overlap} tokens"
794
+ )
795
+ markdown_lines.append("")
796
+
797
+ # Full document text
798
+ markdown_lines.append("---\n")
799
+ markdown_lines.append("## Full Document Text (As Extracted)")
800
+ markdown_lines.append("\n```")
801
+ markdown_lines.append(extracted_text)
802
+ markdown_lines.append("```\n")
803
+
804
+ # Chunked version with boundaries
805
+ markdown_lines.append("---\n")
806
+ markdown_lines.append("## Document with Chunk Boundaries\n")
807
+
808
+ for i, chunk_idx in enumerate(chunk_indices, 1):
809
+ if chunk_idx < len(agent.rag.chunks):
810
+ chunk = agent.rag.chunks[chunk_idx]
811
+
812
+ # Extract page number from chunk (with lookback)
813
+ from gaia.agents.chat.tools.rag_tools import (
814
+ extract_page_from_chunk,
815
+ )
816
+
817
+ page_num = extract_page_from_chunk(
818
+ chunk, chunk_idx, agent.rag.chunks
819
+ )
820
+ page_info = (
821
+ f"Page {page_num}" if page_num else "Page Unknown"
822
+ )
823
+
824
+ markdown_lines.append(
825
+ f"### 🔷 CHUNK {i} ({page_info}, Index: {chunk_idx})\n"
826
+ )
827
+ markdown_lines.append(
828
+ f"- **Token Count:** {len(chunk) // 4} (estimated)"
829
+ )
830
+ markdown_lines.append(
831
+ f"- **Character Count:** {len(chunk):,}"
832
+ )
833
+ markdown_lines.append("")
834
+ markdown_lines.append("```")
835
+ markdown_lines.append(chunk)
836
+ markdown_lines.append("```\n")
837
+
838
+ markdown_lines.append("---\n")
839
+ markdown_lines.append("## Chunk Summary\n")
840
+ markdown_lines.append("| Chunk | Tokens | Characters | Preview |")
841
+ markdown_lines.append("|-------|--------|------------|---------|")
842
+
843
+ for i, chunk_idx in enumerate(chunk_indices, 1):
844
+ if chunk_idx < len(agent.rag.chunks):
845
+ chunk = agent.rag.chunks[chunk_idx]
846
+ tokens = len(chunk) // 4
847
+ chars = len(chunk)
848
+ preview = chunk[:50].replace("\n", " ").replace("|", "\\|")
849
+ markdown_lines.append(
850
+ f"| {i} | {tokens} | {chars:,} | {preview}... |"
851
+ )
852
+
853
+ # Write to file
854
+ output_filename = f"{Path(file_path).stem}_rag_dump.md"
855
+ output_path = Path.cwd() / output_filename
856
+
857
+ with open(output_path, "w", encoding="utf-8") as f:
858
+ f.write("\n".join(markdown_lines))
859
+
860
+ # Calculate output size
861
+ output_content = "\n".join(markdown_lines)
862
+ output_size = len(output_content)
863
+
864
+ print(f" Markdown dump created: {output_filename}")
865
+ print(f" Path: {output_path}")
866
+ print("\n📊 Statistics:")
867
+ print(f" Document: {len(extracted_text):,} characters")
868
+ print(f" Chunks: {len(chunk_indices)}")
869
+ print(f" Output: {output_size:,} characters")
870
+ print("\nYou can now:")
871
+ print(" 1. Review chunking quality")
872
+ print(" 2. Verify PDF extraction accuracy")
873
+ print(" 3. Search for specific content (Ctrl+F)")
874
+ print(" 4. Share with others for troubleshooting")
875
+
876
+ elif command == "/clear-cache":
877
+ print("\n⚠️ This will clear the RAG cache for all documents.")
878
+ print(
879
+ " Cached chunks will be deleted and documents will need to be re-indexed."
880
+ )
881
+ response = input("\nAre you sure? (yes/no): ").strip().lower()
882
+
883
+ if response == "yes":
884
+ import shutil
885
+
886
+ cache_dir = agent.rag.config.cache_dir
887
+ if os.path.exists(cache_dir):
888
+ shutil.rmtree(cache_dir)
889
+ os.makedirs(cache_dir, exist_ok=True)
890
+ print(f"\n✅ Cache cleared: {cache_dir}")
891
+
892
+ # Clear in-memory state as well
893
+ agent.rag.indexed_files.clear()
894
+ agent.rag.chunks.clear()
895
+ agent.rag.chunk_to_file.clear()
896
+ agent.rag.file_to_chunk_indices.clear()
897
+ agent.rag.file_metadata.clear()
898
+ agent.rag.index = None
899
+ agent.indexed_files.clear()
900
+
901
+ print(
902
+ "\nAll documents will be re-indexed from scratch on next access."
903
+ )
904
+ else:
905
+ print(f"\nℹ️ Cache directory doesn't exist: {cache_dir}")
906
+ else:
907
+ print("\n❌ Cache clear cancelled")
908
+
909
+ elif command == "/search-debug":
910
+ # Toggle RAG debug mode
911
+ current_debug = getattr(agent.rag.config, "show_stats", False)
912
+ agent.rag.config.show_stats = not current_debug
913
+
914
+ if agent.rag.config.show_stats:
915
+ print("\n🔧 RAG Debug Mode: ENABLED")
916
+ print("\nAll queries will now show:")
917
+ print(" - Embedding generation details")
918
+ print(" - Search execution info")
919
+ print(" - Similarity scores")
920
+ print(" - Chunk selection reasoning")
921
+ else:
922
+ print("\n🔧 RAG Debug Mode: DISABLED")
923
+ print("\nQueries will run in normal mode")
924
+
925
+ else:
926
+ print(f"Unknown command: {command}. Type /help for help.")
927
+
928
+ continue
929
+
930
+ # Process regular query
931
+ result = agent.process_query(user_input)
932
+ # The answer is already streamed by the agent, no need to print it again
933
+
934
+ # Update conversation history for session persistence
935
+ if hasattr(agent, "conversation_history"):
936
+ agent.conversation_history.append(
937
+ {"role": "user", "content": user_input}
938
+ )
939
+ if result.get("result"):
940
+ agent.conversation_history.append(
941
+ {"role": "assistant", "content": result["result"]}
942
+ )
943
+
944
+ if result.get("error_count", 0) > 0:
945
+ print(f"\n⚠️ {result['error_count']} error(s) occurred")
946
+
947
+ except KeyboardInterrupt:
948
+ print("\n\nGoodbye!")
949
+ break
950
+ except EOFError:
951
+ print("\n\nGoodbye!")
952
+ break
953
+ except Exception as e:
954
+ logger.error(f"Error: {e}")
955
+ print(f"\n❌ Error: {e}")
956
+
957
+
958
+ def main():
959
+ """Main entry point."""
960
+ args = parse_args()
961
+
962
+ try:
963
+ # Keep console output visible by default so users can see agent reasoning
964
+ # Silent mode can be enabled with --silent flag or for single-query mode
965
+ if args.silent:
966
+ use_silent_mode = True
967
+ elif args.query is not None:
968
+ use_silent_mode = True # Auto-silent for non-interactive queries
969
+ else:
970
+ use_silent_mode = False # Show output in interactive mode
971
+
972
+ # Create agent config
973
+ config = ChatAgentConfig(
974
+ use_claude=args.use_claude,
975
+ use_chatgpt=args.use_chatgpt,
976
+ claude_model=args.claude_model,
977
+ model_id=args.model_id,
978
+ max_steps=args.max_steps,
979
+ show_prompts=args.show_prompts,
980
+ output_dir=args.output_dir,
981
+ streaming=args.streaming,
982
+ show_stats=args.show_stats,
983
+ silent_mode=use_silent_mode,
984
+ debug=args.debug,
985
+ rag_documents=args.documents,
986
+ watch_directories=args.watch,
987
+ chunk_size=args.chunk_size,
988
+ chunk_overlap=args.chunk_overlap,
989
+ max_chunks=args.max_chunks,
990
+ use_llm_chunking=args.use_llm_chunking,
991
+ allowed_paths=args.allowed_paths,
992
+ )
993
+
994
+ # Create agent with config
995
+ agent = ChatAgent(config)
996
+
997
+ # Create initial session if not loading one
998
+ if not agent.current_session:
999
+ agent.current_session = agent.session_manager.create_session()
1000
+ logger.debug(f"Created new session: {agent.current_session.session_id}")
1001
+
1002
+ # Index document if --index flag provided
1003
+ if args.index:
1004
+ index_path = args.index
1005
+ if not os.path.exists(index_path):
1006
+ print(f"❌ File not found: {index_path}")
1007
+ return 1
1008
+
1009
+ print(f"📄 Indexing: {Path(index_path).name}")
1010
+ print("=" * 60)
1011
+
1012
+ result = agent.rag.index_document(str(Path(index_path).absolute()))
1013
+
1014
+ if result.get("success"):
1015
+ print(" INDEXING SUCCESSFUL")
1016
+ print(f"📁 File: {result.get('file_name')}")
1017
+ print(f"📄 Type: {result.get('file_type')}")
1018
+ print(f"💾 Size: {result.get('file_size_mb', 0):.2f} MB")
1019
+ if result.get("num_pages"):
1020
+ print(f"📖 Pages: {result['num_pages']}")
1021
+ print(f"📦 Chunks: {result.get('num_chunks', 0)}")
1022
+ print("=" * 60)
1023
+ else:
1024
+ print(f"❌ Indexing failed: {result.get('error')}")
1025
+ return 1
1026
+
1027
+ # List tools if requested
1028
+ if args.list_tools:
1029
+ agent.list_tools(verbose=True)
1030
+ return 0
1031
+
1032
+ # Single query mode
1033
+ if args.query:
1034
+ result = agent.process_query(args.query)
1035
+ print(f"\n{result['result']}")
1036
+
1037
+ if args.show_stats and result.get("duration"):
1038
+ print("\n📊 Stats:")
1039
+ print(f" Duration: {result['duration']:.2f}s")
1040
+ print(f" Steps: {result['steps_taken']}")
1041
+ print(f" Tokens: {result.get('total_tokens', 0):,}")
1042
+
1043
+ return 0 if result["status"] == "success" else 1
1044
+
1045
+ # Interactive mode
1046
+ interactive_mode(agent)
1047
+ return 0
1048
+
1049
+ except KeyboardInterrupt:
1050
+ print("\n\nInterrupted by user")
1051
+ return 130
1052
+ except Exception as e:
1053
+ logger.error(f"Error: {e}", exc_info=True)
1054
+ print(f"\n❌ Error: {e}")
1055
+ return 1
1056
+ finally:
1057
+ # Cleanup
1058
+ try:
1059
+ agent.stop_watching()
1060
+ except Exception: # pylint: disable=broad-except
1061
+ pass
1062
+
1063
+
1064
+ if __name__ == "__main__":
1065
+ sys.exit(main())