local-deep-research 0.1.26__py3-none-any.whl → 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. local_deep_research/__init__.py +23 -22
  2. local_deep_research/__main__.py +16 -0
  3. local_deep_research/advanced_search_system/__init__.py +7 -0
  4. local_deep_research/advanced_search_system/filters/__init__.py +8 -0
  5. local_deep_research/advanced_search_system/filters/base_filter.py +38 -0
  6. local_deep_research/advanced_search_system/filters/cross_engine_filter.py +200 -0
  7. local_deep_research/advanced_search_system/findings/base_findings.py +81 -0
  8. local_deep_research/advanced_search_system/findings/repository.py +452 -0
  9. local_deep_research/advanced_search_system/knowledge/__init__.py +1 -0
  10. local_deep_research/advanced_search_system/knowledge/base_knowledge.py +151 -0
  11. local_deep_research/advanced_search_system/knowledge/standard_knowledge.py +159 -0
  12. local_deep_research/advanced_search_system/questions/__init__.py +1 -0
  13. local_deep_research/advanced_search_system/questions/base_question.py +64 -0
  14. local_deep_research/advanced_search_system/questions/decomposition_question.py +445 -0
  15. local_deep_research/advanced_search_system/questions/standard_question.py +119 -0
  16. local_deep_research/advanced_search_system/repositories/__init__.py +7 -0
  17. local_deep_research/advanced_search_system/strategies/__init__.py +1 -0
  18. local_deep_research/advanced_search_system/strategies/base_strategy.py +118 -0
  19. local_deep_research/advanced_search_system/strategies/iterdrag_strategy.py +450 -0
  20. local_deep_research/advanced_search_system/strategies/parallel_search_strategy.py +312 -0
  21. local_deep_research/advanced_search_system/strategies/rapid_search_strategy.py +270 -0
  22. local_deep_research/advanced_search_system/strategies/standard_strategy.py +300 -0
  23. local_deep_research/advanced_search_system/tools/__init__.py +1 -0
  24. local_deep_research/advanced_search_system/tools/base_tool.py +100 -0
  25. local_deep_research/advanced_search_system/tools/knowledge_tools/__init__.py +1 -0
  26. local_deep_research/advanced_search_system/tools/question_tools/__init__.py +1 -0
  27. local_deep_research/advanced_search_system/tools/search_tools/__init__.py +1 -0
  28. local_deep_research/api/__init__.py +5 -5
  29. local_deep_research/api/research_functions.py +154 -160
  30. local_deep_research/app.py +8 -0
  31. local_deep_research/citation_handler.py +25 -16
  32. local_deep_research/{config.py → config/config_files.py} +102 -110
  33. local_deep_research/config/llm_config.py +472 -0
  34. local_deep_research/config/search_config.py +77 -0
  35. local_deep_research/defaults/__init__.py +10 -5
  36. local_deep_research/defaults/main.toml +2 -2
  37. local_deep_research/defaults/search_engines.toml +60 -34
  38. local_deep_research/main.py +121 -19
  39. local_deep_research/migrate_db.py +147 -0
  40. local_deep_research/report_generator.py +87 -45
  41. local_deep_research/search_system.py +153 -283
  42. local_deep_research/setup_data_dir.py +35 -0
  43. local_deep_research/test_migration.py +178 -0
  44. local_deep_research/utilities/__init__.py +0 -0
  45. local_deep_research/utilities/db_utils.py +49 -0
  46. local_deep_research/{utilties → utilities}/enums.py +2 -2
  47. local_deep_research/{utilties → utilities}/llm_utils.py +63 -29
  48. local_deep_research/utilities/search_utilities.py +242 -0
  49. local_deep_research/{utilties → utilities}/setup_utils.py +4 -2
  50. local_deep_research/web/__init__.py +0 -1
  51. local_deep_research/web/app.py +86 -1709
  52. local_deep_research/web/app_factory.py +289 -0
  53. local_deep_research/web/database/README.md +70 -0
  54. local_deep_research/web/database/migrate_to_ldr_db.py +289 -0
  55. local_deep_research/web/database/migrations.py +447 -0
  56. local_deep_research/web/database/models.py +117 -0
  57. local_deep_research/web/database/schema_upgrade.py +107 -0
  58. local_deep_research/web/models/database.py +294 -0
  59. local_deep_research/web/models/settings.py +94 -0
  60. local_deep_research/web/routes/api_routes.py +559 -0
  61. local_deep_research/web/routes/history_routes.py +354 -0
  62. local_deep_research/web/routes/research_routes.py +715 -0
  63. local_deep_research/web/routes/settings_routes.py +1583 -0
  64. local_deep_research/web/services/research_service.py +947 -0
  65. local_deep_research/web/services/resource_service.py +149 -0
  66. local_deep_research/web/services/settings_manager.py +669 -0
  67. local_deep_research/web/services/settings_service.py +187 -0
  68. local_deep_research/web/services/socket_service.py +210 -0
  69. local_deep_research/web/static/css/custom_dropdown.css +277 -0
  70. local_deep_research/web/static/css/settings.css +1223 -0
  71. local_deep_research/web/static/css/styles.css +525 -48
  72. local_deep_research/web/static/js/components/custom_dropdown.js +428 -0
  73. local_deep_research/web/static/js/components/detail.js +348 -0
  74. local_deep_research/web/static/js/components/fallback/formatting.js +122 -0
  75. local_deep_research/web/static/js/components/fallback/ui.js +215 -0
  76. local_deep_research/web/static/js/components/history.js +487 -0
  77. local_deep_research/web/static/js/components/logpanel.js +949 -0
  78. local_deep_research/web/static/js/components/progress.js +1107 -0
  79. local_deep_research/web/static/js/components/research.js +1865 -0
  80. local_deep_research/web/static/js/components/results.js +766 -0
  81. local_deep_research/web/static/js/components/settings.js +3981 -0
  82. local_deep_research/web/static/js/components/settings_sync.js +106 -0
  83. local_deep_research/web/static/js/main.js +226 -0
  84. local_deep_research/web/static/js/services/api.js +253 -0
  85. local_deep_research/web/static/js/services/audio.js +31 -0
  86. local_deep_research/web/static/js/services/formatting.js +119 -0
  87. local_deep_research/web/static/js/services/pdf.js +622 -0
  88. local_deep_research/web/static/js/services/socket.js +882 -0
  89. local_deep_research/web/static/js/services/ui.js +546 -0
  90. local_deep_research/web/templates/base.html +72 -0
  91. local_deep_research/web/templates/components/custom_dropdown.html +47 -0
  92. local_deep_research/web/templates/components/log_panel.html +32 -0
  93. local_deep_research/web/templates/components/mobile_nav.html +22 -0
  94. local_deep_research/web/templates/components/settings_form.html +299 -0
  95. local_deep_research/web/templates/components/sidebar.html +21 -0
  96. local_deep_research/web/templates/pages/details.html +73 -0
  97. local_deep_research/web/templates/pages/history.html +51 -0
  98. local_deep_research/web/templates/pages/progress.html +57 -0
  99. local_deep_research/web/templates/pages/research.html +139 -0
  100. local_deep_research/web/templates/pages/results.html +59 -0
  101. local_deep_research/web/templates/settings_dashboard.html +78 -192
  102. local_deep_research/web/utils/__init__.py +0 -0
  103. local_deep_research/web/utils/formatters.py +76 -0
  104. local_deep_research/web_search_engines/engines/full_search.py +18 -16
  105. local_deep_research/web_search_engines/engines/meta_search_engine.py +182 -131
  106. local_deep_research/web_search_engines/engines/search_engine_arxiv.py +224 -139
  107. local_deep_research/web_search_engines/engines/search_engine_brave.py +88 -71
  108. local_deep_research/web_search_engines/engines/search_engine_ddg.py +48 -39
  109. local_deep_research/web_search_engines/engines/search_engine_github.py +415 -204
  110. local_deep_research/web_search_engines/engines/search_engine_google_pse.py +123 -90
  111. local_deep_research/web_search_engines/engines/search_engine_guardian.py +210 -157
  112. local_deep_research/web_search_engines/engines/search_engine_local.py +532 -369
  113. local_deep_research/web_search_engines/engines/search_engine_local_all.py +42 -36
  114. local_deep_research/web_search_engines/engines/search_engine_pubmed.py +358 -266
  115. local_deep_research/web_search_engines/engines/search_engine_searxng.py +212 -160
  116. local_deep_research/web_search_engines/engines/search_engine_semantic_scholar.py +213 -170
  117. local_deep_research/web_search_engines/engines/search_engine_serpapi.py +84 -68
  118. local_deep_research/web_search_engines/engines/search_engine_wayback.py +186 -154
  119. local_deep_research/web_search_engines/engines/search_engine_wikipedia.py +115 -77
  120. local_deep_research/web_search_engines/search_engine_base.py +174 -99
  121. local_deep_research/web_search_engines/search_engine_factory.py +192 -102
  122. local_deep_research/web_search_engines/search_engines_config.py +22 -15
  123. {local_deep_research-0.1.26.dist-info → local_deep_research-0.2.2.dist-info}/METADATA +177 -97
  124. local_deep_research-0.2.2.dist-info/RECORD +135 -0
  125. {local_deep_research-0.1.26.dist-info → local_deep_research-0.2.2.dist-info}/WHEEL +1 -2
  126. {local_deep_research-0.1.26.dist-info → local_deep_research-0.2.2.dist-info}/entry_points.txt +3 -0
  127. local_deep_research/defaults/llm_config.py +0 -338
  128. local_deep_research/utilties/search_utilities.py +0 -114
  129. local_deep_research/web/static/js/app.js +0 -3763
  130. local_deep_research/web/templates/api_keys_config.html +0 -82
  131. local_deep_research/web/templates/collections_config.html +0 -90
  132. local_deep_research/web/templates/index.html +0 -348
  133. local_deep_research/web/templates/llm_config.html +0 -120
  134. local_deep_research/web/templates/main_config.html +0 -89
  135. local_deep_research/web/templates/search_engines_config.html +0 -154
  136. local_deep_research/web/templates/settings.html +0 -519
  137. local_deep_research-0.1.26.dist-info/RECORD +0 -61
  138. local_deep_research-0.1.26.dist-info/top_level.txt +0 -1
  139. /local_deep_research/{utilties → config}/__init__.py +0 -0
  140. {local_deep_research-0.1.26.dist-info → local_deep_research-0.2.2.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,949 @@
1
+ /**
2
+ * LogPanel Component
3
+ * Handles the display and interaction with the research log panel
4
+ * Used by both progress.js and results.js
5
+ */
6
+ (function() {
7
+ // Shared state for log panel
8
+ window._logPanelState = window._logPanelState || {
9
+ expanded: false,
10
+ queuedLogs: [],
11
+ logCount: 0,
12
+ initialized: false, // Track initialization state
13
+ connectedResearchId: null, // Track which research we're connected to
14
+ currentFilter: 'all' // Track current filter type
15
+ };
16
+
17
+ /**
18
+ * Initialize the log panel
19
+ * @param {string} researchId - Optional research ID to load logs for
20
+ */
21
+ function initializeLogPanel(researchId = null) {
22
+ // Check if already initialized
23
+ if (window._logPanelState.initialized) {
24
+ console.log('Log panel already initialized, checking if research ID has changed');
25
+
26
+ // If we're already connected to this research, do nothing
27
+ if (window._logPanelState.connectedResearchId === researchId) {
28
+ console.log('Already connected to research ID:', researchId);
29
+ return;
30
+ }
31
+
32
+ // If the research ID has changed, we'll update our connection
33
+ console.log('Research ID changed from', window._logPanelState.connectedResearchId, 'to', researchId);
34
+ window._logPanelState.connectedResearchId = researchId;
35
+ }
36
+
37
+ console.log('Initializing shared log panel, research ID:', researchId);
38
+
39
+ // Check if we're on a research-specific page (progress, results)
40
+ const isResearchPage = window.location.pathname.includes('/research/progress/') ||
41
+ window.location.pathname.includes('/research/results/') ||
42
+ document.getElementById('research-progress') ||
43
+ document.getElementById('research-results');
44
+
45
+ // Get all log panels on the page (there might be duplicates)
46
+ const logPanels = document.querySelectorAll('.collapsible-log-panel');
47
+
48
+ if (logPanels.length > 1) {
49
+ console.warn(`Found ${logPanels.length} log panels, removing duplicates`);
50
+
51
+ // Keep only the first one and remove others
52
+ for (let i = 1; i < logPanels.length; i++) {
53
+ console.log(`Removing duplicate log panel #${i}`);
54
+ logPanels[i].remove();
55
+ }
56
+ } else if (logPanels.length === 0) {
57
+ console.error('No log panel found in the DOM!');
58
+ return;
59
+ }
60
+
61
+ // Get log panel elements with both old and new names for compatibility
62
+ let logPanelToggle = document.getElementById('log-panel-toggle');
63
+ let logPanelContent = document.getElementById('log-panel-content');
64
+
65
+ // Fallback to the old element IDs if needed
66
+ if (!logPanelToggle) logPanelToggle = document.getElementById('logToggle');
67
+ if (!logPanelContent) logPanelContent = document.getElementById('logPanel');
68
+
69
+ if (!logPanelToggle || !logPanelContent) {
70
+ console.warn('Log panel elements not found, skipping initialization');
71
+ return;
72
+ }
73
+
74
+ // Handle visibility based on page type
75
+ if (!isResearchPage) {
76
+ console.log('Not on a research-specific page, hiding log panel');
77
+
78
+ // Hide the log panel on non-research pages
79
+ const panel = logPanelContent.closest('.collapsible-log-panel');
80
+ if (panel) {
81
+ panel.style.display = 'none';
82
+ } else if (logPanelContent.parentElement) {
83
+ logPanelContent.parentElement.style.display = 'none';
84
+ } else {
85
+ logPanelContent.style.display = 'none';
86
+ }
87
+ return;
88
+ } else {
89
+ // Ensure log panel is visible on research pages
90
+ console.log('On a research page, ensuring log panel is shown');
91
+ const panel = logPanelContent.closest('.collapsible-log-panel');
92
+ if (panel) {
93
+ panel.style.display = 'flex';
94
+ }
95
+ }
96
+
97
+ console.log('Log panel elements found, setting up handlers');
98
+
99
+ // Mark as initialized to prevent double initialization
100
+ window._logPanelState.initialized = true;
101
+
102
+ // Check for CSS issue - if the panel's computed style has display:none, the panel won't be visible
103
+ const computedStyle = window.getComputedStyle(logPanelContent);
104
+ console.log('Log panel CSS visibility:', {
105
+ display: computedStyle.display,
106
+ visibility: computedStyle.visibility,
107
+ height: computedStyle.height,
108
+ overflow: computedStyle.overflow
109
+ });
110
+
111
+ // Ensure the panel is visible in the DOM
112
+ if (computedStyle.display === 'none') {
113
+ console.warn('Log panel has display:none - forcing display:flex');
114
+ logPanelContent.style.display = 'flex';
115
+ }
116
+
117
+ // Ensure we have a console log container
118
+ const consoleLogContainer = document.getElementById('console-log-container');
119
+ if (!consoleLogContainer) {
120
+ console.error('Console log container not found, logs will not be displayed');
121
+ } else {
122
+ // Add placeholder message
123
+ consoleLogContainer.innerHTML = '<div class="empty-log-message">No logs available. Expand panel to load logs.</div>';
124
+ }
125
+
126
+ // Set up toggle click handler
127
+ logPanelToggle.addEventListener('click', function() {
128
+ console.log('Log panel toggle clicked');
129
+
130
+ // Toggle collapsed state
131
+ logPanelContent.classList.toggle('collapsed');
132
+ logPanelToggle.classList.toggle('collapsed');
133
+
134
+ // Update toggle icon
135
+ const toggleIcon = logPanelToggle.querySelector('.toggle-icon');
136
+ if (toggleIcon) {
137
+ if (logPanelContent.classList.contains('collapsed')) {
138
+ toggleIcon.className = 'fas fa-chevron-right toggle-icon';
139
+ } else {
140
+ toggleIcon.className = 'fas fa-chevron-down toggle-icon';
141
+
142
+ // Load logs if not already loaded
143
+ if (!logPanelContent.dataset.loaded && researchId) {
144
+ console.log('First expansion of log panel, loading logs');
145
+ loadLogsForResearch(researchId);
146
+ logPanelContent.dataset.loaded = 'true';
147
+ }
148
+
149
+ // Process any queued logs
150
+ if (window._logPanelState.queuedLogs.length > 0) {
151
+ console.log(`Processing ${window._logPanelState.queuedLogs.length} queued logs`);
152
+ window._logPanelState.queuedLogs.forEach(logEntry => {
153
+ addLogEntryToPanel(logEntry);
154
+ });
155
+ window._logPanelState.queuedLogs = [];
156
+ }
157
+ }
158
+ }
159
+
160
+ // Track expanded state
161
+ window._logPanelState.expanded = !logPanelContent.classList.contains('collapsed');
162
+ });
163
+
164
+ // Set up filter button click handlers
165
+ const filterButtons = document.querySelectorAll('.log-filter .filter-buttons button');
166
+ filterButtons.forEach(button => {
167
+ button.addEventListener('click', function() {
168
+ const type = this.textContent.toLowerCase();
169
+ console.log(`Filtering logs by type: ${type}`);
170
+
171
+ // Update active state
172
+ filterButtons.forEach(btn => btn.classList.remove('selected'));
173
+ this.classList.add('selected');
174
+
175
+ // Apply filtering
176
+ filterLogsByType(type);
177
+ });
178
+ });
179
+
180
+ // Start with panel collapsed and fix initial chevron direction
181
+ logPanelContent.classList.add('collapsed');
182
+ const initialToggleIcon = logPanelToggle.querySelector('.toggle-icon');
183
+ if (initialToggleIcon) {
184
+ initialToggleIcon.className = 'fas fa-chevron-right toggle-icon';
185
+ }
186
+
187
+ // Initialize the log count
188
+ const logIndicators = document.querySelectorAll('.log-indicator');
189
+ if (logIndicators.length > 0) {
190
+ // Set count on all indicators
191
+ logIndicators.forEach(indicator => {
192
+ indicator.textContent = '0';
193
+ });
194
+ } else {
195
+ console.warn('No log indicators found for initialization');
196
+ }
197
+
198
+ // Check CSS display property of the log panel
199
+ const logPanel = document.querySelector('.collapsible-log-panel');
200
+ if (logPanel) {
201
+ const panelStyle = window.getComputedStyle(logPanel);
202
+ console.log('Log panel CSS display:', panelStyle.display);
203
+
204
+ if (panelStyle.display === 'none') {
205
+ console.warn('Log panel has CSS display:none - forcing display:flex');
206
+ logPanel.style.display = 'flex';
207
+ }
208
+ }
209
+
210
+ // Pre-load logs if hash includes #logs
211
+ if (window.location.hash === '#logs' && researchId) {
212
+ console.log('Auto-loading logs due to #logs in URL');
213
+ setTimeout(() => {
214
+ logPanelToggle.click();
215
+ }, 500);
216
+ }
217
+
218
+ // DEBUG: Force expand the log panel if URL has debug parameter
219
+ if (window.location.search.includes('debug=logs') || window.location.hash.includes('debug')) {
220
+ console.log('DEBUG: Force-expanding log panel');
221
+ setTimeout(() => {
222
+ if (logPanelContent.classList.contains('collapsed')) {
223
+ logPanelToggle.click();
224
+ }
225
+ }, 800);
226
+ }
227
+
228
+ // Register global functions to ensure they work across modules
229
+ window.addConsoleLog = addConsoleLog;
230
+ window.filterLogsByType = filterLogsByType;
231
+
232
+ // Add a connector to socket.js
233
+ // Track when we last received this exact message to avoid re-adding within 10 seconds
234
+ const processedMessages = new Map();
235
+ window._socketAddLogEntry = function(logEntry) {
236
+ // Simple message deduplication for socket events
237
+ const message = logEntry.message || logEntry.content || '';
238
+ const messageKey = `${message}-${logEntry.type || 'info'}`;
239
+ const now = Date.now();
240
+
241
+ // Check if we've seen this message recently (within 10 seconds)
242
+ if (processedMessages.has(messageKey)) {
243
+ const lastProcessed = processedMessages.get(messageKey);
244
+ const timeDiff = now - lastProcessed;
245
+
246
+ if (timeDiff < 10000) { // 10 seconds
247
+ console.log(`Skipping duplicate socket message received within ${timeDiff}ms:`, message);
248
+ return;
249
+ }
250
+ }
251
+
252
+ // Update our tracking
253
+ processedMessages.set(messageKey, now);
254
+
255
+ // Clean up old entries (keep map from growing indefinitely)
256
+ if (processedMessages.size > 100) {
257
+ // Remove entries older than 60 seconds
258
+ for (const [key, timestamp] of processedMessages.entries()) {
259
+ if (now - timestamp > 60000) {
260
+ processedMessages.delete(key);
261
+ }
262
+ }
263
+ }
264
+
265
+ // Process the log entry
266
+ addLogEntryToPanel(logEntry);
267
+ };
268
+
269
+ console.log('Log panel initialized');
270
+ }
271
+
272
+ /**
273
+ * Load logs for a specific research
274
+ * @param {string} researchId - The research ID to load logs for
275
+ */
276
+ async function loadLogsForResearch(researchId) {
277
+ try {
278
+ // Show loading state
279
+ const logContent = document.getElementById('console-log-container');
280
+ if (logContent) {
281
+ logContent.innerHTML = '<div class="loading-spinner centered"><div class="spinner"></div><div style="margin-left: 10px;">Loading logs...</div></div>';
282
+ }
283
+
284
+ console.log('Loading logs for research ID:', researchId);
285
+
286
+ // Fetch logs from API
287
+ const response = await fetch(`/research/api/logs/${researchId}`);
288
+ const data = await response.json();
289
+
290
+ console.log('Logs API response:', data);
291
+
292
+ // Initialize array to hold all logs from different sources
293
+ const allLogs = [];
294
+
295
+ // Track seen messages to avoid duplicate content with different timestamps
296
+ const seenMessages = new Map();
297
+
298
+ // Process progress_log if available
299
+ if (data.progress_log && typeof data.progress_log === 'string') {
300
+ try {
301
+ const progressLogs = JSON.parse(data.progress_log);
302
+ if (Array.isArray(progressLogs) && progressLogs.length > 0) {
303
+ console.log(`Found ${progressLogs.length} logs in progress_log`);
304
+
305
+ // Process progress logs
306
+ progressLogs.forEach(logItem => {
307
+ if (!logItem.time || !logItem.message) return; // Skip invalid logs
308
+
309
+ // Skip if we've seen this exact message before
310
+ const messageKey = normalizeMessage(logItem.message);
311
+ if (seenMessages.has(messageKey)) {
312
+ // Only consider logs within 1 minute of each other as duplicates
313
+ const previousLog = seenMessages.get(messageKey);
314
+ const previousTime = new Date(previousLog.time);
315
+ const currentTime = new Date(logItem.time);
316
+ const timeDiff = Math.abs(currentTime - previousTime) / 1000; // in seconds
317
+
318
+ if (timeDiff < 60) { // Within 1 minute
319
+ // Use the newer timestamp if available
320
+ if (currentTime > previousTime) {
321
+ previousLog.time = logItem.time;
322
+ }
323
+ return; // Skip this duplicate
324
+ }
325
+
326
+ // If we get here, it's the same message but far apart in time (e.g., a repeated step)
327
+ // We'll include it as a separate entry
328
+ }
329
+
330
+ // Determine log type based on metadata
331
+ let logType = 'info';
332
+ if (logItem.metadata) {
333
+ if (logItem.metadata.phase === 'iteration_complete' ||
334
+ logItem.metadata.phase === 'report_complete' ||
335
+ logItem.metadata.phase === 'complete' ||
336
+ logItem.metadata.is_milestone === true) {
337
+ logType = 'milestone';
338
+ } else if (logItem.metadata.phase === 'error') {
339
+ logType = 'error';
340
+ }
341
+ }
342
+
343
+ // Add message keywords for better type detection
344
+ if (logType !== 'milestone') {
345
+ const msg = logItem.message.toLowerCase();
346
+ if (msg.includes('complete') ||
347
+ msg.includes('finished') ||
348
+ msg.includes('starting phase') ||
349
+ msg.includes('generated report')) {
350
+ logType = 'milestone';
351
+ } else if (msg.includes('error') || msg.includes('failed')) {
352
+ logType = 'error';
353
+ }
354
+ }
355
+
356
+ // Create a log entry object with a unique ID for deduplication
357
+ const logEntry = {
358
+ id: `${logItem.time}-${hashString(logItem.message)}`,
359
+ time: logItem.time,
360
+ message: logItem.message,
361
+ type: logType,
362
+ metadata: logItem.metadata || {},
363
+ source: 'progress_log'
364
+ };
365
+
366
+ // Track this message to avoid showing exact duplicates with different timestamps
367
+ seenMessages.set(messageKey, logEntry);
368
+
369
+ // Add to all logs array
370
+ allLogs.push(logEntry);
371
+ });
372
+ }
373
+ } catch (e) {
374
+ console.error('Error parsing progress_log:', e);
375
+ }
376
+ }
377
+
378
+ // Standard logs array processing
379
+ if (data && Array.isArray(data.logs)) {
380
+ console.log(`Processing ${data.logs.length} standard logs`);
381
+
382
+ // Process each standard log
383
+ data.logs.forEach(log => {
384
+ if (!log.timestamp && !log.time) return; // Skip invalid logs
385
+
386
+ // Skip duplicates based on message content
387
+ const messageKey = normalizeMessage(log.message || log.content || '');
388
+ if (seenMessages.has(messageKey)) {
389
+ // Only consider logs within 1 minute of each other as duplicates
390
+ const previousLog = seenMessages.get(messageKey);
391
+ const previousTime = new Date(previousLog.time);
392
+ const currentTime = new Date(log.timestamp || log.time);
393
+ const timeDiff = Math.abs(currentTime - previousTime) / 1000; // in seconds
394
+
395
+ if (timeDiff < 60) { // Within 1 minute
396
+ // Use the newer timestamp if available
397
+ if (currentTime > previousTime) {
398
+ previousLog.time = log.timestamp || log.time;
399
+ }
400
+ return; // Skip this duplicate
401
+ }
402
+ }
403
+
404
+ // Create standardized log entry
405
+ const logEntry = {
406
+ id: `${log.timestamp || log.time}-${hashString(log.message || log.content || '')}`,
407
+ time: log.timestamp || log.time,
408
+ message: log.message || log.content || 'No message',
409
+ type: log.type || log.level || 'info',
410
+ metadata: log.metadata || {},
411
+ source: 'standard_logs'
412
+ };
413
+
414
+ // Track this message
415
+ seenMessages.set(messageKey, logEntry);
416
+
417
+ // Add to all logs array
418
+ allLogs.push(logEntry);
419
+ });
420
+ }
421
+
422
+ // Clear container
423
+ if (logContent) {
424
+ if (allLogs.length === 0) {
425
+ logContent.innerHTML = '<div class="empty-log-message">No logs available for this research.</div>';
426
+ return;
427
+ }
428
+
429
+ logContent.innerHTML = '';
430
+
431
+ // Normalize timestamps - in case there are logs with mismatched AM/PM time zones
432
+ // This attempts to ensure logs are in a proper chronological order
433
+ normalizeTimestamps(allLogs);
434
+
435
+ // Deduplicate logs by ID and sort by timestamp (oldest first)
436
+ const uniqueLogsMap = new Map();
437
+ allLogs.forEach(log => {
438
+ // Use the ID as the key for deduplication
439
+ uniqueLogsMap.set(log.id, log);
440
+ });
441
+
442
+ // Convert map back to array
443
+ const uniqueLogs = Array.from(uniqueLogsMap.values());
444
+
445
+ // Sort logs by timestamp (oldest first)
446
+ const sortedLogs = uniqueLogs.sort((a, b) => {
447
+ return new Date(a.time) - new Date(b.time);
448
+ });
449
+
450
+ console.log(`Displaying ${sortedLogs.length} logs after deduplication (from original ${allLogs.length})`);
451
+
452
+ // Add each log entry to panel
453
+ sortedLogs.forEach(log => {
454
+ addLogEntryToPanel(log, false); // False means don't increment counter
455
+ });
456
+
457
+ // Update log count indicator
458
+ const logIndicators = document.querySelectorAll('.log-indicator');
459
+ if (logIndicators.length > 0) {
460
+ // Set all indicators to the same count
461
+ logIndicators.forEach(indicator => {
462
+ indicator.textContent = sortedLogs.length;
463
+ });
464
+ }
465
+ }
466
+
467
+ } catch (error) {
468
+ console.error('Error loading logs:', error);
469
+
470
+ // Show error in log panel
471
+ const logContent = document.getElementById('console-log-container');
472
+ if (logContent) {
473
+ logContent.innerHTML = `<div class="error-message">Error loading logs: ${error.message}</div>`;
474
+ }
475
+ }
476
+ }
477
+
478
+ /**
479
+ * Normalize a message for deduplication comparison
480
+ * @param {string} message - The message to normalize
481
+ * @returns {string} - Normalized message for comparison
482
+ */
483
+ function normalizeMessage(message) {
484
+ if (!message) return '';
485
+ // Remove extra whitespace and lowercase
486
+ return message.trim().toLowerCase();
487
+ }
488
+
489
+ /**
490
+ * Normalize timestamps across logs to ensure consistent ordering
491
+ * @param {Array} logs - The logs to normalize
492
+ */
493
+ function normalizeTimestamps(logs) {
494
+ // Find the most common date in the logs (ignoring the time)
495
+ const dateFrequency = new Map();
496
+
497
+ logs.forEach(log => {
498
+ try {
499
+ const date = new Date(log.time);
500
+ // Extract just the date part (YYYY-MM-DD)
501
+ const dateStr = date.toISOString().split('T')[0];
502
+ dateFrequency.set(dateStr, (dateFrequency.get(dateStr) || 0) + 1);
503
+ } catch (e) {
504
+ console.error('Error parsing date:', log.time);
505
+ }
506
+ });
507
+
508
+ // Find the most frequent date
509
+ let mostCommonDate = null;
510
+ let highestFrequency = 0;
511
+
512
+ dateFrequency.forEach((count, date) => {
513
+ if (count > highestFrequency) {
514
+ highestFrequency = count;
515
+ mostCommonDate = date;
516
+ }
517
+ });
518
+
519
+ console.log(`Most common date: ${mostCommonDate} with ${highestFrequency} occurrences`);
520
+
521
+ if (!mostCommonDate) return; // Can't normalize without a common date
522
+
523
+ // Normalize all logs to the most common date
524
+ logs.forEach(log => {
525
+ try {
526
+ const date = new Date(log.time);
527
+ const dateStr = date.toISOString().split('T')[0];
528
+
529
+ // If this log is from a different date, adjust it to the most common date
530
+ // while preserving the time portion
531
+ if (dateStr !== mostCommonDate) {
532
+ const [year, month, day] = mostCommonDate.split('-');
533
+ date.setFullYear(parseInt(year));
534
+ date.setMonth(parseInt(month) - 1); // Months are 0-indexed
535
+ date.setDate(parseInt(day));
536
+
537
+ // Update the log time
538
+ log.time = date.toISOString();
539
+ log.id = `${log.time}-${hashString(log.message)}`;
540
+ console.log(`Normalized timestamp for "${log.message.substring(0, 30)}..." from ${dateStr} to ${mostCommonDate}`);
541
+ }
542
+ } catch (e) {
543
+ console.error('Error normalizing date:', log.time);
544
+ }
545
+ });
546
+ }
547
+
548
+ /**
549
+ * Simple hash function for strings
550
+ * @param {string} str - String to hash
551
+ * @returns {string} - Hashed string for use as ID
552
+ */
553
+ function hashString(str) {
554
+ if (!str) return '0';
555
+ let hash = 0;
556
+ for (let i = 0; i < str.length; i++) {
557
+ const char = str.charCodeAt(i);
558
+ hash = ((hash << 5) - hash) + char;
559
+ hash = hash & hash; // Convert to 32bit integer
560
+ }
561
+ return hash.toString();
562
+ }
563
+
564
+ /**
565
+ * Add a log entry to the console - public API
566
+ * @param {string} message - Log message
567
+ * @param {string} level - Log level (info, milestone, error)
568
+ * @param {Object} metadata - Optional metadata
569
+ */
570
+ function addConsoleLog(message, level = 'info', metadata = null) {
571
+ console.log(`[${level.toUpperCase()}] ${message}`);
572
+
573
+ const timestamp = new Date().toISOString();
574
+ const logEntry = {
575
+ id: `${timestamp}-${hashString(message)}`,
576
+ time: timestamp,
577
+ message: message,
578
+ type: level,
579
+ metadata: metadata || { type: level }
580
+ };
581
+
582
+ // Queue log entries if panel is not expanded yet
583
+ if (!window._logPanelState.expanded) {
584
+ window._logPanelState.queuedLogs.push(logEntry);
585
+ console.log('Queued log entry for later display');
586
+
587
+ // Update log count even if not displaying yet
588
+ updateLogCounter(1);
589
+
590
+ // Auto-expand log panel on first log
591
+ const logPanelToggle = document.getElementById('log-panel-toggle');
592
+ if (logPanelToggle) {
593
+ console.log('Auto-expanding log panel because logs are available');
594
+ logPanelToggle.click();
595
+ }
596
+
597
+ return;
598
+ }
599
+
600
+ // Add directly to panel if it's expanded
601
+ addLogEntryToPanel(logEntry, true);
602
+ }
603
+
604
+ /**
605
+ * Add a log entry directly to the panel
606
+ * @param {Object} logEntry - The log entry to add
607
+ * @param {boolean} incrementCounter - Whether to increment the log counter
608
+ */
609
+ function addLogEntryToPanel(logEntry, incrementCounter = true) {
610
+ console.log('Adding log entry to panel:', logEntry);
611
+
612
+ const consoleLogContainer = document.getElementById('console-log-container');
613
+ if (!consoleLogContainer) {
614
+ console.warn('Console log container not found');
615
+ return;
616
+ }
617
+
618
+ // Clear empty message if present
619
+ const emptyMessage = consoleLogContainer.querySelector('.empty-log-message');
620
+ if (emptyMessage) {
621
+ emptyMessage.remove();
622
+ }
623
+
624
+ // Ensure the log entry has an ID
625
+ if (!logEntry.id) {
626
+ const timestamp = logEntry.time || logEntry.timestamp || new Date().toISOString();
627
+ const message = logEntry.message || logEntry.content || 'No message';
628
+ logEntry.id = `${timestamp}-${hashString(message)}`;
629
+ }
630
+
631
+ // More robust deduplication: First check by ID if available
632
+ if (logEntry.id) {
633
+ const existingEntryById = consoleLogContainer.querySelector(`.console-log-entry[data-log-id="${logEntry.id}"]`);
634
+ if (existingEntryById) {
635
+ console.log('Skipping duplicate log entry by ID:', logEntry.id);
636
+
637
+ // Increment counter on existing entry
638
+ let counter = parseInt(existingEntryById.dataset.counter || '1');
639
+ counter++;
640
+ existingEntryById.dataset.counter = counter;
641
+
642
+ // Update visual counter badge
643
+ if (counter > 1) {
644
+ let counterBadge = existingEntryById.querySelector('.duplicate-counter');
645
+ if (!counterBadge) {
646
+ counterBadge = document.createElement('span');
647
+ counterBadge.className = 'duplicate-counter';
648
+ existingEntryById.appendChild(counterBadge);
649
+ }
650
+ counterBadge.textContent = `(${counter}×)`;
651
+ }
652
+
653
+ // Still update the global counter if needed
654
+ if (incrementCounter) {
655
+ updateLogCounter(1);
656
+ }
657
+
658
+ return;
659
+ }
660
+ }
661
+
662
+ // Secondary check for duplicate by message content (for backward compatibility)
663
+ const existingEntries = consoleLogContainer.querySelectorAll('.console-log-entry');
664
+ if (existingEntries.length > 0) {
665
+ const message = logEntry.message || logEntry.content || '';
666
+ const logType = (logEntry.type || 'info').toLowerCase();
667
+
668
+ // Start from the end since newest logs are now at the bottom
669
+ for (let i = existingEntries.length - 1; i >= Math.max(0, existingEntries.length - 10); i--) {
670
+ // Only check the 10 most recent entries for efficiency
671
+ const entry = existingEntries[i];
672
+ const entryMessage = entry.querySelector('.log-message')?.textContent;
673
+ const entryType = entry.dataset.logType;
674
+
675
+ // If message and type match, consider it a duplicate (unless it's a milestone)
676
+ if (entryMessage === message &&
677
+ entryType === logType &&
678
+ logType !== 'milestone') {
679
+
680
+ console.log('Skipping duplicate log entry by content:', message);
681
+
682
+ // Increment counter on existing entry
683
+ let counter = parseInt(entry.dataset.counter || '1');
684
+ counter++;
685
+ entry.dataset.counter = counter;
686
+
687
+ // Update visual counter badge
688
+ if (counter > 1) {
689
+ let counterBadge = entry.querySelector('.duplicate-counter');
690
+ if (!counterBadge) {
691
+ counterBadge = document.createElement('span');
692
+ counterBadge.className = 'duplicate-counter';
693
+ entry.appendChild(counterBadge);
694
+ }
695
+ counterBadge.textContent = `(${counter}×)`;
696
+ }
697
+
698
+ // Still update the global counter if needed
699
+ if (incrementCounter) {
700
+ updateLogCounter(1);
701
+ }
702
+
703
+ return;
704
+ }
705
+ }
706
+ }
707
+
708
+ // Get the log template
709
+ const template = document.getElementById('console-log-entry-template');
710
+
711
+ // Determine log level - CHECK FOR DIRECT TYPE FIELD FIRST
712
+ let logLevel = 'info';
713
+ if (logEntry.type) {
714
+ logLevel = logEntry.type;
715
+ } else if (logEntry.metadata && logEntry.metadata.type) {
716
+ logLevel = logEntry.metadata.type;
717
+ } else if (logEntry.metadata && logEntry.metadata.phase) {
718
+ if (logEntry.metadata.phase === 'complete' ||
719
+ logEntry.metadata.phase === 'iteration_complete' ||
720
+ logEntry.metadata.phase === 'report_complete') {
721
+ logLevel = 'milestone';
722
+ } else if (logEntry.metadata.phase === 'error') {
723
+ logLevel = 'error';
724
+ }
725
+ } else if (logEntry.level) {
726
+ logLevel = logEntry.level;
727
+ }
728
+
729
+ // Format timestamp
730
+ const timestamp = new Date(logEntry.time || logEntry.timestamp || new Date());
731
+ const timeStr = timestamp.toLocaleTimeString();
732
+
733
+ // Get message
734
+ const message = logEntry.message || logEntry.content || 'No message';
735
+
736
+ if (template) {
737
+ // Create a new log entry from the template
738
+ const entry = document.importNode(template.content, true);
739
+ const logEntryElement = entry.querySelector('.console-log-entry');
740
+
741
+ // Add the log type as data attribute for filtering
742
+ if (logEntryElement) {
743
+ logEntryElement.dataset.logType = logLevel.toLowerCase();
744
+ logEntryElement.classList.add(`log-${logLevel.toLowerCase()}`);
745
+ // Initialize counter for duplicate tracking
746
+ logEntryElement.dataset.counter = '1';
747
+ // Store log ID for deduplication
748
+ if (logEntry.id) {
749
+ logEntryElement.dataset.logId = logEntry.id;
750
+ }
751
+ }
752
+
753
+ // Set content
754
+ entry.querySelector('.log-timestamp').textContent = timeStr;
755
+ entry.querySelector('.log-badge').textContent = logLevel.charAt(0).toUpperCase() + logLevel.slice(1);
756
+ entry.querySelector('.log-badge').className = `log-badge ${logLevel.toLowerCase()}`;
757
+ entry.querySelector('.log-message').textContent = message;
758
+
759
+ // Add to container (at the end for oldest first)
760
+ consoleLogContainer.appendChild(entry);
761
+ } else {
762
+ // Create a simple log entry without template
763
+ const entry = document.createElement('div');
764
+ entry.className = 'console-log-entry';
765
+ entry.dataset.logType = logLevel.toLowerCase();
766
+ entry.classList.add(`log-${logLevel.toLowerCase()}`);
767
+ entry.dataset.counter = '1';
768
+ if (logEntry.id) {
769
+ entry.dataset.logId = logEntry.id;
770
+ }
771
+
772
+ // Create log content
773
+ entry.innerHTML = `
774
+ <span class="log-timestamp">${timeStr}</span>
775
+ <span class="log-badge ${logLevel.toLowerCase()}">${logLevel.charAt(0).toUpperCase() + logLevel.slice(1)}</span>
776
+ <span class="log-message">${message}</span>
777
+ `;
778
+
779
+ // Add to container (at the end for oldest first)
780
+ consoleLogContainer.appendChild(entry);
781
+ }
782
+
783
+ // Check if the entry should be visible based on current filter
784
+ const currentFilter = window._logPanelState.currentFilter || 'all';
785
+ const shouldShow = checkLogVisibility(logLevel.toLowerCase(), currentFilter);
786
+
787
+ // Apply visibility based on the current filter
788
+ const newEntry = consoleLogContainer.lastElementChild;
789
+ if (newEntry) {
790
+ newEntry.style.display = shouldShow ? '' : 'none';
791
+ }
792
+
793
+ // Update log count using helper function if needed
794
+ if (incrementCounter) {
795
+ updateLogCounter(1);
796
+ }
797
+
798
+ // No need to scroll when loading all logs
799
+ // Scroll will be handled after all logs are loaded
800
+ if (incrementCounter) {
801
+ // Auto-scroll to newest log (at the bottom)
802
+ setTimeout(() => {
803
+ consoleLogContainer.scrollTop = consoleLogContainer.scrollHeight;
804
+ }, 0);
805
+ }
806
+ }
807
+
808
+ /**
809
+ * Helper function to update the log counter
810
+ * @param {number} increment - Amount to increment the counter by
811
+ */
812
+ function updateLogCounter(increment) {
813
+ const logIndicators = document.querySelectorAll('.log-indicator');
814
+ if (logIndicators.length > 0) {
815
+ const currentCount = parseInt(logIndicators[0].textContent) || 0;
816
+ const newCount = currentCount + increment;
817
+
818
+ // Update all indicators
819
+ logIndicators.forEach(indicator => {
820
+ indicator.textContent = newCount;
821
+ });
822
+ }
823
+ }
824
+
825
+ /**
826
+ * Check if a log entry should be visible based on filter type
827
+ * @param {string} logType - The type of log (info, milestone, error)
828
+ * @param {string} filterType - The selected filter (all, info, milestone, error)
829
+ * @returns {boolean} - Whether the log should be visible
830
+ */
831
+ function checkLogVisibility(logType, filterType) {
832
+ switch (filterType) {
833
+ case 'all':
834
+ return true;
835
+ case 'info':
836
+ return logType === 'info';
837
+ case 'milestone':
838
+ case 'milestones': // Handle plural form too
839
+ return logType === 'milestone';
840
+ case 'error':
841
+ case 'errors': // Handle plural form too
842
+ return logType === 'error';
843
+ default:
844
+ return true; // Default to showing everything
845
+ }
846
+ }
847
+
848
+ /**
849
+ * Filter logs by type
850
+ * @param {string} filterType - The type to filter by (all, info, milestone, error)
851
+ */
852
+ function filterLogsByType(filterType = 'all') {
853
+ console.log('Filtering logs by type:', filterType);
854
+
855
+ filterType = filterType.toLowerCase();
856
+
857
+ // Store current filter in shared state
858
+ window._logPanelState.currentFilter = filterType;
859
+
860
+ // Get all log entries from the DOM
861
+ const logEntries = document.querySelectorAll('.console-log-entry');
862
+ console.log(`Found ${logEntries.length} log entries to filter`);
863
+
864
+ let visibleCount = 0;
865
+
866
+ // Apply filters
867
+ logEntries.forEach(entry => {
868
+ // Use data attribute for log type
869
+ const logType = entry.dataset.logType || 'info';
870
+
871
+ // Determine visibility based on filter type
872
+ const shouldShow = checkLogVisibility(logType, filterType);
873
+
874
+ // Set display style based on filter result
875
+ entry.style.display = shouldShow ? '' : 'none';
876
+
877
+ if (shouldShow) {
878
+ visibleCount++;
879
+ }
880
+ });
881
+
882
+ console.log(`Filtering complete. Showing ${visibleCount} of ${logEntries.length} logs`);
883
+
884
+ // Show 'no logs' message if all logs are filtered out
885
+ const consoleContainer = document.getElementById('console-log-container');
886
+ if (consoleContainer && logEntries.length > 0) {
887
+ // Remove any existing empty message
888
+ const existingEmptyMessage = consoleContainer.querySelector('.empty-log-message');
889
+ if (existingEmptyMessage) {
890
+ existingEmptyMessage.remove();
891
+ }
892
+
893
+ // Add empty message if needed
894
+ if (visibleCount === 0) {
895
+ console.log(`Adding 'no logs' message for filter: ${filterType}`);
896
+ const newEmptyMessage = document.createElement('div');
897
+ newEmptyMessage.className = 'empty-log-message';
898
+ newEmptyMessage.textContent = `No ${filterType} logs to display.`;
899
+ consoleContainer.appendChild(newEmptyMessage);
900
+ }
901
+ }
902
+ }
903
+
904
+ // Expose public API
905
+ window.logPanel = {
906
+ initialize: initializeLogPanel,
907
+ addLog: addConsoleLog,
908
+ filterLogs: filterLogsByType,
909
+ loadLogs: loadLogsForResearch
910
+ };
911
+
912
+ // Self-invoke to initialize when DOM content is loaded
913
+ document.addEventListener('DOMContentLoaded', function() {
914
+ console.log('DOM ready - checking if log panel should be initialized');
915
+
916
+ // Find research ID from URL if available
917
+ let researchId = null;
918
+ const urlMatch = window.location.pathname.match(/\/research\/(progress|results)\/(\d+)/);
919
+ if (urlMatch && urlMatch[2]) {
920
+ researchId = urlMatch[2];
921
+ console.log('Found research ID in URL:', researchId);
922
+
923
+ // Store the current research ID in the state
924
+ window._logPanelState.connectedResearchId = researchId;
925
+ }
926
+
927
+ // Check for research page elements
928
+ const isResearchPage = window.location.pathname.includes('/research/progress/') ||
929
+ window.location.pathname.includes('/research/results/') ||
930
+ document.getElementById('research-progress') ||
931
+ document.getElementById('research-results');
932
+
933
+ // Initialize log panel if on a research page
934
+ if (isResearchPage) {
935
+ console.log('On a research page, initializing log panel for research ID:', researchId);
936
+ initializeLogPanel(researchId);
937
+
938
+ // Extra check: If we have a research ID but panel not initialized properly
939
+ setTimeout(() => {
940
+ if (researchId && !window._logPanelState.initialized) {
941
+ console.log('Log panel not initialized properly, retrying...');
942
+ initializeLogPanel(researchId);
943
+ }
944
+ }, 1000);
945
+ } else {
946
+ console.log('Not on a research page, skipping log panel initialization');
947
+ }
948
+ });
949
+ })();