cursorflow 1.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,410 @@
1
+ """
2
+ Universal Log Collector
3
+
4
+ Framework-agnostic log monitoring that works with any log source:
5
+ SSH remote, local files, Docker containers, cloud services.
6
+ """
7
+
8
+ import asyncio
9
+ import time
10
+ import re
11
+ from typing import Dict, List, Optional, Any
12
+ from datetime import datetime
13
+ import logging
14
+ from pathlib import Path
15
+
16
+ from ..log_sources.ssh_remote import SSHRemoteLogSource
17
+ from ..log_sources.local_file import LocalFileLogSource
18
+
19
+
20
+ class LogCollector:
21
+ """
22
+ Universal log collection - works with any backend technology
23
+
24
+ Supports multiple log sources simultaneously and provides
25
+ unified interface for correlation with browser events.
26
+ """
27
+
28
+ def __init__(self, config: Dict):
29
+ """
30
+ Initialize log collector with source configuration
31
+
32
+ Args:
33
+ config: {
34
+ "source": "ssh|local|docker|cloud",
35
+ "host": "server.com", # for SSH
36
+ "user": "deploy", # for SSH
37
+ "key_file": "~/.ssh/key", # for SSH
38
+ "paths": ["/var/log/app.log", "logs/error.log"],
39
+ "containers": ["app", "nginx"] # for Docker
40
+ }
41
+ """
42
+ self.config = config
43
+ self.source_type = config.get("source", "local")
44
+ self.log_sources = []
45
+ self.monitoring = False
46
+ self.collected_logs = []
47
+
48
+ self.logger = logging.getLogger(__name__)
49
+
50
+ # Initialize log sources
51
+ self._initialize_sources()
52
+
53
+ def _initialize_sources(self):
54
+ """Initialize appropriate log sources based on configuration"""
55
+ try:
56
+ if self.source_type == "ssh":
57
+ self._init_ssh_sources()
58
+ elif self.source_type == "local":
59
+ self._init_local_sources()
60
+ elif self.source_type == "docker":
61
+ self._init_docker_sources()
62
+ elif self.source_type == "cloud":
63
+ self._init_cloud_sources()
64
+ else:
65
+ raise ValueError(f"Unsupported log source type: {self.source_type}")
66
+
67
+ self.logger.info(f"Initialized {len(self.log_sources)} log sources ({self.source_type})")
68
+
69
+ except Exception as e:
70
+ self.logger.error(f"Log source initialization failed: {e}")
71
+ raise
72
+
73
+ def _init_ssh_sources(self):
74
+ """Initialize SSH remote log sources"""
75
+ ssh_config = {
76
+ "hostname": self.config.get("host"),
77
+ "username": self.config.get("user", "deploy"),
78
+ "key_filename": self.config.get("key_file")
79
+ }
80
+
81
+ paths = self.config.get("paths", [])
82
+ for path in paths:
83
+ source = SSHRemoteLogSource(ssh_config, path)
84
+ self.log_sources.append(source)
85
+
86
+ def _init_local_sources(self):
87
+ """Initialize local file log sources"""
88
+ paths = self.config.get("paths", ["logs/app.log"])
89
+
90
+ for path in paths:
91
+ if Path(path).exists() or self.config.get("create_if_missing", True):
92
+ source = LocalFileLogSource(path)
93
+ self.log_sources.append(source)
94
+ else:
95
+ self.logger.warning(f"Log file not found: {path}")
96
+
97
+ def _init_docker_sources(self):
98
+ """Initialize Docker container log sources"""
99
+ containers = self.config.get("containers", [])
100
+
101
+ for container in containers:
102
+ try:
103
+ source = DockerLogSource(container)
104
+ self.log_sources.append(source)
105
+ except Exception as e:
106
+ self.logger.warning(f"Docker container {container} not available: {e}")
107
+
108
+ def _init_cloud_sources(self):
109
+ """Initialize cloud log sources (AWS, GCP, Azure)"""
110
+ provider = self.config.get("provider", "aws")
111
+
112
+ if provider == "aws":
113
+ source = AWSCloudWatchSource(self.config)
114
+ self.log_sources.append(source)
115
+ elif provider == "gcp":
116
+ source = GCPLoggingSource(self.config)
117
+ self.log_sources.append(source)
118
+ else:
119
+ raise ValueError(f"Unsupported cloud provider: {provider}")
120
+
121
+ async def start_monitoring(self):
122
+ """Start monitoring all configured log sources"""
123
+ if self.monitoring:
124
+ return
125
+
126
+ self.monitoring = True
127
+ self.collected_logs = []
128
+
129
+ try:
130
+ # Start all log sources
131
+ tasks = []
132
+ for source in self.log_sources:
133
+ task = asyncio.create_task(self._monitor_source(source))
134
+ tasks.append(task)
135
+
136
+ self.logger.info(f"Started monitoring {len(self.log_sources)} log sources")
137
+
138
+ # Let monitoring run (don't await tasks - they run continuously)
139
+
140
+ except Exception as e:
141
+ self.logger.error(f"Failed to start log monitoring: {e}")
142
+ raise
143
+
144
+ async def stop_monitoring(self) -> List[Dict]:
145
+ """Stop monitoring and return collected logs"""
146
+ if not self.monitoring:
147
+ return []
148
+
149
+ self.monitoring = False
150
+
151
+ try:
152
+ # Stop all sources
153
+ for source in self.log_sources:
154
+ if hasattr(source, 'stop'):
155
+ await source.stop()
156
+
157
+ self.logger.info(f"Stopped monitoring. Collected {len(self.collected_logs)} log entries")
158
+ return self.collected_logs.copy()
159
+
160
+ except Exception as e:
161
+ self.logger.error(f"Failed to stop log monitoring: {e}")
162
+ return self.collected_logs.copy()
163
+
164
+ async def _monitor_source(self, source):
165
+ """Monitor a single log source"""
166
+ try:
167
+ await source.connect()
168
+
169
+ while self.monitoring:
170
+ try:
171
+ # Get new log entries
172
+ entries = await source.get_new_entries()
173
+
174
+ for entry in entries:
175
+ processed_entry = self._process_log_entry(entry, source)
176
+ self.collected_logs.append(processed_entry)
177
+
178
+ # Brief pause to avoid overwhelming
179
+ await asyncio.sleep(0.1)
180
+
181
+ except Exception as e:
182
+ self.logger.warning(f"Log source error: {e}")
183
+ await asyncio.sleep(1) # Wait before retry
184
+
185
+ except Exception as e:
186
+ self.logger.error(f"Log source monitoring failed: {e}")
187
+ finally:
188
+ try:
189
+ await source.disconnect()
190
+ except:
191
+ pass
192
+
193
+ def _process_log_entry(self, entry: str, source) -> Dict:
194
+ """Process raw log entry into structured format"""
195
+ timestamp = time.time()
196
+
197
+ # Parse timestamp from log entry if present
198
+ parsed_timestamp = self._extract_timestamp(entry)
199
+ if parsed_timestamp:
200
+ timestamp = parsed_timestamp
201
+
202
+ # Classify log level
203
+ level = self._classify_log_level(entry)
204
+
205
+ # Extract relevant information
206
+ processed = {
207
+ "timestamp": timestamp,
208
+ "source": getattr(source, 'name', str(source)),
209
+ "source_type": self.source_type,
210
+ "level": level,
211
+ "content": entry.strip(),
212
+ "raw": entry
213
+ }
214
+
215
+ # Add error classification if it's an error
216
+ if level in ["error", "critical"]:
217
+ processed["error_type"] = self._classify_error_type(entry)
218
+
219
+ return processed
220
+
221
+ def _extract_timestamp(self, entry: str) -> Optional[float]:
222
+ """Extract timestamp from log entry"""
223
+ # Common timestamp patterns
224
+ patterns = [
225
+ # Apache/Nginx: [Mon Dec 04 15:30:45 2023]
226
+ r'\[([A-Za-z]{3} [A-Za-z]{3} \d{2} \d{2}:\d{2}:\d{2} \d{4})\]',
227
+ # ISO format: 2023-12-04T15:30:45.123Z
228
+ r'(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z?)',
229
+ # Simple format: 2023-12-04 15:30:45
230
+ r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})',
231
+ ]
232
+
233
+ for pattern in patterns:
234
+ match = re.search(pattern, entry)
235
+ if match:
236
+ try:
237
+ timestamp_str = match.group(1)
238
+ # Convert to Unix timestamp
239
+ if 'T' in timestamp_str:
240
+ dt = datetime.fromisoformat(timestamp_str.replace('Z', '+00:00'))
241
+ else:
242
+ dt = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S')
243
+ return dt.timestamp()
244
+ except:
245
+ continue
246
+
247
+ return None
248
+
249
+ def _classify_log_level(self, entry: str) -> str:
250
+ """Classify log entry level"""
251
+ entry_lower = entry.lower()
252
+
253
+ if any(word in entry_lower for word in ['error', 'failed', 'exception', 'fatal']):
254
+ return "error"
255
+ elif any(word in entry_lower for word in ['warning', 'warn']):
256
+ return "warning"
257
+ elif any(word in entry_lower for word in ['info', 'information']):
258
+ return "info"
259
+ elif any(word in entry_lower for word in ['debug', 'trace']):
260
+ return "debug"
261
+ else:
262
+ return "info"
263
+
264
+ def _classify_error_type(self, entry: str) -> str:
265
+ """Classify type of error for better correlation"""
266
+ entry_lower = entry.lower()
267
+
268
+ # Database errors
269
+ if any(word in entry_lower for word in ['mysql', 'postgres', 'database', 'sql', 'dbd']):
270
+ return "database"
271
+
272
+ # Authentication errors
273
+ elif any(word in entry_lower for word in ['auth', 'login', 'permission', 'unauthorized']):
274
+ return "authentication"
275
+
276
+ # Network errors
277
+ elif any(word in entry_lower for word in ['connection', 'network', 'timeout', 'refused']):
278
+ return "network"
279
+
280
+ # File system errors
281
+ elif any(word in entry_lower for word in ['file', 'permission', 'not found', 'locate']):
282
+ return "filesystem"
283
+
284
+ # Application errors
285
+ elif any(word in entry_lower for word in ['can\'t', 'cannot', 'undefined', 'null']):
286
+ return "application"
287
+
288
+ else:
289
+ return "general"
290
+
291
+ def get_logs_in_timeframe(self, start_time: float, end_time: float) -> List[Dict]:
292
+ """Get logs within specific timeframe for correlation"""
293
+ matching_logs = []
294
+
295
+ for log_entry in self.collected_logs:
296
+ timestamp = log_entry.get("timestamp", 0)
297
+ if start_time <= timestamp <= end_time:
298
+ matching_logs.append(log_entry)
299
+
300
+ return matching_logs
301
+
302
+ def get_error_logs_since(self, since_time: float) -> List[Dict]:
303
+ """Get error logs since specific time"""
304
+ error_logs = []
305
+
306
+ for log_entry in self.collected_logs:
307
+ if (log_entry.get("timestamp", 0) >= since_time and
308
+ log_entry.get("level") in ["error", "critical"]):
309
+ error_logs.append(log_entry)
310
+
311
+ return error_logs
312
+
313
+
314
+ # Docker log source implementation
315
+ class DockerLogSource:
316
+ """Log source for Docker containers"""
317
+
318
+ def __init__(self, container_name: str):
319
+ self.container_name = container_name
320
+ self.name = f"docker:{container_name}"
321
+ self.process = None
322
+
323
+ async def connect(self):
324
+ """Connect to Docker container logs"""
325
+ try:
326
+ import subprocess
327
+ self.process = await asyncio.create_subprocess_exec(
328
+ "docker", "logs", "-f", self.container_name,
329
+ stdout=asyncio.subprocess.PIPE,
330
+ stderr=asyncio.subprocess.STDOUT
331
+ )
332
+ except Exception as e:
333
+ raise Exception(f"Failed to connect to Docker container {self.container_name}: {e}")
334
+
335
+ async def get_new_entries(self) -> List[str]:
336
+ """Get new log entries from Docker container"""
337
+ if not self.process:
338
+ return []
339
+
340
+ try:
341
+ # Read with timeout
342
+ line = await asyncio.wait_for(
343
+ self.process.stdout.readline(),
344
+ timeout=1.0
345
+ )
346
+
347
+ if line:
348
+ return [line.decode('utf-8', errors='ignore')]
349
+ else:
350
+ return []
351
+
352
+ except asyncio.TimeoutError:
353
+ return []
354
+ except Exception:
355
+ return []
356
+
357
+ async def disconnect(self):
358
+ """Disconnect from Docker logs"""
359
+ if self.process:
360
+ try:
361
+ self.process.terminate()
362
+ await self.process.wait()
363
+ except:
364
+ pass
365
+
366
+
367
+ # AWS CloudWatch source implementation
368
+ class AWSCloudWatchSource:
369
+ """Log source for AWS CloudWatch"""
370
+
371
+ def __init__(self, config: Dict):
372
+ self.config = config
373
+ self.name = f"aws:{config.get('log_group', 'unknown')}"
374
+
375
+ async def connect(self):
376
+ """Connect to AWS CloudWatch"""
377
+ # Implementation would use boto3
378
+ pass
379
+
380
+ async def get_new_entries(self) -> List[str]:
381
+ """Get new entries from CloudWatch"""
382
+ # Implementation would query CloudWatch logs
383
+ return []
384
+
385
+ async def disconnect(self):
386
+ """Disconnect from CloudWatch"""
387
+ pass
388
+
389
+
390
+ # GCP Logging source implementation
391
+ class GCPLoggingSource:
392
+ """Log source for Google Cloud Logging"""
393
+
394
+ def __init__(self, config: Dict):
395
+ self.config = config
396
+ self.name = f"gcp:{config.get('project_id', 'unknown')}"
397
+
398
+ async def connect(self):
399
+ """Connect to GCP Logging"""
400
+ # Implementation would use google-cloud-logging
401
+ pass
402
+
403
+ async def get_new_entries(self) -> List[str]:
404
+ """Get new entries from GCP Logging"""
405
+ # Implementation would query GCP logs
406
+ return []
407
+
408
+ async def disconnect(self):
409
+ """Disconnect from GCP Logging"""
410
+ pass
@@ -0,0 +1,179 @@
1
+ """
2
+ Universal Log Monitor
3
+
4
+ Coordinates different log sources (SSH, local files, Docker, etc.)
5
+ and provides a unified interface for log collection and filtering.
6
+ """
7
+
8
+ import asyncio
9
+ import logging
10
+ from typing import Dict, List, Optional, Any
11
+ from datetime import datetime, timedelta
12
+
13
+ class LogMonitor:
14
+ """Universal log monitoring coordinator"""
15
+
16
+ def __init__(self, log_source_type: str, config: Dict):
17
+ """
18
+ Initialize log monitor with specified source type
19
+
20
+ Args:
21
+ log_source_type: Type of log source ('ssh', 'local', 'docker', 'systemd')
22
+ config: Configuration for the log source
23
+ """
24
+ self.log_source_type = log_source_type
25
+ self.config = config
26
+ self.log_source = None
27
+ self.monitoring = False
28
+
29
+ self.logger = logging.getLogger(__name__)
30
+
31
+ # Load appropriate log source
32
+ self._load_log_source()
33
+
34
+ def _load_log_source(self):
35
+ """Dynamically load the appropriate log source"""
36
+
37
+ source_map = {
38
+ 'ssh': ('ssh_remote', 'SSHRemoteLogSource'),
39
+ 'local': ('local_file', 'LocalFileLogSource'),
40
+ 'docker': ('docker_logs', 'DockerLogSource'),
41
+ 'systemd': ('systemd_logs', 'SystemdLogSource')
42
+ }
43
+
44
+ if self.log_source_type not in source_map:
45
+ raise ValueError(f"Unsupported log source type: {self.log_source_type}")
46
+
47
+ module_name, class_name = source_map[self.log_source_type]
48
+
49
+ try:
50
+ # Dynamic import
51
+ module = __import__(f"..log_sources.{module_name}", fromlist=[class_name], level=1)
52
+ source_class = getattr(module, class_name)
53
+
54
+ # Initialize with config
55
+ if self.log_source_type == 'ssh':
56
+ self.log_source = source_class(
57
+ self.config.get('ssh_config', {}),
58
+ self.config.get('log_paths', {})
59
+ )
60
+ elif self.log_source_type == 'local':
61
+ self.log_source = source_class(
62
+ self.config.get('log_paths', {})
63
+ )
64
+ else:
65
+ self.log_source = source_class(self.config)
66
+
67
+ except ImportError as e:
68
+ raise ImportError(f"Log source {self.log_source_type} not available: {e}")
69
+
70
+ async def start_monitoring(self) -> bool:
71
+ """Start log monitoring"""
72
+
73
+ if self.monitoring:
74
+ return True
75
+
76
+ self.logger.info(f"Starting {self.log_source_type} log monitoring")
77
+
78
+ success = await self.log_source.start_monitoring()
79
+ self.monitoring = success
80
+
81
+ return success
82
+
83
+ async def stop_monitoring(self) -> List[Dict]:
84
+ """Stop monitoring and return all collected logs"""
85
+
86
+ if not self.monitoring:
87
+ return []
88
+
89
+ self.logger.info("Stopping log monitoring")
90
+
91
+ logs = await self.log_source.stop_monitoring()
92
+ self.monitoring = False
93
+
94
+ return logs
95
+
96
+ def get_recent_logs(self, seconds: int = 10) -> List[Dict]:
97
+ """Get recent log entries without stopping monitoring"""
98
+
99
+ if not self.monitoring or not self.log_source:
100
+ return []
101
+
102
+ return self.log_source.get_recent_logs(seconds)
103
+
104
+ def filter_logs(self, logs: List[Dict], filters: Dict) -> List[Dict]:
105
+ """Filter logs based on criteria"""
106
+
107
+ filtered = logs
108
+
109
+ # Filter by time range
110
+ if 'since' in filters:
111
+ since_time = filters['since']
112
+ if isinstance(since_time, str):
113
+ # Parse string to datetime
114
+ since_time = datetime.fromisoformat(since_time)
115
+ filtered = [log for log in filtered if log['timestamp'] >= since_time]
116
+
117
+ # Filter by log source
118
+ if 'source' in filters:
119
+ source = filters['source']
120
+ filtered = [log for log in filtered if log['source'] == source]
121
+
122
+ # Filter by content pattern
123
+ if 'pattern' in filters:
124
+ import re
125
+ pattern = filters['pattern']
126
+ filtered = [log for log in filtered if re.search(pattern, log['content'], re.IGNORECASE)]
127
+
128
+ # Filter by severity (if log source provides it)
129
+ if 'severity' in filters:
130
+ severity = filters['severity']
131
+ filtered = [log for log in filtered if log.get('severity') == severity]
132
+
133
+ return filtered
134
+
135
+ def categorize_logs(self, logs: List[Dict], error_patterns: Dict) -> Dict[str, List[Dict]]:
136
+ """Categorize logs by error patterns"""
137
+
138
+ categorized = {
139
+ 'errors': [],
140
+ 'warnings': [],
141
+ 'info': [],
142
+ 'unknown': []
143
+ }
144
+
145
+ for log_entry in logs:
146
+ content = log_entry['content']
147
+ categorized_entry = log_entry.copy()
148
+
149
+ # Try to match against error patterns
150
+ matched = False
151
+ for pattern_name, pattern_config in error_patterns.items():
152
+ if re.search(pattern_config['regex'], content):
153
+ categorized_entry['error_type'] = pattern_name
154
+ categorized_entry['severity'] = pattern_config['severity']
155
+ categorized_entry['description'] = pattern_config['description']
156
+ categorized_entry['suggested_fix'] = pattern_config['suggested_fix']
157
+
158
+ # Categorize by severity
159
+ if pattern_config['severity'] in ['critical', 'high']:
160
+ categorized['errors'].append(categorized_entry)
161
+ elif pattern_config['severity'] == 'medium':
162
+ categorized['warnings'].append(categorized_entry)
163
+ else:
164
+ categorized['info'].append(categorized_entry)
165
+
166
+ matched = True
167
+ break
168
+
169
+ if not matched:
170
+ # Basic severity detection from log content
171
+ content_lower = content.lower()
172
+ if any(word in content_lower for word in ['error', 'failed', 'exception', 'critical']):
173
+ categorized['errors'].append(categorized_entry)
174
+ elif any(word in content_lower for word in ['warning', 'warn']):
175
+ categorized['warnings'].append(categorized_entry)
176
+ else:
177
+ categorized['info'].append(categorized_entry)
178
+
179
+ return categorized