logler 1.0.7__cp311-cp311-macosx_11_0_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- logler/__init__.py +22 -0
- logler/bootstrap.py +57 -0
- logler/cache.py +75 -0
- logler/cli.py +589 -0
- logler/helpers.py +282 -0
- logler/investigate.py +3962 -0
- logler/llm_cli.py +1426 -0
- logler/log_reader.py +267 -0
- logler/parser.py +207 -0
- logler/safe_regex.py +124 -0
- logler/terminal.py +252 -0
- logler/tracker.py +138 -0
- logler/tree_formatter.py +807 -0
- logler/watcher.py +55 -0
- logler/web/__init__.py +3 -0
- logler/web/app.py +810 -0
- logler/web/static/css/tailwind.css +1 -0
- logler/web/static/css/tailwind.input.css +3 -0
- logler/web/static/logler-logo.png +0 -0
- logler/web/tailwind.config.cjs +9 -0
- logler/web/templates/index.html +1454 -0
- logler-1.0.7.dist-info/METADATA +584 -0
- logler-1.0.7.dist-info/RECORD +28 -0
- logler-1.0.7.dist-info/WHEEL +4 -0
- logler-1.0.7.dist-info/entry_points.txt +2 -0
- logler-1.0.7.dist-info/licenses/LICENSE +21 -0
- logler_rs/__init__.py +5 -0
- logler_rs/logler_rs.cpython-311-darwin.so +0 -0
logler/web/app.py
ADDED
|
@@ -0,0 +1,810 @@
|
|
|
1
|
+
"""
|
|
2
|
+
FastAPI web application for Logler.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import glob
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import List, Optional, Dict, Any, Tuple
|
|
12
|
+
from fastapi import FastAPI, HTTPException, WebSocket, WebSocketDisconnect, Request
|
|
13
|
+
from fastapi.responses import HTMLResponse
|
|
14
|
+
from fastapi.staticfiles import StaticFiles
|
|
15
|
+
from fastapi.templating import Jinja2Templates
|
|
16
|
+
from pydantic import BaseModel
|
|
17
|
+
|
|
18
|
+
from ..parser import LogEntry, LogParser
|
|
19
|
+
from ..log_reader import LogReader
|
|
20
|
+
from ..tracker import ThreadTracker
|
|
21
|
+
from ..investigate import follow_thread_hierarchy, analyze_error_flow
|
|
22
|
+
|
|
23
|
+
# Get package directory
|
|
24
|
+
PACKAGE_DIR = Path(__file__).parent
|
|
25
|
+
TEMPLATES_DIR = PACKAGE_DIR / "templates"
|
|
26
|
+
STATIC_DIR = PACKAGE_DIR / "static"
|
|
27
|
+
|
|
28
|
+
# Create FastAPI app
|
|
29
|
+
LOG_ROOT = Path(os.environ.get("LOGLER_ROOT", ".")).expanduser().resolve()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _ensure_within_root(path: Path) -> Path:
|
|
33
|
+
resolved = path.expanduser().resolve()
|
|
34
|
+
if resolved == LOG_ROOT or LOG_ROOT in resolved.parents:
|
|
35
|
+
return resolved
|
|
36
|
+
raise HTTPException(status_code=403, detail="Requested path is outside the configured log root")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _sanitize_glob_pattern(pattern: str) -> str:
|
|
40
|
+
"""Remove path traversal sequences from glob patterns."""
|
|
41
|
+
import re as _re
|
|
42
|
+
|
|
43
|
+
# Remove any ../ or ..\ sequences that could escape the root
|
|
44
|
+
# Use a loop to handle multiple consecutive traversal attempts like ../../
|
|
45
|
+
while ".." in pattern:
|
|
46
|
+
old_pattern = pattern
|
|
47
|
+
pattern = _re.sub(r"\.\.[\\/]", "", pattern)
|
|
48
|
+
pattern = _re.sub(r"[\\/]\.\.", "", pattern)
|
|
49
|
+
pattern = _re.sub(r"^\.\.", "", pattern) # Leading ..
|
|
50
|
+
if pattern == old_pattern:
|
|
51
|
+
break # No more changes possible
|
|
52
|
+
return pattern
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _glob_within_root(pattern: str) -> List[Path]:
|
|
56
|
+
"""
|
|
57
|
+
Run a glob pattern scoped to LOG_ROOT, returning file paths only.
|
|
58
|
+
"""
|
|
59
|
+
if not pattern:
|
|
60
|
+
return []
|
|
61
|
+
|
|
62
|
+
# Sanitize the pattern to prevent path traversal
|
|
63
|
+
pattern = _sanitize_glob_pattern(pattern)
|
|
64
|
+
|
|
65
|
+
# Normalize relative patterns to LOG_ROOT
|
|
66
|
+
raw_pattern = pattern
|
|
67
|
+
if not Path(pattern).is_absolute():
|
|
68
|
+
raw_pattern = str(LOG_ROOT / pattern)
|
|
69
|
+
|
|
70
|
+
matches = glob.glob(raw_pattern, recursive=True)
|
|
71
|
+
results: List[Path] = []
|
|
72
|
+
seen = set()
|
|
73
|
+
for match in matches:
|
|
74
|
+
p = Path(match)
|
|
75
|
+
try:
|
|
76
|
+
p = _ensure_within_root(p)
|
|
77
|
+
except HTTPException:
|
|
78
|
+
continue
|
|
79
|
+
if not p.is_file():
|
|
80
|
+
continue
|
|
81
|
+
key = str(p)
|
|
82
|
+
if key in seen:
|
|
83
|
+
continue
|
|
84
|
+
seen.add(key)
|
|
85
|
+
results.append(p)
|
|
86
|
+
return sorted(results)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
app = FastAPI(
|
|
90
|
+
title="Logler",
|
|
91
|
+
description="Beautiful log viewer",
|
|
92
|
+
summary="Legacy web UI (Python FastAPI) with log root restrictions",
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
# Mount static files if directory exists
|
|
96
|
+
if STATIC_DIR.exists():
|
|
97
|
+
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
|
|
98
|
+
|
|
99
|
+
# Setup templates
|
|
100
|
+
templates = Jinja2Templates(directory=str(TEMPLATES_DIR))
|
|
101
|
+
|
|
102
|
+
# Global state
|
|
103
|
+
parser = LogParser()
|
|
104
|
+
tracker = ThreadTracker()
|
|
105
|
+
active_files: List[str] = []
|
|
106
|
+
websocket_clients: List[WebSocket] = []
|
|
107
|
+
MAX_RETURNED_ENTRIES = 10000
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _normalize_level(level: str) -> str:
|
|
111
|
+
return level.upper()
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _entry_matches(entry: Dict[str, Any], filters: Dict[str, Any]) -> bool:
|
|
115
|
+
if not filters:
|
|
116
|
+
return True
|
|
117
|
+
|
|
118
|
+
levels = filters.get("levels") or []
|
|
119
|
+
if levels:
|
|
120
|
+
normalized = {_normalize_level(lvl) for lvl in levels}
|
|
121
|
+
if entry.get("level") not in normalized:
|
|
122
|
+
return False
|
|
123
|
+
|
|
124
|
+
threads = set(filters.get("threads") or [])
|
|
125
|
+
if threads and entry.get("thread_id") not in threads:
|
|
126
|
+
return False
|
|
127
|
+
|
|
128
|
+
corr = (filters.get("correlation") or "").lower()
|
|
129
|
+
if corr and corr not in (entry.get("correlation_id") or "").lower():
|
|
130
|
+
return False
|
|
131
|
+
|
|
132
|
+
query = (filters.get("query") or "").lower()
|
|
133
|
+
if query and query not in (entry.get("message") or "").lower():
|
|
134
|
+
return False
|
|
135
|
+
|
|
136
|
+
return True
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _parse_timestamp(ts: Any) -> Optional[datetime]:
|
|
140
|
+
if isinstance(ts, datetime):
|
|
141
|
+
return ts
|
|
142
|
+
if isinstance(ts, str):
|
|
143
|
+
try:
|
|
144
|
+
return datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
|
145
|
+
except ValueError:
|
|
146
|
+
return None
|
|
147
|
+
return None
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _normalize_entry_dict(entry: Dict[str, Any]) -> Dict[str, Any]:
|
|
151
|
+
if not isinstance(entry, dict):
|
|
152
|
+
return entry
|
|
153
|
+
level = entry.get("level")
|
|
154
|
+
if isinstance(level, str):
|
|
155
|
+
entry["level"] = level.upper()
|
|
156
|
+
if entry.get("service_name") is None and entry.get("service") is not None:
|
|
157
|
+
entry["service_name"] = entry.get("service")
|
|
158
|
+
return entry
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _track_entries(entries: List[Dict[str, Any]]):
|
|
162
|
+
for entry in entries:
|
|
163
|
+
if not isinstance(entry, dict):
|
|
164
|
+
continue
|
|
165
|
+
tracker.track(
|
|
166
|
+
LogEntry(
|
|
167
|
+
line_number=entry.get("line_number") or 0,
|
|
168
|
+
raw=entry.get("raw") or entry.get("message") or "",
|
|
169
|
+
timestamp=_parse_timestamp(entry.get("timestamp")),
|
|
170
|
+
level=str(entry.get("level") or "UNKNOWN"),
|
|
171
|
+
message=entry.get("message") or entry.get("raw") or "",
|
|
172
|
+
thread_id=entry.get("thread_id"),
|
|
173
|
+
correlation_id=entry.get("correlation_id"),
|
|
174
|
+
trace_id=entry.get("trace_id"),
|
|
175
|
+
span_id=entry.get("span_id"),
|
|
176
|
+
service_name=entry.get("service_name") or entry.get("service"),
|
|
177
|
+
fields=entry.get("fields") or {},
|
|
178
|
+
)
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _rust_filter(
|
|
183
|
+
files: List[str],
|
|
184
|
+
filters: Dict[str, Any],
|
|
185
|
+
limit: int,
|
|
186
|
+
track: bool,
|
|
187
|
+
) -> Optional[List[Dict[str, Any]]]:
|
|
188
|
+
try:
|
|
189
|
+
import logler_rs # type: ignore
|
|
190
|
+
except ImportError:
|
|
191
|
+
return None
|
|
192
|
+
|
|
193
|
+
try:
|
|
194
|
+
from ..cache import get_cached_investigator
|
|
195
|
+
|
|
196
|
+
inv = get_cached_investigator(files)
|
|
197
|
+
except (ImportError, AttributeError, TypeError):
|
|
198
|
+
inv = logler_rs.PyInvestigator()
|
|
199
|
+
inv.load_files(files)
|
|
200
|
+
|
|
201
|
+
base_filters: Dict[str, Any] = {}
|
|
202
|
+
if filters.get("levels"):
|
|
203
|
+
base_filters["levels"] = [_normalize_level(lvl) for lvl in filters["levels"]]
|
|
204
|
+
thread_list = list(filters.get("threads") or [])
|
|
205
|
+
if filters.get("correlation"):
|
|
206
|
+
base_filters["correlation_id"] = filters["correlation"]
|
|
207
|
+
|
|
208
|
+
filter_sets: List[Dict[str, Any]] = []
|
|
209
|
+
if thread_list:
|
|
210
|
+
for tid in thread_list:
|
|
211
|
+
f = dict(base_filters)
|
|
212
|
+
f["thread_id"] = tid
|
|
213
|
+
filter_sets.append(f)
|
|
214
|
+
else:
|
|
215
|
+
filter_sets.append(base_filters)
|
|
216
|
+
|
|
217
|
+
entries: List[Dict[str, Any]] = []
|
|
218
|
+
|
|
219
|
+
try:
|
|
220
|
+
for rust_filters in filter_sets:
|
|
221
|
+
query_dict = {
|
|
222
|
+
"files": files,
|
|
223
|
+
"query": filters.get("query"),
|
|
224
|
+
"filters": rust_filters,
|
|
225
|
+
"limit": limit,
|
|
226
|
+
"context_lines": 0,
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
result_json = inv.search(json.dumps(query_dict))
|
|
230
|
+
result = json.loads(result_json)
|
|
231
|
+
for item in result.get("results", []):
|
|
232
|
+
entry = item.get("entry", {}) if isinstance(item, dict) else {}
|
|
233
|
+
entry.setdefault("entry_id", f"{entry.get('file','')}:{entry.get('line_number',0)}")
|
|
234
|
+
entry = _normalize_entry_dict(entry)
|
|
235
|
+
entries.append(entry)
|
|
236
|
+
|
|
237
|
+
entries.sort(
|
|
238
|
+
key=lambda e: (
|
|
239
|
+
e.get("timestamp") or "",
|
|
240
|
+
e.get("line_number") or 0,
|
|
241
|
+
)
|
|
242
|
+
)
|
|
243
|
+
if len(entries) > limit:
|
|
244
|
+
entries = entries[-limit:]
|
|
245
|
+
if track:
|
|
246
|
+
_track_entries(entries)
|
|
247
|
+
return entries
|
|
248
|
+
except (json.JSONDecodeError, KeyError, ValueError, AttributeError, RuntimeError):
|
|
249
|
+
# Rust filter failed - fall back to Python implementation
|
|
250
|
+
return None
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _python_filter(
|
|
254
|
+
files: List[str],
|
|
255
|
+
filters: Dict[str, Any],
|
|
256
|
+
limit: int,
|
|
257
|
+
track: bool,
|
|
258
|
+
) -> List[Dict[str, Any]]:
|
|
259
|
+
matched: List[Dict[str, Any]] = []
|
|
260
|
+
for raw_path in files:
|
|
261
|
+
path = _ensure_within_root(Path(raw_path))
|
|
262
|
+
if not path.exists():
|
|
263
|
+
continue
|
|
264
|
+
with open(path, "r") as f:
|
|
265
|
+
for line_number, line in enumerate(f, start=1):
|
|
266
|
+
entry = parser.parse_line(line_number, line.rstrip())
|
|
267
|
+
if track:
|
|
268
|
+
tracker.track(entry)
|
|
269
|
+
entry_dict = {
|
|
270
|
+
"entry_id": f"{path}:{line_number}",
|
|
271
|
+
"file": str(path),
|
|
272
|
+
"line_number": line_number,
|
|
273
|
+
"timestamp": entry.timestamp.isoformat() if entry.timestamp else None,
|
|
274
|
+
"level": entry.level,
|
|
275
|
+
"message": entry.message,
|
|
276
|
+
"thread_id": entry.thread_id,
|
|
277
|
+
"correlation_id": entry.correlation_id,
|
|
278
|
+
"service_name": entry.service_name,
|
|
279
|
+
"trace_id": entry.trace_id,
|
|
280
|
+
"span_id": entry.span_id,
|
|
281
|
+
}
|
|
282
|
+
if _entry_matches(entry_dict, filters):
|
|
283
|
+
matched.append(entry_dict)
|
|
284
|
+
return matched[-limit:]
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def filter_entries(
|
|
288
|
+
files: List[str],
|
|
289
|
+
filters: Optional[Dict[str, Any]],
|
|
290
|
+
limit: int = MAX_RETURNED_ENTRIES,
|
|
291
|
+
track_threads: bool = False,
|
|
292
|
+
) -> List[Dict[str, Any]]:
|
|
293
|
+
clean_filters = dict(filters or {})
|
|
294
|
+
clean_filters["threads"] = set(clean_filters.get("threads") or [])
|
|
295
|
+
entries = _rust_filter(files, clean_filters, limit, track_threads)
|
|
296
|
+
if entries is None:
|
|
297
|
+
entries = _python_filter(files, clean_filters, limit, track_threads)
|
|
298
|
+
return [_normalize_entry_dict(e) for e in entries if isinstance(e, dict)]
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _tail_entries(path: Path, limit: int) -> Tuple[List[Dict[str, Any]], int]:
|
|
302
|
+
"""
|
|
303
|
+
Fast tail path: avoids indexing the entire file and only parses the last N lines.
|
|
304
|
+
Returns parsed entries and the total line count.
|
|
305
|
+
"""
|
|
306
|
+
total_lines = sum(1 for _ in path.open("r", encoding="utf-8", errors="replace"))
|
|
307
|
+
reader = LogReader(str(path))
|
|
308
|
+
raw_lines = list(reader.tail(num_lines=limit, follow=False))
|
|
309
|
+
start_line = max(1, total_lines - len(raw_lines) + 1)
|
|
310
|
+
|
|
311
|
+
entries: List[Dict[str, Any]] = []
|
|
312
|
+
for idx, raw in enumerate(raw_lines):
|
|
313
|
+
line_no = start_line + idx
|
|
314
|
+
entry = parser.parse_line(line_no, raw.rstrip())
|
|
315
|
+
entries.append(
|
|
316
|
+
{
|
|
317
|
+
"entry_id": f"{path}:{line_no}",
|
|
318
|
+
"file": str(path),
|
|
319
|
+
"line_number": line_no,
|
|
320
|
+
"timestamp": entry.timestamp.isoformat() if entry.timestamp else None,
|
|
321
|
+
"level": entry.level,
|
|
322
|
+
"message": entry.message,
|
|
323
|
+
"thread_id": entry.thread_id,
|
|
324
|
+
"correlation_id": entry.correlation_id,
|
|
325
|
+
"service_name": entry.service_name,
|
|
326
|
+
"trace_id": entry.trace_id,
|
|
327
|
+
"span_id": entry.span_id,
|
|
328
|
+
}
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
return entries, total_lines
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def sample_entries(
|
|
335
|
+
entries: List[Dict[str, Any]], per_level: Optional[int], per_thread: Optional[int]
|
|
336
|
+
) -> List[Dict[str, Any]]:
|
|
337
|
+
if not entries:
|
|
338
|
+
return entries
|
|
339
|
+
|
|
340
|
+
sampled = []
|
|
341
|
+
|
|
342
|
+
if per_level:
|
|
343
|
+
by_level: Dict[str, List[Dict[str, Any]]] = {}
|
|
344
|
+
for e in entries:
|
|
345
|
+
lvl = e.get("level", "INFO")
|
|
346
|
+
by_level.setdefault(lvl, []).append(e)
|
|
347
|
+
for level_entries in by_level.values():
|
|
348
|
+
sampled.extend(level_entries[-per_level:])
|
|
349
|
+
|
|
350
|
+
if per_thread:
|
|
351
|
+
by_thread: Dict[str, List[Dict[str, Any]]] = {}
|
|
352
|
+
for e in entries:
|
|
353
|
+
tid = e.get("thread_id")
|
|
354
|
+
if not tid:
|
|
355
|
+
continue
|
|
356
|
+
by_thread.setdefault(tid, []).append(e)
|
|
357
|
+
for thread_entries in by_thread.values():
|
|
358
|
+
sampled.extend(thread_entries[-per_thread:])
|
|
359
|
+
|
|
360
|
+
# If neither sampling applied, return original
|
|
361
|
+
if not per_level and not per_thread:
|
|
362
|
+
return entries
|
|
363
|
+
|
|
364
|
+
# Deduplicate by entry_id
|
|
365
|
+
seen = set()
|
|
366
|
+
deduped = []
|
|
367
|
+
for e in sampled:
|
|
368
|
+
eid = e.get("entry_id")
|
|
369
|
+
if eid and eid in seen:
|
|
370
|
+
continue
|
|
371
|
+
if eid:
|
|
372
|
+
seen.add(eid)
|
|
373
|
+
deduped.append(e)
|
|
374
|
+
return deduped
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
class FileRequest(BaseModel):
|
|
378
|
+
path: str
|
|
379
|
+
filters: Optional[Dict[str, Any]] = None
|
|
380
|
+
limit: Optional[int] = None
|
|
381
|
+
quick: Optional[bool] = None
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
class FilesRequest(BaseModel):
|
|
385
|
+
paths: List[str]
|
|
386
|
+
filters: Optional[Dict[str, Any]] = None
|
|
387
|
+
limit: Optional[int] = None
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
class FilterRequest(BaseModel):
|
|
391
|
+
paths: List[str]
|
|
392
|
+
filters: Optional[Dict[str, Any]] = None
|
|
393
|
+
limit: Optional[int] = None
|
|
394
|
+
sample_per_level: Optional[int] = None
|
|
395
|
+
sample_per_thread: Optional[int] = None
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
@app.get("/", response_class=HTMLResponse)
|
|
399
|
+
async def index(request: Request):
|
|
400
|
+
"""Main page."""
|
|
401
|
+
return templates.TemplateResponse(
|
|
402
|
+
"index.html",
|
|
403
|
+
{
|
|
404
|
+
"request": request,
|
|
405
|
+
"active_files": active_files,
|
|
406
|
+
},
|
|
407
|
+
)
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
@app.get("/api/files/browse")
|
|
411
|
+
async def browse_files(directory: str = "."):
|
|
412
|
+
"""Browse files in a directory."""
|
|
413
|
+
dir_path = _ensure_within_root(Path(directory))
|
|
414
|
+
|
|
415
|
+
if not dir_path.exists() or not dir_path.is_dir():
|
|
416
|
+
return {"error": "Invalid directory", "files": []}
|
|
417
|
+
|
|
418
|
+
files = []
|
|
419
|
+
directories = []
|
|
420
|
+
try:
|
|
421
|
+
for item in sorted(dir_path.iterdir()):
|
|
422
|
+
if item.is_dir() and _ensure_within_root(item):
|
|
423
|
+
directories.append(
|
|
424
|
+
{
|
|
425
|
+
"name": item.name,
|
|
426
|
+
"path": str(item.absolute()),
|
|
427
|
+
}
|
|
428
|
+
)
|
|
429
|
+
if item.is_file() and (item.suffix in [".log", ".txt"] or "log" in item.name.lower()):
|
|
430
|
+
files.append(
|
|
431
|
+
{
|
|
432
|
+
"name": item.name,
|
|
433
|
+
"path": str(item.absolute()),
|
|
434
|
+
"size": item.stat().st_size,
|
|
435
|
+
}
|
|
436
|
+
)
|
|
437
|
+
except PermissionError:
|
|
438
|
+
return {"error": "Permission denied", "files": []}
|
|
439
|
+
|
|
440
|
+
parent_dir = dir_path.parent if dir_path.parent != dir_path else None
|
|
441
|
+
if parent_dir and not (parent_dir == LOG_ROOT or LOG_ROOT in parent_dir.parents):
|
|
442
|
+
parent_dir = None
|
|
443
|
+
|
|
444
|
+
return {
|
|
445
|
+
"current_dir": str(dir_path),
|
|
446
|
+
"parent_dir": str(parent_dir) if parent_dir else None,
|
|
447
|
+
"files": files,
|
|
448
|
+
"directories": directories,
|
|
449
|
+
"log_root": str(LOG_ROOT),
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
@app.get("/api/files/glob")
|
|
454
|
+
async def glob_files(pattern: str = "**/*.log", base_dir: str = ".", limit: int = 200):
|
|
455
|
+
"""Search for files by glob pattern within LOG_ROOT. When base_dir is provided, pattern is resolved relative to it."""
|
|
456
|
+
try:
|
|
457
|
+
base = _ensure_within_root(Path(base_dir))
|
|
458
|
+
except HTTPException:
|
|
459
|
+
base = LOG_ROOT
|
|
460
|
+
|
|
461
|
+
raw_pattern = pattern
|
|
462
|
+
if not Path(pattern).is_absolute():
|
|
463
|
+
raw_pattern = str((base / pattern))
|
|
464
|
+
|
|
465
|
+
matches = _glob_within_root(raw_pattern)
|
|
466
|
+
files = []
|
|
467
|
+
for p in matches[:limit]:
|
|
468
|
+
try:
|
|
469
|
+
stat = p.stat()
|
|
470
|
+
files.append(
|
|
471
|
+
{
|
|
472
|
+
"name": p.name,
|
|
473
|
+
"path": str(p),
|
|
474
|
+
"size": stat.st_size,
|
|
475
|
+
"modified": stat.st_mtime,
|
|
476
|
+
}
|
|
477
|
+
)
|
|
478
|
+
except OSError:
|
|
479
|
+
continue
|
|
480
|
+
return {
|
|
481
|
+
"pattern": pattern,
|
|
482
|
+
"count": len(matches),
|
|
483
|
+
"files": files,
|
|
484
|
+
"truncated": len(matches) > limit,
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
@app.post("/api/files/open")
|
|
489
|
+
async def open_file(request: FileRequest):
|
|
490
|
+
"""Open a log file."""
|
|
491
|
+
global tracker
|
|
492
|
+
file_path = _ensure_within_root(Path(request.path))
|
|
493
|
+
|
|
494
|
+
if not file_path.exists():
|
|
495
|
+
return {"error": "File not found"}
|
|
496
|
+
|
|
497
|
+
quick_mode = request.quick is not False # default to quick unless explicitly disabled
|
|
498
|
+
if quick_mode:
|
|
499
|
+
tracker = ThreadTracker()
|
|
500
|
+
quick_limit = min(request.limit or 1000, MAX_RETURNED_ENTRIES)
|
|
501
|
+
entries, total_lines = _tail_entries(file_path, quick_limit)
|
|
502
|
+
tracker = ThreadTracker()
|
|
503
|
+
_track_entries(entries)
|
|
504
|
+
return {
|
|
505
|
+
"file_path": str(file_path),
|
|
506
|
+
"entries": entries,
|
|
507
|
+
"total": total_lines,
|
|
508
|
+
"partial": True,
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
# Reset tracker to avoid double-counting between file loads
|
|
512
|
+
tracker = ThreadTracker()
|
|
513
|
+
|
|
514
|
+
if str(file_path) not in active_files:
|
|
515
|
+
active_files.append(str(file_path))
|
|
516
|
+
|
|
517
|
+
entries = filter_entries(
|
|
518
|
+
[str(file_path)],
|
|
519
|
+
request.filters,
|
|
520
|
+
limit=request.limit or MAX_RETURNED_ENTRIES,
|
|
521
|
+
track_threads=True,
|
|
522
|
+
)
|
|
523
|
+
total_count = len(entries)
|
|
524
|
+
entries = entries[-1000:]
|
|
525
|
+
|
|
526
|
+
return {
|
|
527
|
+
"file_path": str(file_path),
|
|
528
|
+
"entries": entries,
|
|
529
|
+
"total": total_count,
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
@app.post("/api/files/open_many")
|
|
534
|
+
async def open_many(request: FilesRequest):
|
|
535
|
+
"""Open multiple log files and interleave entries."""
|
|
536
|
+
global tracker
|
|
537
|
+
tracker = ThreadTracker()
|
|
538
|
+
valid_files = []
|
|
539
|
+
for raw_path in request.paths:
|
|
540
|
+
try:
|
|
541
|
+
file_path = _ensure_within_root(Path(raw_path))
|
|
542
|
+
except HTTPException:
|
|
543
|
+
continue
|
|
544
|
+
if file_path.exists():
|
|
545
|
+
valid_files.append(str(file_path))
|
|
546
|
+
|
|
547
|
+
entries = filter_entries(
|
|
548
|
+
valid_files,
|
|
549
|
+
request.filters,
|
|
550
|
+
limit=request.limit or MAX_RETURNED_ENTRIES,
|
|
551
|
+
track_threads=True,
|
|
552
|
+
)
|
|
553
|
+
|
|
554
|
+
# Sort by timestamp if available
|
|
555
|
+
entries.sort(key=lambda e: e["timestamp"] or "")
|
|
556
|
+
|
|
557
|
+
file_counts: Dict[str, int] = {}
|
|
558
|
+
file_meta: Dict[str, Dict[str, Any]] = {}
|
|
559
|
+
for entry in entries:
|
|
560
|
+
file = entry.get("file")
|
|
561
|
+
if file:
|
|
562
|
+
file_counts[file] = file_counts.get(file, 0) + 1
|
|
563
|
+
ts = entry.get("timestamp")
|
|
564
|
+
meta = file_meta.setdefault(file, {"first": None, "last": None})
|
|
565
|
+
if ts:
|
|
566
|
+
if meta["first"] is None or ts < meta["first"]:
|
|
567
|
+
meta["first"] = ts
|
|
568
|
+
if meta["last"] is None or ts > meta["last"]:
|
|
569
|
+
meta["last"] = ts
|
|
570
|
+
|
|
571
|
+
for lf in valid_files:
|
|
572
|
+
if lf not in active_files:
|
|
573
|
+
active_files.append(lf)
|
|
574
|
+
|
|
575
|
+
return {
|
|
576
|
+
"files": valid_files,
|
|
577
|
+
"entries": entries,
|
|
578
|
+
"total": len(entries),
|
|
579
|
+
"file_counts": file_counts,
|
|
580
|
+
"file_meta": [
|
|
581
|
+
{
|
|
582
|
+
"file": f,
|
|
583
|
+
"count": file_counts.get(f, 0),
|
|
584
|
+
"first": meta.get("first"),
|
|
585
|
+
"last": meta.get("last"),
|
|
586
|
+
}
|
|
587
|
+
for f, meta in file_meta.items()
|
|
588
|
+
],
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
@app.get("/api/threads")
|
|
593
|
+
async def get_threads():
|
|
594
|
+
"""Get all tracked threads."""
|
|
595
|
+
threads = tracker.get_all_threads()
|
|
596
|
+
# Convert datetime to ISO format
|
|
597
|
+
for thread in threads:
|
|
598
|
+
if thread.get("first_seen"):
|
|
599
|
+
thread["first_seen"] = thread["first_seen"].isoformat()
|
|
600
|
+
if thread.get("last_seen"):
|
|
601
|
+
thread["last_seen"] = thread["last_seen"].isoformat()
|
|
602
|
+
return threads
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
@app.post("/api/files/filter")
|
|
606
|
+
async def filter_files(request: FilterRequest):
|
|
607
|
+
"""Filter entries on the server to reduce payload size."""
|
|
608
|
+
files = []
|
|
609
|
+
for raw in request.paths:
|
|
610
|
+
try:
|
|
611
|
+
files.append(str(_ensure_within_root(Path(raw))))
|
|
612
|
+
except HTTPException:
|
|
613
|
+
continue
|
|
614
|
+
entries = filter_entries(files, request.filters, limit=request.limit or MAX_RETURNED_ENTRIES)
|
|
615
|
+
entries = sample_entries(entries, request.sample_per_level, request.sample_per_thread)
|
|
616
|
+
return {"entries": entries, "total": len(entries)}
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
@app.post("/api/files/sample")
|
|
620
|
+
async def sample_files(request: FilterRequest):
|
|
621
|
+
"""Filter entries and return samples by level/thread to lighten payloads."""
|
|
622
|
+
files = []
|
|
623
|
+
for raw in request.paths:
|
|
624
|
+
try:
|
|
625
|
+
files.append(str(_ensure_within_root(Path(raw))))
|
|
626
|
+
except HTTPException:
|
|
627
|
+
continue
|
|
628
|
+
entries = filter_entries(files, request.filters, limit=request.limit or MAX_RETURNED_ENTRIES)
|
|
629
|
+
sampled = sample_entries(entries, request.sample_per_level, request.sample_per_thread)
|
|
630
|
+
return {"entries": sampled, "total": len(entries), "sampled": len(sampled)}
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
@app.get("/api/traces")
|
|
634
|
+
async def get_traces():
|
|
635
|
+
"""Get all tracked traces."""
|
|
636
|
+
traces = tracker.get_all_traces()
|
|
637
|
+
for trace in traces:
|
|
638
|
+
if trace.get("start_time"):
|
|
639
|
+
trace["start_time"] = trace["start_time"].isoformat()
|
|
640
|
+
if trace.get("end_time"):
|
|
641
|
+
trace["end_time"] = trace["end_time"].isoformat()
|
|
642
|
+
for span in trace.get("spans", []):
|
|
643
|
+
if span.get("timestamp"):
|
|
644
|
+
span["timestamp"] = span["timestamp"].isoformat()
|
|
645
|
+
return traces
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
@app.websocket("/ws")
|
|
649
|
+
async def websocket_endpoint(websocket: WebSocket):
|
|
650
|
+
"""WebSocket endpoint for real-time updates."""
|
|
651
|
+
await websocket.accept()
|
|
652
|
+
websocket_clients.append(websocket)
|
|
653
|
+
current_filters: Dict[str, Any] = {}
|
|
654
|
+
drop_count = 0
|
|
655
|
+
|
|
656
|
+
try:
|
|
657
|
+
while True:
|
|
658
|
+
# Receive messages (for file selection, etc.)
|
|
659
|
+
data = await websocket.receive_text()
|
|
660
|
+
message = json.loads(data)
|
|
661
|
+
|
|
662
|
+
if message.get("action") == "follow":
|
|
663
|
+
file_path = message.get("file_path")
|
|
664
|
+
current_filters = message.get("filters") or {}
|
|
665
|
+
drop_count = 0
|
|
666
|
+
await follow_file(websocket, file_path, current_filters, drop_count)
|
|
667
|
+
|
|
668
|
+
except WebSocketDisconnect:
|
|
669
|
+
pass # Normal disconnect
|
|
670
|
+
finally:
|
|
671
|
+
if websocket in websocket_clients:
|
|
672
|
+
websocket_clients.remove(websocket)
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
async def follow_file(
|
|
676
|
+
websocket: WebSocket, file_path: str, filters: Dict[str, Any], drop_count: int
|
|
677
|
+
):
|
|
678
|
+
"""Follow a log file and send updates via WebSocket."""
|
|
679
|
+
try:
|
|
680
|
+
path = _ensure_within_root(Path(file_path))
|
|
681
|
+
except HTTPException as exc:
|
|
682
|
+
await websocket.send_json({"error": exc.detail})
|
|
683
|
+
return
|
|
684
|
+
|
|
685
|
+
if not path.exists():
|
|
686
|
+
await websocket.send_json({"error": "File not found"})
|
|
687
|
+
return
|
|
688
|
+
|
|
689
|
+
# Get initial position (end of file)
|
|
690
|
+
with open(path, "r") as f:
|
|
691
|
+
f.seek(0, 2)
|
|
692
|
+
position = f.tell()
|
|
693
|
+
with open(path, "r") as f:
|
|
694
|
+
line_number = sum(1 for _ in f)
|
|
695
|
+
|
|
696
|
+
# Follow file
|
|
697
|
+
try:
|
|
698
|
+
while True:
|
|
699
|
+
with open(path, "r") as f:
|
|
700
|
+
current_size = path.stat().st_size
|
|
701
|
+
if current_size < position:
|
|
702
|
+
# File was truncated/rotated; restart from beginning
|
|
703
|
+
position = 0
|
|
704
|
+
line_number = 0
|
|
705
|
+
f.seek(position)
|
|
706
|
+
new_lines = f.readlines()
|
|
707
|
+
position = f.tell()
|
|
708
|
+
|
|
709
|
+
for line in new_lines:
|
|
710
|
+
line_number += 1
|
|
711
|
+
entry = parser.parse_line(line_number, line.rstrip())
|
|
712
|
+
tracker.track(entry)
|
|
713
|
+
|
|
714
|
+
entry_dict = {
|
|
715
|
+
"entry_id": f"{path}:{line_number}",
|
|
716
|
+
"file": str(path),
|
|
717
|
+
"line_number": entry.line_number,
|
|
718
|
+
"timestamp": entry.timestamp.isoformat() if entry.timestamp else None,
|
|
719
|
+
"level": entry.level,
|
|
720
|
+
"message": entry.message,
|
|
721
|
+
"thread_id": entry.thread_id,
|
|
722
|
+
"correlation_id": entry.correlation_id,
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
if not _entry_matches(entry_dict, filters):
|
|
726
|
+
continue
|
|
727
|
+
|
|
728
|
+
try:
|
|
729
|
+
await asyncio.wait_for(
|
|
730
|
+
websocket.send_json({"type": "log_entry", "entry": entry_dict}),
|
|
731
|
+
timeout=0.25,
|
|
732
|
+
)
|
|
733
|
+
except asyncio.TimeoutError:
|
|
734
|
+
drop_count += 1
|
|
735
|
+
# Occasionally inform client of drops to avoid flooding
|
|
736
|
+
if drop_count % 50 == 0:
|
|
737
|
+
try:
|
|
738
|
+
await websocket.send_json({"type": "dropped", "count": drop_count})
|
|
739
|
+
except Exception:
|
|
740
|
+
pass
|
|
741
|
+
except Exception:
|
|
742
|
+
return
|
|
743
|
+
|
|
744
|
+
await asyncio.sleep(0.1)
|
|
745
|
+
|
|
746
|
+
except Exception as e:
|
|
747
|
+
await websocket.send_json({"error": str(e)})
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
class HierarchyRequest(BaseModel):
|
|
751
|
+
paths: List[str]
|
|
752
|
+
root_identifier: str
|
|
753
|
+
max_depth: Optional[int] = None
|
|
754
|
+
min_confidence: float = 0.0
|
|
755
|
+
use_naming_patterns: bool = True
|
|
756
|
+
use_temporal_inference: bool = True
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
@app.post("/api/hierarchy")
|
|
760
|
+
async def get_hierarchy(request: HierarchyRequest):
|
|
761
|
+
"""
|
|
762
|
+
Build and return thread/span hierarchy for visualization.
|
|
763
|
+
|
|
764
|
+
Returns a hierarchical tree structure with:
|
|
765
|
+
- Parent-child relationships
|
|
766
|
+
- Duration and timing information
|
|
767
|
+
- Error counts and propagation
|
|
768
|
+
- Bottleneck detection
|
|
769
|
+
"""
|
|
770
|
+
files = []
|
|
771
|
+
for raw in request.paths:
|
|
772
|
+
try:
|
|
773
|
+
files.append(str(_ensure_within_root(Path(raw))))
|
|
774
|
+
except HTTPException:
|
|
775
|
+
continue
|
|
776
|
+
|
|
777
|
+
if not files:
|
|
778
|
+
return {"error": "No valid files provided", "hierarchy": None}
|
|
779
|
+
|
|
780
|
+
try:
|
|
781
|
+
hierarchy = follow_thread_hierarchy(
|
|
782
|
+
files=files,
|
|
783
|
+
root_identifier=request.root_identifier,
|
|
784
|
+
max_depth=request.max_depth,
|
|
785
|
+
use_naming_patterns=request.use_naming_patterns,
|
|
786
|
+
use_temporal_inference=request.use_temporal_inference,
|
|
787
|
+
min_confidence=request.min_confidence,
|
|
788
|
+
)
|
|
789
|
+
|
|
790
|
+
# Also analyze error flow
|
|
791
|
+
error_analysis = analyze_error_flow(hierarchy)
|
|
792
|
+
|
|
793
|
+
return {
|
|
794
|
+
"hierarchy": hierarchy,
|
|
795
|
+
"error_analysis": error_analysis,
|
|
796
|
+
}
|
|
797
|
+
except Exception as e:
|
|
798
|
+
return {"error": str(e), "hierarchy": None}
|
|
799
|
+
|
|
800
|
+
|
|
801
|
+
async def run_server(host: str, port: int, initial_files: List[str]):
|
|
802
|
+
"""Run the FastAPI server."""
|
|
803
|
+
import uvicorn
|
|
804
|
+
|
|
805
|
+
global active_files
|
|
806
|
+
active_files = initial_files
|
|
807
|
+
|
|
808
|
+
config = uvicorn.Config(app, host=host, port=port, log_level="info")
|
|
809
|
+
server = uvicorn.Server(config)
|
|
810
|
+
await server.serve()
|