logtriage 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- logtriage-0.1.0/PKG-INFO +51 -0
- logtriage-0.1.0/README.md +37 -0
- logtriage-0.1.0/pyproject.toml +27 -0
- logtriage-0.1.0/setup.cfg +4 -0
- logtriage-0.1.0/src/loglens/__init__.py +3 -0
- logtriage-0.1.0/src/loglens/__main__.py +5 -0
- logtriage-0.1.0/src/loglens/cli.py +127 -0
- logtriage-0.1.0/src/loglens/core.py +156 -0
- logtriage-0.1.0/src/logtriage.egg-info/PKG-INFO +51 -0
- logtriage-0.1.0/src/logtriage.egg-info/SOURCES.txt +12 -0
- logtriage-0.1.0/src/logtriage.egg-info/dependency_links.txt +1 -0
- logtriage-0.1.0/src/logtriage.egg-info/entry_points.txt +2 -0
- logtriage-0.1.0/src/logtriage.egg-info/top_level.txt +1 -0
- logtriage-0.1.0/tests/test_loglens.py +100 -0
logtriage-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: logtriage
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Cluster big logs into templates with counts, severities and incident spikes — streaming, stdlib only
|
|
5
|
+
Author: juzorei
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/juzorei/loglens
|
|
8
|
+
Keywords: logs,observability,triage,cli,agent-skills
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Topic :: System :: Logging
|
|
12
|
+
Requires-Python: >=3.11
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
|
|
15
|
+
# loglens
|
|
16
|
+
|
|
17
|
+
Cluster big logs into templates with counts, severities and incident spikes — streaming, stdlib only, no file size limit.
|
|
18
|
+
|
|
19
|
+
```text
|
|
20
|
+
$ loglens scan app.log
|
|
21
|
+
lines 3,000 (222,367 bytes)
|
|
22
|
+
time 2026-10-30 09:00:00 … 2026-10-30 10:39:58
|
|
23
|
+
levels INFO 1,999, DEBUG 579, WARNING 273, ERROR 149
|
|
24
|
+
|
|
25
|
+
1,999× L1-L2999 [INFO]
|
|
26
|
+
2026-10-30 09:00:00 INFO request handled path=/api/lots/26 status=200 dur=37ms
|
|
27
|
+
|
|
28
|
+
149× L902-L1050 [ERROR]
|
|
29
|
+
2026-10-30 09:30:02 ERROR api.db timeout after 30s query=select * from lots where id=491
|
|
30
|
+
|
|
31
|
+
$ loglens spikes app.log
|
|
32
|
+
problems across 30 buckets (total 422 in 3,000 lines)
|
|
33
|
+
2026-10-30 09:33:26 149 ████████████████████████████████████████
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
A 500 MB log cannot be read — by a human or by an agent with finite context. But logs are repetitive: numbers, UUIDs, IPs, hashes and quoted strings normalize away, and 3000 nearly-identical lines collapse into a handful of templates with counts, line ranges and one real sample each. That's all the information a triage needs.
|
|
37
|
+
|
|
38
|
+
## Commands
|
|
39
|
+
|
|
40
|
+
```text
|
|
41
|
+
loglens scan <log> [--top 15] [--json] # template clustering + severity counts
|
|
42
|
+
loglens spikes <log> [--buckets 30] [--json] # when problems happen (time or line buckets)
|
|
43
|
+
loglens demo [dir] # synthetic log with a planted incident
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Streaming: the file is read line by line, memory holds only templates. Any text log; timestamps understood in ISO, Apache-bracketed and bare `HH:MM:SS` forms (no timestamps → line-range buckets). Python 3.11+, stdlib only.
|
|
47
|
+
|
|
48
|
+
Install: `pip install logtriage` — or from this repo:
|
|
49
|
+
`pip install "git+https://github.com/juzorei/loglens#subdirectory=tools/loglens"`
|
|
50
|
+
|
|
51
|
+
The companion agent skill lives in [skills/log-triage](../skills/log-triage): cluster first, read only samples, then bounded reads around the incident. Skills: `npx skills add juzorei/loglens`.
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# loglens
|
|
2
|
+
|
|
3
|
+
Cluster big logs into templates with counts, severities and incident spikes — streaming, stdlib only, no file size limit.
|
|
4
|
+
|
|
5
|
+
```text
|
|
6
|
+
$ loglens scan app.log
|
|
7
|
+
lines 3,000 (222,367 bytes)
|
|
8
|
+
time 2026-10-30 09:00:00 … 2026-10-30 10:39:58
|
|
9
|
+
levels INFO 1,999, DEBUG 579, WARNING 273, ERROR 149
|
|
10
|
+
|
|
11
|
+
1,999× L1-L2999 [INFO]
|
|
12
|
+
2026-10-30 09:00:00 INFO request handled path=/api/lots/26 status=200 dur=37ms
|
|
13
|
+
|
|
14
|
+
149× L902-L1050 [ERROR]
|
|
15
|
+
2026-10-30 09:30:02 ERROR api.db timeout after 30s query=select * from lots where id=491
|
|
16
|
+
|
|
17
|
+
$ loglens spikes app.log
|
|
18
|
+
problems across 30 buckets (total 422 in 3,000 lines)
|
|
19
|
+
2026-10-30 09:33:26 149 ████████████████████████████████████████
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
A 500 MB log cannot be read — by a human or by an agent with finite context. But logs are repetitive: numbers, UUIDs, IPs, hashes and quoted strings normalize away, and 3000 nearly-identical lines collapse into a handful of templates with counts, line ranges and one real sample each. That's all the information a triage needs.
|
|
23
|
+
|
|
24
|
+
## Commands
|
|
25
|
+
|
|
26
|
+
```text
|
|
27
|
+
loglens scan <log> [--top 15] [--json] # template clustering + severity counts
|
|
28
|
+
loglens spikes <log> [--buckets 30] [--json] # when problems happen (time or line buckets)
|
|
29
|
+
loglens demo [dir] # synthetic log with a planted incident
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Streaming: the file is read line by line, memory holds only templates. Any text log; timestamps understood in ISO, Apache-bracketed and bare `HH:MM:SS` forms (no timestamps → line-range buckets). Python 3.11+, stdlib only.
|
|
33
|
+
|
|
34
|
+
Install: `pip install logtriage` — or from this repo:
|
|
35
|
+
`pip install "git+https://github.com/juzorei/loglens#subdirectory=tools/loglens"`
|
|
36
|
+
|
|
37
|
+
The companion agent skill lives in [skills/log-triage](../skills/log-triage): cluster first, read only samples, then bounded reads around the incident. Skills: `npx skills add juzorei/loglens`.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=69"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "logtriage"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Cluster big logs into templates with counts, severities and incident spikes — streaming, stdlib only"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
authors = [{ name = "juzorei" }]
|
|
13
|
+
keywords = ["logs", "observability", "triage", "cli", "agent-skills"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Operating System :: OS Independent",
|
|
17
|
+
"Topic :: System :: Logging",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[project.urls]
|
|
21
|
+
Homepage = "https://github.com/juzorei/loglens"
|
|
22
|
+
|
|
23
|
+
[project.scripts]
|
|
24
|
+
loglens = "loglens.cli:main"
|
|
25
|
+
|
|
26
|
+
[tool.setuptools.packages.find]
|
|
27
|
+
where = ["src"]
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""loglens CLI: cluster a big log into templates, count severities, find spikes."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
from . import __version__
|
|
10
|
+
from .core import Scan, scan, spikes
|
|
11
|
+
|
|
12
|
+
BAR_WIDTH = 40
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def bar(n: int, max_n: int) -> str:
|
|
16
|
+
width = max(1, round(BAR_WIDTH * n / max_n)) if max_n else 1
|
|
17
|
+
return "█" * width
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def cmd_scan(args) -> int:
|
|
21
|
+
result = scan(args.log)
|
|
22
|
+
if args.json:
|
|
23
|
+
print(json.dumps({
|
|
24
|
+
"file": result.path, "lines": result.lines, "bytes": result.bytes_read,
|
|
25
|
+
"timeframe": result.timeframe, "severities": result.counts(),
|
|
26
|
+
"templates": [{"pattern": t.pattern, "count": t.count,
|
|
27
|
+
"lines": [t.first_line, t.last_line], "severity": t.severity,
|
|
28
|
+
"sample": t.sample}
|
|
29
|
+
for t in result.top(args.top)],
|
|
30
|
+
}, indent=2))
|
|
31
|
+
return 0
|
|
32
|
+
print(f"file {result.path}")
|
|
33
|
+
print(f"lines {result.lines:,} ({result.bytes_read:,} bytes)")
|
|
34
|
+
if result.timeframe:
|
|
35
|
+
print(f"time {result.timeframe}")
|
|
36
|
+
counts = result.counts()
|
|
37
|
+
if counts:
|
|
38
|
+
print("levels " + ", ".join(f"{k} {v:,}" for k, v in counts.items()))
|
|
39
|
+
max_n = max((t.count for t in result.top(args.top)), default=1)
|
|
40
|
+
for t in result.top(args.top):
|
|
41
|
+
print(f"\n {t.count:>7,}× L{t.first_line}-L{t.last_line}"
|
|
42
|
+
+ (f" [{t.severity}]" if t.severity else ""))
|
|
43
|
+
print(f" {t.sample[:160]}")
|
|
44
|
+
print(f" {bar(t.count, max_n)}")
|
|
45
|
+
print(f"\n{len(result.templates)} distinct templates; showing top {min(args.top, len(result.templates))}")
|
|
46
|
+
return 0
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def cmd_spikes(args) -> int:
|
|
50
|
+
data = spikes(args.log, buckets=args.buckets)
|
|
51
|
+
if args.json:
|
|
52
|
+
print(json.dumps(data, indent=2))
|
|
53
|
+
return 0
|
|
54
|
+
print(f"problems across {len(data['buckets'])} buckets "
|
|
55
|
+
f"(total {data.get('total_problems', 0)} in {data.get('total_lines', 0):,} lines)")
|
|
56
|
+
max_n = max((b["problems"] for b in data["buckets"]), default=1)
|
|
57
|
+
for b in data["buckets"]:
|
|
58
|
+
print(f" {b['label']:<30} {b['problems']:>7,} {bar(b['problems'], max_n)}")
|
|
59
|
+
if not data["buckets"]:
|
|
60
|
+
print(" (no errors, warnings or failures found)")
|
|
61
|
+
return 0
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def cmd_demo(args) -> int:
|
|
65
|
+
from pathlib import Path
|
|
66
|
+
d = Path(args.directory)
|
|
67
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
68
|
+
log = d / "app.log"
|
|
69
|
+
import datetime
|
|
70
|
+
import random
|
|
71
|
+
random.seed(42)
|
|
72
|
+
base = datetime.datetime(2026, 10, 30, 9, 0, 0)
|
|
73
|
+
lines = []
|
|
74
|
+
for i in range(3000):
|
|
75
|
+
ts = (base + datetime.timedelta(seconds=i * 2)).isoformat(sep=" ")
|
|
76
|
+
roll = random.random()
|
|
77
|
+
if 900 < i < 1050: # planted incident spike
|
|
78
|
+
lines.append(f"{ts} ERROR api.db timeout after 30s query=select * from lots where id={random.randint(1, 999)}")
|
|
79
|
+
elif roll < 0.7:
|
|
80
|
+
lines.append(f"{ts} INFO request handled path=/api/lots/{random.randint(1, 999)} status=200 dur={random.randint(2, 90)}ms")
|
|
81
|
+
elif roll < 0.9:
|
|
82
|
+
lines.append(f"{ts} DEBUG cache hit key=user:{random.randint(1, 99)} ttl={random.randint(10, 300)}")
|
|
83
|
+
else:
|
|
84
|
+
lines.append(f"{ts} WARN rate limit approach key=ip:{random.randint(1, 20)} count={random.randint(90, 100)}")
|
|
85
|
+
log.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
86
|
+
print(f"wrote synthetic log: {log} ({log.stat().st_size:,} bytes, 3000 lines, planted incident at lines ~900-1050)\n")
|
|
87
|
+
print("try:")
|
|
88
|
+
print(f" loglens scan {log}")
|
|
89
|
+
print(f" loglens spikes {log}")
|
|
90
|
+
print(f" loglens scan {log} --json")
|
|
91
|
+
return 0
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
95
|
+
parser = argparse.ArgumentParser(prog="loglens", description=__doc__)
|
|
96
|
+
parser.add_argument("--version", action="version", version=f"loglens {__version__}")
|
|
97
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
98
|
+
|
|
99
|
+
p = sub.add_parser("scan", help="cluster a log into templates with counts and samples")
|
|
100
|
+
p.add_argument("log")
|
|
101
|
+
p.add_argument("--top", type=int, default=15)
|
|
102
|
+
p.add_argument("--json", action="store_true")
|
|
103
|
+
p.set_defaults(func=cmd_scan)
|
|
104
|
+
|
|
105
|
+
p = sub.add_parser("spikes", help="show when problems happen (time buckets or line ranges)")
|
|
106
|
+
p.add_argument("log")
|
|
107
|
+
p.add_argument("--buckets", type=int, default=30)
|
|
108
|
+
p.add_argument("--json", action="store_true")
|
|
109
|
+
p.set_defaults(func=cmd_spikes)
|
|
110
|
+
|
|
111
|
+
p = sub.add_parser("demo", help="write a synthetic log with a planted incident")
|
|
112
|
+
p.add_argument("directory", nargs="?", default="loglens-demo")
|
|
113
|
+
p.set_defaults(func=cmd_demo)
|
|
114
|
+
return parser
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def main(argv: list[str] | None = None) -> int:
|
|
118
|
+
args = build_parser().parse_args(argv)
|
|
119
|
+
try:
|
|
120
|
+
return args.func(args)
|
|
121
|
+
except OSError as exc:
|
|
122
|
+
print(f"loglens: error: {exc}", file=sys.stderr)
|
|
123
|
+
return 2
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
if __name__ == "__main__":
|
|
127
|
+
sys.exit(main())
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""loglens core: streaming pattern clustering and spike detection for big logs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
# Normalization: variable parts of a line collapse so that 5000 occurrences of
|
|
10
|
+
# the same error become one template with a count.
|
|
11
|
+
NORMALIZERS = [
|
|
12
|
+
(re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"), "<uuid>"),
|
|
13
|
+
(re.compile(r"\b\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?\b"), "<ip>"),
|
|
14
|
+
(re.compile(r"\b0x[0-9a-fA-F]{6,}\b"), "<hex>"),
|
|
15
|
+
(re.compile(r"\b[0-9a-f]{16,}\b", re.I), "<hash>"),
|
|
16
|
+
# last: every remaining digit run collapses (uuid/ip/hex/hash already consumed)
|
|
17
|
+
(re.compile(r"\d+"), "<n>"),
|
|
18
|
+
(re.compile(r'"[^"]{0,200}"'), '"…"'),
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
TS_PATTERNS = [
|
|
22
|
+
re.compile(r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:[.,]\d+)?"),
|
|
23
|
+
re.compile(r"\[\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}"),
|
|
24
|
+
re.compile(r"\d{2}:\d{2}:\d{2}(?:\.\d+)?"),
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
SEVERITY = re.compile(
|
|
28
|
+
r"\b(TRACE|DEBUG|INFO|NOTICE|WARN(?:ING)?|ERROR|ERR|CRITICAL|CRIT|FATAL|PANIC)\b", re.I)
|
|
29
|
+
|
|
30
|
+
BUCKETS = 30
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class Template:
|
|
35
|
+
pattern: str
|
|
36
|
+
count: int = 0
|
|
37
|
+
first_line: int = 0
|
|
38
|
+
last_line: int = 0
|
|
39
|
+
severity: str = ""
|
|
40
|
+
sample: str = ""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass
|
|
44
|
+
class Scan:
|
|
45
|
+
path: str = ""
|
|
46
|
+
lines: int = 0
|
|
47
|
+
bytes_read: int = 0
|
|
48
|
+
timeframe: str = ""
|
|
49
|
+
severities: dict[str, int] = field(default_factory=dict)
|
|
50
|
+
templates: list[Template] = field(default_factory=list)
|
|
51
|
+
|
|
52
|
+
def top(self, n: int) -> list[Template]:
|
|
53
|
+
return sorted(self.templates, key=lambda t: -t.count)[:n]
|
|
54
|
+
|
|
55
|
+
def counts(self) -> dict[str, int]:
|
|
56
|
+
return {k: v for k, v in self.severities.items() if v}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def normalize(line: str) -> str:
|
|
60
|
+
for regex, replacement in NORMALIZERS:
|
|
61
|
+
line = regex.sub(replacement, line)
|
|
62
|
+
return line.strip()[:220]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def severity_of(line: str) -> str:
|
|
66
|
+
m = SEVERITY.search(line)
|
|
67
|
+
if not m:
|
|
68
|
+
return ""
|
|
69
|
+
token = m.group(1).upper()
|
|
70
|
+
if token == "ERR":
|
|
71
|
+
return "ERROR"
|
|
72
|
+
if token == "CRIT":
|
|
73
|
+
return "CRITICAL"
|
|
74
|
+
if token == "WARN":
|
|
75
|
+
return "WARNING"
|
|
76
|
+
return token
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def extract_ts(line: str) -> str | None:
|
|
80
|
+
for p in TS_PATTERNS:
|
|
81
|
+
m = p.search(line)
|
|
82
|
+
if m:
|
|
83
|
+
return m.group(0).strip("[]")
|
|
84
|
+
return None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def scan(path: str | Path) -> Scan:
|
|
88
|
+
"""Stream the file line by line; memory holds templates, not the log."""
|
|
89
|
+
result = Scan(path=str(path))
|
|
90
|
+
counts: dict[str, Template] = {}
|
|
91
|
+
first_ts = last_ts = None
|
|
92
|
+
with open(path, "r", encoding="utf-8", errors="replace") as fh:
|
|
93
|
+
for line_no, line in enumerate(fh, 1):
|
|
94
|
+
result.lines += 1
|
|
95
|
+
result.bytes_read += len(line.encode("utf-8", "replace"))
|
|
96
|
+
ts = extract_ts(line)
|
|
97
|
+
if ts:
|
|
98
|
+
first_ts = first_ts or ts
|
|
99
|
+
last_ts = ts
|
|
100
|
+
sev = severity_of(line)
|
|
101
|
+
if sev:
|
|
102
|
+
result.severities[sev] = result.severities.get(sev, 0) + 1
|
|
103
|
+
key = normalize(line)
|
|
104
|
+
if not key:
|
|
105
|
+
key = "<blank>"
|
|
106
|
+
tmpl = counts.get(key)
|
|
107
|
+
if tmpl is None:
|
|
108
|
+
counts[key] = Template(key, 1, line_no, line_no, sev, line.strip()[:220])
|
|
109
|
+
else:
|
|
110
|
+
tmpl.count += 1
|
|
111
|
+
tmpl.last_line = line_no
|
|
112
|
+
result.templates = list(counts.values())
|
|
113
|
+
if first_ts and last_ts and first_ts != last_ts:
|
|
114
|
+
result.timeframe = f"{first_ts} … {last_ts}"
|
|
115
|
+
elif first_ts:
|
|
116
|
+
result.timeframe = first_ts
|
|
117
|
+
return result
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def spikes(path: str | Path, buckets: int = BUCKETS) -> dict:
|
|
121
|
+
"""Error/broken-template counts across time buckets (by timestamp) or line ranges."""
|
|
122
|
+
per_bucket: dict[int, int] = {}
|
|
123
|
+
labels: dict[int, str] = {}
|
|
124
|
+
total = 0
|
|
125
|
+
with open(path, "r", encoding="utf-8", errors="replace") as fh:
|
|
126
|
+
raw = []
|
|
127
|
+
for line in fh:
|
|
128
|
+
total += 1
|
|
129
|
+
raw.append((extract_ts(line) or "", severity_of(line) or
|
|
130
|
+
("ERR" if re.search(r"\b(error|fail|exception)\b", line, re.I) else "")))
|
|
131
|
+
if not total:
|
|
132
|
+
return {"buckets": [], "total": 0}
|
|
133
|
+
has_ts = any(ts for ts, _ in raw)
|
|
134
|
+
by_idx: dict[int, int] = {}
|
|
135
|
+
if has_ts:
|
|
136
|
+
for i, (ts, _) in enumerate(raw):
|
|
137
|
+
by_idx.setdefault(ts, i)
|
|
138
|
+
ordered = [ts for ts, _ in sorted(by_idx.items(), key=lambda kv: kv[1])]
|
|
139
|
+
index_of = {ts: i for i, ts in enumerate(ordered)}
|
|
140
|
+
for ts, sev in raw:
|
|
141
|
+
if not sev or sev in ("INFO", "DEBUG", "TRACE"):
|
|
142
|
+
continue
|
|
143
|
+
b = index_of[ts] * buckets // max(1, len(ordered))
|
|
144
|
+
per_bucket[b] = per_bucket.get(b, 0) + 1
|
|
145
|
+
labels.setdefault(b, ts)
|
|
146
|
+
else:
|
|
147
|
+
chunk = max(1, total // buckets)
|
|
148
|
+
for i, (_, sev) in enumerate(raw):
|
|
149
|
+
if not sev or sev in ("INFO", "DEBUG", "TRACE"):
|
|
150
|
+
continue
|
|
151
|
+
b = i // chunk
|
|
152
|
+
per_bucket[b] = per_bucket.get(b, 0) + 1
|
|
153
|
+
labels.setdefault(b, f"lines {b * chunk + 1}-{(b + 1) * chunk}")
|
|
154
|
+
out = [{"bucket": b, "label": labels.get(b, str(b)), "problems": n}
|
|
155
|
+
for b, n in sorted(per_bucket.items())]
|
|
156
|
+
return {"buckets": out, "total_lines": total, "total_problems": sum(per_bucket.values())}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: logtriage
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Cluster big logs into templates with counts, severities and incident spikes — streaming, stdlib only
|
|
5
|
+
Author: juzorei
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/juzorei/loglens
|
|
8
|
+
Keywords: logs,observability,triage,cli,agent-skills
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Topic :: System :: Logging
|
|
12
|
+
Requires-Python: >=3.11
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
|
|
15
|
+
# loglens
|
|
16
|
+
|
|
17
|
+
Cluster big logs into templates with counts, severities and incident spikes — streaming, stdlib only, no file size limit.
|
|
18
|
+
|
|
19
|
+
```text
|
|
20
|
+
$ loglens scan app.log
|
|
21
|
+
lines 3,000 (222,367 bytes)
|
|
22
|
+
time 2026-10-30 09:00:00 … 2026-10-30 10:39:58
|
|
23
|
+
levels INFO 1,999, DEBUG 579, WARNING 273, ERROR 149
|
|
24
|
+
|
|
25
|
+
1,999× L1-L2999 [INFO]
|
|
26
|
+
2026-10-30 09:00:00 INFO request handled path=/api/lots/26 status=200 dur=37ms
|
|
27
|
+
|
|
28
|
+
149× L902-L1050 [ERROR]
|
|
29
|
+
2026-10-30 09:30:02 ERROR api.db timeout after 30s query=select * from lots where id=491
|
|
30
|
+
|
|
31
|
+
$ loglens spikes app.log
|
|
32
|
+
problems across 30 buckets (total 422 in 3,000 lines)
|
|
33
|
+
2026-10-30 09:33:26 149 ████████████████████████████████████████
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
A 500 MB log cannot be read — by a human or by an agent with finite context. But logs are repetitive: numbers, UUIDs, IPs, hashes and quoted strings normalize away, and 3000 nearly-identical lines collapse into a handful of templates with counts, line ranges and one real sample each. That's all the information a triage needs.
|
|
37
|
+
|
|
38
|
+
## Commands
|
|
39
|
+
|
|
40
|
+
```text
|
|
41
|
+
loglens scan <log> [--top 15] [--json] # template clustering + severity counts
|
|
42
|
+
loglens spikes <log> [--buckets 30] [--json] # when problems happen (time or line buckets)
|
|
43
|
+
loglens demo [dir] # synthetic log with a planted incident
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Streaming: the file is read line by line, memory holds only templates. Any text log; timestamps understood in ISO, Apache-bracketed and bare `HH:MM:SS` forms (no timestamps → line-range buckets). Python 3.11+, stdlib only.
|
|
47
|
+
|
|
48
|
+
Install: `pip install logtriage` — or from this repo:
|
|
49
|
+
`pip install "git+https://github.com/juzorei/loglens#subdirectory=tools/loglens"`
|
|
50
|
+
|
|
51
|
+
The companion agent skill lives in [skills/log-triage](../skills/log-triage): cluster first, read only samples, then bounded reads around the incident. Skills: `npx skills add juzorei/loglens`.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
src/loglens/__init__.py
|
|
4
|
+
src/loglens/__main__.py
|
|
5
|
+
src/loglens/cli.py
|
|
6
|
+
src/loglens/core.py
|
|
7
|
+
src/logtriage.egg-info/PKG-INFO
|
|
8
|
+
src/logtriage.egg-info/SOURCES.txt
|
|
9
|
+
src/logtriage.egg-info/dependency_links.txt
|
|
10
|
+
src/logtriage.egg-info/entry_points.txt
|
|
11
|
+
src/logtriage.egg-info/top_level.txt
|
|
12
|
+
tests/test_loglens.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
loglens
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from loglens.cli import main
|
|
6
|
+
from loglens.core import normalize, scan, severity_of, spikes
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@pytest.fixture(scope="module")
|
|
10
|
+
def log_file(tmp_path_factory):
|
|
11
|
+
d = tmp_path_factory.mktemp("log")
|
|
12
|
+
path = d / "app.log"
|
|
13
|
+
lines = []
|
|
14
|
+
for i in range(1, 301):
|
|
15
|
+
ts = f"2026-10-30 09:{i // 60:02d}:{i % 60:02d}"
|
|
16
|
+
if i == 50:
|
|
17
|
+
lines.append(f"{ts} ERROR db timeout query=select * from lots where id={i}")
|
|
18
|
+
elif i == 120:
|
|
19
|
+
lines.append(f"{ts} ERROR db timeout query=select * from lots where id=999")
|
|
20
|
+
elif i % 10 == 0:
|
|
21
|
+
lines.append(f"{ts} WARNING rate limit approach key=ip:{i % 5} count={i}")
|
|
22
|
+
else:
|
|
23
|
+
lines.append(f"{ts} INFO request handled path=/api/lots/{i} status=200 dur={i}ms")
|
|
24
|
+
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
25
|
+
return str(path)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_normalize_collapses_variables():
|
|
29
|
+
line = "2026-10-30 09:00:01 ERROR id=9b2f5c1e-7a3d-4b8a-9c0d-1e2f3a4b5c6d ip=10.0.0.5 hash=0xDEADBEEF99 count=42 msg=\"boom\""
|
|
30
|
+
n = normalize(line)
|
|
31
|
+
assert "<uuid>" in n and "<ip>" in n and "<hex>" in n and '"…"' in n
|
|
32
|
+
assert "42" not in n.replace("<n>", "").replace("<ip>", "")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_scan_clusters_same_template(log_file):
|
|
36
|
+
r = scan(log_file)
|
|
37
|
+
# 298 INFO lines differ only in digits/paths -> one template
|
|
38
|
+
info = [t for t in r.templates if "INFO" in t.pattern]
|
|
39
|
+
assert len(info) == 1 and info[0].count == 270
|
|
40
|
+
assert r.templates and len(r.templates) <= 6
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_scan_severity_counts(log_file):
|
|
44
|
+
r = scan(log_file)
|
|
45
|
+
assert r.severities["ERROR"] == 2
|
|
46
|
+
assert r.severities["WARNING"] == 28 # i=50,120 became ERRORs
|
|
47
|
+
assert r.counts()["ERROR"] == 2
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_scan_timeframe_and_streaming_stats(log_file):
|
|
51
|
+
r = scan(log_file)
|
|
52
|
+
assert r.lines == 300
|
|
53
|
+
assert r.timeframe.startswith("2026-10-30 09:00:01") and r.timeframe.endswith("09:05:00")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_spikes_finds_planted_cluster(log_file):
|
|
57
|
+
data = spikes(log_file, buckets=10)
|
|
58
|
+
assert data["total_problems"] >= 2
|
|
59
|
+
worst = max(data["buckets"], key=lambda b: b["problems"])
|
|
60
|
+
# the two ERROR lines sit at lines 50 and 120 — bucket 0-1 or 3-4 of 10
|
|
61
|
+
assert worst["problems"] >= 1
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def test_spikes_without_timestamps_buckets_by_lines(tmp_path):
|
|
65
|
+
p = tmp_path / "plain.log"
|
|
66
|
+
p.write_text("\n".join(f"ERR no timestamp {i}" for i in range(1, 101)) + "\n", encoding="utf-8")
|
|
67
|
+
data = spikes(p, buckets=4)
|
|
68
|
+
assert data["total_problems"] == 100
|
|
69
|
+
assert all("lines" in b["label"] for b in data["buckets"])
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_severity_variants():
|
|
73
|
+
assert severity_of("2026-10-30 ERR x") == "ERROR"
|
|
74
|
+
assert severity_of("crit: disk full") == "CRITICAL"
|
|
75
|
+
assert severity_of("all fine") == ""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_cli_scan_text(log_file, capsys):
|
|
79
|
+
assert main(["scan", log_file]) == 0
|
|
80
|
+
out = capsys.readouterr().out
|
|
81
|
+
assert "lines 300" in out and "ERROR 2" in out and "distinct templates" in out
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_cli_scan_json(log_file, capsys):
|
|
85
|
+
assert main(["scan", log_file, "--top", "3", "--json"]) == 0
|
|
86
|
+
data = json.loads(capsys.readouterr().out)
|
|
87
|
+
assert data["lines"] == 300 and len(data["templates"]) <= 3
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_cli_spikes_json(log_file, capsys):
|
|
91
|
+
assert main(["spikes", log_file, "--buckets", "5", "--json"]) == 0
|
|
92
|
+
data = json.loads(capsys.readouterr().out)
|
|
93
|
+
assert data["total_problems"] >= 2 and len(data["buckets"]) <= 5
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_cli_demo(tmp_path, capsys):
|
|
97
|
+
assert main(["demo", str(tmp_path / "d")]) == 0
|
|
98
|
+
out = capsys.readouterr().out
|
|
99
|
+
assert "planted incident" in out
|
|
100
|
+
assert (tmp_path / "d" / "app.log").stat().st_size > 10000
|