mdbkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mdbkit/__init__.py +3 -0
- mdbkit/advisor.py +302 -0
- mdbkit/analysis.py +436 -0
- mdbkit/cli.py +256 -0
- mdbkit/explain.py +244 -0
- mdbkit/filtering.py +55 -0
- mdbkit/parser.py +149 -0
- mdbkit/render.py +141 -0
- mdbkit/scripts.py +69 -0
- mdbkit/triage.py +360 -0
- mdbkit-0.1.0.dist-info/METADATA +207 -0
- mdbkit-0.1.0.dist-info/RECORD +16 -0
- mdbkit-0.1.0.dist-info/WHEEL +5 -0
- mdbkit-0.1.0.dist-info/entry_points.txt +2 -0
- mdbkit-0.1.0.dist-info/licenses/LICENSE +21 -0
- mdbkit-0.1.0.dist-info/top_level.txt +1 -0
mdbkit/analysis.py
ADDED
|
@@ -0,0 +1,436 @@
|
|
|
1
|
+
"""Aggregation of parsed log entries into DBA-useful summaries.
|
|
2
|
+
|
|
3
|
+
Three analyses, mirroring the most-used mtools workflows:
|
|
4
|
+
|
|
5
|
+
* summarize -> `mdbkit loginfo` (like mloginfo)
|
|
6
|
+
* QueryAggregator -> `mdbkit queries` (like mloginfo --queries)
|
|
7
|
+
* ConnectionAggregator -> `mdbkit connections` (like mloginfo --connections)
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import math
|
|
13
|
+
from collections import Counter, defaultdict
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from typing import Dict, List, Optional, Tuple
|
|
16
|
+
|
|
17
|
+
from .parser import (
|
|
18
|
+
ID_BUILD_INFO,
|
|
19
|
+
ID_CLIENT_METADATA,
|
|
20
|
+
ID_CONN_ACCEPTED,
|
|
21
|
+
ID_CONN_ENDED,
|
|
22
|
+
ID_STARTUP,
|
|
23
|
+
LogEntry,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
# ---------------------------------------------------------------------------
|
|
27
|
+
# Query shape extraction
|
|
28
|
+
# ---------------------------------------------------------------------------
|
|
29
|
+
|
|
30
|
+
# Operators that keep equality semantics for index purposes.
|
|
31
|
+
EQUALITY_OPS = {"$eq", "$in"}
|
|
32
|
+
RANGE_OPS = {"$gt", "$gte", "$lt", "$lte"}
|
|
33
|
+
LOW_SELECTIVITY_OPS = {"$ne", "$nin", "$exists", "$not"}
|
|
34
|
+
SPECIAL_OPS = {"$text", "$where", "$expr", "$geoWithin", "$geoIntersects", "$near", "$nearSphere"}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class QueryShape:
|
|
39
|
+
"""A literal-free signature of a query: what fields, which operators, what sort."""
|
|
40
|
+
|
|
41
|
+
ns: str
|
|
42
|
+
operation: str
|
|
43
|
+
filter_fields: Tuple[Tuple[str, Tuple[str, ...]], ...] # ((path, (ops...)), ...)
|
|
44
|
+
sort_fields: Tuple[Tuple[str, int], ...] # ((path, direction), ...)
|
|
45
|
+
flags: Tuple[str, ...] = () # e.g. ("$or", "$text")
|
|
46
|
+
|
|
47
|
+
def pretty(self) -> str:
|
|
48
|
+
parts = []
|
|
49
|
+
for path, ops in self.filter_fields:
|
|
50
|
+
parts.append(f"{path}:{'|'.join(o.lstrip('$') for o in ops)}")
|
|
51
|
+
text = "{" + ", ".join(parts) + "}"
|
|
52
|
+
if self.sort_fields:
|
|
53
|
+
text += " sort:{" + ", ".join(f"{p}:{d}" for p, d in self.sort_fields) + "}"
|
|
54
|
+
if self.flags:
|
|
55
|
+
text += " [" + ",".join(self.flags) + "]"
|
|
56
|
+
return text
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _walk_filter(node, prefix: str, out: Dict[str, set], flags: set, depth: int = 0):
|
|
60
|
+
"""Recursively collect (field path -> operator set) from a filter document."""
|
|
61
|
+
if depth > 12 or not isinstance(node, dict):
|
|
62
|
+
return
|
|
63
|
+
for key, value in node.items():
|
|
64
|
+
if key in ("$and",):
|
|
65
|
+
if isinstance(value, list):
|
|
66
|
+
for sub in value:
|
|
67
|
+
_walk_filter(sub, prefix, out, flags, depth + 1)
|
|
68
|
+
elif key in ("$or", "$nor"):
|
|
69
|
+
flags.add(key)
|
|
70
|
+
if isinstance(value, list):
|
|
71
|
+
for sub in value:
|
|
72
|
+
_walk_filter(sub, prefix, out, flags, depth + 1)
|
|
73
|
+
elif key in SPECIAL_OPS:
|
|
74
|
+
flags.add(key)
|
|
75
|
+
elif key.startswith("$"):
|
|
76
|
+
# An operator appearing at document level we don't model; note it.
|
|
77
|
+
flags.add(key)
|
|
78
|
+
else:
|
|
79
|
+
path = f"{prefix}.{key}" if prefix else key
|
|
80
|
+
if isinstance(value, dict):
|
|
81
|
+
ops = {op for op in value.keys() if op.startswith("$")}
|
|
82
|
+
if ops:
|
|
83
|
+
out[path].update(ops)
|
|
84
|
+
inner = value.get("$elemMatch")
|
|
85
|
+
if isinstance(inner, dict):
|
|
86
|
+
_walk_filter(inner, path, out, flags, depth + 1)
|
|
87
|
+
if "$regex" in ops:
|
|
88
|
+
flags.add("$regex")
|
|
89
|
+
else:
|
|
90
|
+
# Sub-document equality match on the whole object.
|
|
91
|
+
out[path].add("$eq")
|
|
92
|
+
else:
|
|
93
|
+
out[path].add("$eq")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def extract_shape(entry: LogEntry) -> Optional[QueryShape]:
|
|
97
|
+
"""Build a QueryShape from a 'Slow query' log entry, or None if not applicable."""
|
|
98
|
+
attr = entry.attr
|
|
99
|
+
command = attr.get("command") if isinstance(attr.get("command"), dict) else {}
|
|
100
|
+
op_type = attr.get("type", "")
|
|
101
|
+
|
|
102
|
+
ns = attr.get("ns", "")
|
|
103
|
+
if (not ns or ns.endswith(".$cmd") or ns.endswith(".")) and command:
|
|
104
|
+
coll = (command.get("find") or command.get("aggregate")
|
|
105
|
+
or command.get("update") or command.get("delete")
|
|
106
|
+
or command.get("insert") or command.get("findAndModify")
|
|
107
|
+
or command.get("count") or command.get("distinct") or "")
|
|
108
|
+
db = command.get("$db", "")
|
|
109
|
+
if db and coll:
|
|
110
|
+
ns = f"{db}.{coll}"
|
|
111
|
+
|
|
112
|
+
filter_doc: dict = {}
|
|
113
|
+
sort_doc: dict = {}
|
|
114
|
+
operation = "unknown"
|
|
115
|
+
|
|
116
|
+
if "find" in command:
|
|
117
|
+
operation = "find"
|
|
118
|
+
filter_doc = command.get("filter") or {}
|
|
119
|
+
sort_doc = command.get("sort") or {}
|
|
120
|
+
elif "aggregate" in command:
|
|
121
|
+
operation = "aggregate"
|
|
122
|
+
pipeline = command.get("pipeline") or []
|
|
123
|
+
for stage in pipeline:
|
|
124
|
+
if not isinstance(stage, dict):
|
|
125
|
+
continue
|
|
126
|
+
if "$match" in stage and not filter_doc:
|
|
127
|
+
filter_doc = stage["$match"] or {}
|
|
128
|
+
elif "$sort" in stage and not sort_doc:
|
|
129
|
+
sort_doc = stage["$sort"] or {}
|
|
130
|
+
elif filter_doc or sort_doc:
|
|
131
|
+
break # only leading $match/$sort benefit from an index
|
|
132
|
+
elif "count" in command:
|
|
133
|
+
operation = "count"
|
|
134
|
+
filter_doc = command.get("query") or {}
|
|
135
|
+
elif "distinct" in command:
|
|
136
|
+
operation = "distinct"
|
|
137
|
+
filter_doc = command.get("query") or {}
|
|
138
|
+
elif "getMore" in command:
|
|
139
|
+
origin = command.get("originatingCommand")
|
|
140
|
+
if isinstance(origin, dict):
|
|
141
|
+
fake = LogEntry(
|
|
142
|
+
ts=entry.ts, severity=entry.severity, component=entry.component,
|
|
143
|
+
msg_id=entry.msg_id, ctx=entry.ctx, msg=entry.msg,
|
|
144
|
+
attr={"ns": ns, "command": origin},
|
|
145
|
+
)
|
|
146
|
+
shape = extract_shape(fake)
|
|
147
|
+
if shape:
|
|
148
|
+
return QueryShape(shape.ns, "getMore(" + shape.operation + ")",
|
|
149
|
+
shape.filter_fields, shape.sort_fields, shape.flags)
|
|
150
|
+
operation = "getMore"
|
|
151
|
+
elif "update" in command and isinstance(command.get("updates"), list):
|
|
152
|
+
operation = "update"
|
|
153
|
+
first = command["updates"][0] if command["updates"] else {}
|
|
154
|
+
if isinstance(first, dict):
|
|
155
|
+
filter_doc = first.get("q") or {}
|
|
156
|
+
elif "delete" in command and isinstance(command.get("deletes"), list):
|
|
157
|
+
operation = "delete"
|
|
158
|
+
first = command["deletes"][0] if command["deletes"] else {}
|
|
159
|
+
if isinstance(first, dict):
|
|
160
|
+
filter_doc = first.get("q") or {}
|
|
161
|
+
elif op_type in ("update", "remove"):
|
|
162
|
+
operation = op_type
|
|
163
|
+
filter_doc = command.get("q") or {}
|
|
164
|
+
elif "findAndModify" in command:
|
|
165
|
+
operation = "findAndModify"
|
|
166
|
+
filter_doc = command.get("query") or {}
|
|
167
|
+
sort_doc = command.get("sort") or {}
|
|
168
|
+
elif "insert" in command or op_type == "insert":
|
|
169
|
+
operation = "insert"
|
|
170
|
+
elif command:
|
|
171
|
+
operation = next(iter(command.keys()), "unknown")
|
|
172
|
+
|
|
173
|
+
fields: Dict[str, set] = defaultdict(set)
|
|
174
|
+
flags: set = set()
|
|
175
|
+
_walk_filter(filter_doc, "", fields, flags)
|
|
176
|
+
|
|
177
|
+
filter_fields = tuple(sorted((p, tuple(sorted(ops))) for p, ops in fields.items()))
|
|
178
|
+
sort_fields = tuple(
|
|
179
|
+
(k, int(v) if isinstance(v, (int, float)) else 1) for k, v in sort_doc.items()
|
|
180
|
+
) if isinstance(sort_doc, dict) else ()
|
|
181
|
+
|
|
182
|
+
return QueryShape(ns=ns, operation=operation, filter_fields=filter_fields,
|
|
183
|
+
sort_fields=sort_fields, flags=tuple(sorted(flags)))
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
# ---------------------------------------------------------------------------
|
|
187
|
+
# Slow query aggregation
|
|
188
|
+
# ---------------------------------------------------------------------------
|
|
189
|
+
|
|
190
|
+
@dataclass
|
|
191
|
+
class ShapeStats:
|
|
192
|
+
shape: QueryShape
|
|
193
|
+
count: int = 0
|
|
194
|
+
durations: List[int] = field(default_factory=list)
|
|
195
|
+
docs_examined: int = 0
|
|
196
|
+
keys_examined: int = 0
|
|
197
|
+
n_returned: int = 0
|
|
198
|
+
collscan: bool = False
|
|
199
|
+
in_memory_sort: bool = False
|
|
200
|
+
plan_summaries: Counter = field(default_factory=Counter)
|
|
201
|
+
example: str = ""
|
|
202
|
+
|
|
203
|
+
def add(self, entry: LogEntry):
|
|
204
|
+
attr = entry.attr
|
|
205
|
+
self.count += 1
|
|
206
|
+
self.durations.append(int(attr.get("durationMillis", 0) or 0))
|
|
207
|
+
self.docs_examined += int(attr.get("docsExamined", 0) or 0)
|
|
208
|
+
self.keys_examined += int(attr.get("keysExamined", 0) or 0)
|
|
209
|
+
n = attr.get("nreturned")
|
|
210
|
+
if n is None:
|
|
211
|
+
n = attr.get("nMatched")
|
|
212
|
+
if n is None:
|
|
213
|
+
n = attr.get("ndeleted", attr.get("nDeleted"))
|
|
214
|
+
self.n_returned += int(n or 0)
|
|
215
|
+
plan = attr.get("planSummary", "")
|
|
216
|
+
if plan:
|
|
217
|
+
self.plan_summaries[plan] += 1
|
|
218
|
+
if "COLLSCAN" in plan:
|
|
219
|
+
self.collscan = True
|
|
220
|
+
if attr.get("hasSortStage"):
|
|
221
|
+
self.in_memory_sort = True
|
|
222
|
+
if not self.example:
|
|
223
|
+
self.example = entry.raw[:2000]
|
|
224
|
+
|
|
225
|
+
# -- derived metrics ---------------------------------------------------
|
|
226
|
+
@property
|
|
227
|
+
def total_ms(self) -> int:
|
|
228
|
+
return sum(self.durations)
|
|
229
|
+
|
|
230
|
+
@property
|
|
231
|
+
def mean_ms(self) -> float:
|
|
232
|
+
return self.total_ms / self.count if self.count else 0.0
|
|
233
|
+
|
|
234
|
+
@property
|
|
235
|
+
def max_ms(self) -> int:
|
|
236
|
+
return max(self.durations) if self.durations else 0
|
|
237
|
+
|
|
238
|
+
@property
|
|
239
|
+
def p95_ms(self) -> int:
|
|
240
|
+
if not self.durations:
|
|
241
|
+
return 0
|
|
242
|
+
ordered = sorted(self.durations)
|
|
243
|
+
idx = max(0, math.ceil(0.95 * len(ordered)) - 1)
|
|
244
|
+
return ordered[idx]
|
|
245
|
+
|
|
246
|
+
@property
|
|
247
|
+
def scan_ratio(self) -> float:
|
|
248
|
+
"""docsExamined per document returned — the classic inefficiency signal."""
|
|
249
|
+
return self.docs_examined / self.n_returned if self.n_returned else float(
|
|
250
|
+
self.docs_examined or 0
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
def to_dict(self) -> dict:
|
|
254
|
+
return {
|
|
255
|
+
"ns": self.shape.ns,
|
|
256
|
+
"operation": self.shape.operation,
|
|
257
|
+
"shape": self.shape.pretty(),
|
|
258
|
+
"count": self.count,
|
|
259
|
+
"totalMs": self.total_ms,
|
|
260
|
+
"meanMs": round(self.mean_ms, 1),
|
|
261
|
+
"maxMs": self.max_ms,
|
|
262
|
+
"p95Ms": self.p95_ms,
|
|
263
|
+
"docsExamined": self.docs_examined,
|
|
264
|
+
"keysExamined": self.keys_examined,
|
|
265
|
+
"nReturned": self.n_returned,
|
|
266
|
+
"scanRatio": round(self.scan_ratio, 1),
|
|
267
|
+
"collscan": self.collscan,
|
|
268
|
+
"inMemorySort": self.in_memory_sort,
|
|
269
|
+
"planSummaries": dict(self.plan_summaries),
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
class QueryAggregator:
|
|
274
|
+
def __init__(self, min_ms: int = 0):
|
|
275
|
+
self.min_ms = min_ms
|
|
276
|
+
self.shapes: Dict[QueryShape, ShapeStats] = {}
|
|
277
|
+
|
|
278
|
+
NOISE_OPS = frozenset({
|
|
279
|
+
"hello", "isMaster", "ismaster", "ping", "endSessions", "saslStart",
|
|
280
|
+
"saslContinue", "buildInfo", "getParameter", "serverStatus", "getLog",
|
|
281
|
+
"replSetGetStatus", "connPoolStats", "getCmdLineOpts", "whatsmyuri",
|
|
282
|
+
"logout", "killCursors", "listCollections", "listIndexes",
|
|
283
|
+
"listDatabases", "getFreeMonitoringStatus", "dbStats", "collStats",
|
|
284
|
+
"top", "currentOp", "profile", "hostInfo", "createIndexes",
|
|
285
|
+
})
|
|
286
|
+
|
|
287
|
+
def consume(self, entry: LogEntry):
|
|
288
|
+
if not entry.is_slow_query:
|
|
289
|
+
return
|
|
290
|
+
if int(entry.attr.get("durationMillis", 0) or 0) < self.min_ms:
|
|
291
|
+
return
|
|
292
|
+
shape = extract_shape(entry)
|
|
293
|
+
if shape is None or shape.operation == "insert":
|
|
294
|
+
return
|
|
295
|
+
if shape.operation in self.NOISE_OPS and not shape.filter_fields:
|
|
296
|
+
return
|
|
297
|
+
# Batched update/delete at COMMAND level often omits the per-op q;
|
|
298
|
+
# the paired WRITE entry carries the real shape and metrics.
|
|
299
|
+
if (shape.operation in ("update", "delete") and not shape.filter_fields
|
|
300
|
+
and entry.component == "COMMAND"):
|
|
301
|
+
return
|
|
302
|
+
stats = self.shapes.get(shape)
|
|
303
|
+
if stats is None:
|
|
304
|
+
stats = self.shapes[shape] = ShapeStats(shape=shape)
|
|
305
|
+
stats.add(entry)
|
|
306
|
+
|
|
307
|
+
def results(self, sort_by: str = "totalMs", limit: int = 0) -> List[ShapeStats]:
|
|
308
|
+
keymap = {
|
|
309
|
+
"duration": lambda s: s.total_ms,
|
|
310
|
+
"totalMs": lambda s: s.total_ms,
|
|
311
|
+
"count": lambda s: s.count,
|
|
312
|
+
"mean": lambda s: s.mean_ms,
|
|
313
|
+
"max": lambda s: s.max_ms,
|
|
314
|
+
"docsExamined": lambda s: s.docs_examined,
|
|
315
|
+
"scanRatio": lambda s: s.scan_ratio,
|
|
316
|
+
}
|
|
317
|
+
key = keymap.get(sort_by, keymap["totalMs"])
|
|
318
|
+
ordered = sorted(self.shapes.values(), key=key, reverse=True)
|
|
319
|
+
return ordered[:limit] if limit else ordered
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
# ---------------------------------------------------------------------------
|
|
323
|
+
# Connections
|
|
324
|
+
# ---------------------------------------------------------------------------
|
|
325
|
+
|
|
326
|
+
def _ip_of(remote: str) -> str:
|
|
327
|
+
return remote.rsplit(":", 1)[0] if remote else "unknown"
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
@dataclass
|
|
331
|
+
class ConnectionReport:
|
|
332
|
+
accepted: Counter = field(default_factory=Counter)
|
|
333
|
+
ended: Counter = field(default_factory=Counter)
|
|
334
|
+
app_names: Counter = field(default_factory=Counter)
|
|
335
|
+
drivers: Counter = field(default_factory=Counter)
|
|
336
|
+
peak_count: int = 0
|
|
337
|
+
|
|
338
|
+
def to_dict(self) -> dict:
|
|
339
|
+
return {
|
|
340
|
+
"totalAccepted": sum(self.accepted.values()),
|
|
341
|
+
"totalEnded": sum(self.ended.values()),
|
|
342
|
+
"peakConnectionCount": self.peak_count,
|
|
343
|
+
"byIp": [
|
|
344
|
+
{"ip": ip, "accepted": n, "ended": self.ended.get(ip, 0)}
|
|
345
|
+
for ip, n in self.accepted.most_common()
|
|
346
|
+
],
|
|
347
|
+
"appNames": dict(self.app_names.most_common()),
|
|
348
|
+
"drivers": dict(self.drivers.most_common()),
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
class ConnectionAggregator:
|
|
353
|
+
def __init__(self):
|
|
354
|
+
self.report = ConnectionReport()
|
|
355
|
+
|
|
356
|
+
def consume(self, entry: LogEntry):
|
|
357
|
+
attr = entry.attr
|
|
358
|
+
if entry.msg_id == ID_CONN_ACCEPTED:
|
|
359
|
+
self.report.accepted[_ip_of(attr.get("remote", ""))] += 1
|
|
360
|
+
self.report.peak_count = max(
|
|
361
|
+
self.report.peak_count, int(attr.get("connectionCount", 0) or 0)
|
|
362
|
+
)
|
|
363
|
+
elif entry.msg_id == ID_CONN_ENDED:
|
|
364
|
+
self.report.ended[_ip_of(attr.get("remote", ""))] += 1
|
|
365
|
+
self.report.peak_count = max(
|
|
366
|
+
self.report.peak_count, int(attr.get("connectionCount", 0) or 0)
|
|
367
|
+
)
|
|
368
|
+
elif entry.msg_id == ID_CLIENT_METADATA:
|
|
369
|
+
doc = attr.get("doc", {}) or {}
|
|
370
|
+
app = (doc.get("application") or {}).get("name")
|
|
371
|
+
if app:
|
|
372
|
+
self.report.app_names[app] += 1
|
|
373
|
+
driver = (doc.get("driver") or {}).get("name")
|
|
374
|
+
if driver:
|
|
375
|
+
self.report.drivers[driver] += 1
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
# ---------------------------------------------------------------------------
|
|
379
|
+
# Whole-log summary (loginfo)
|
|
380
|
+
# ---------------------------------------------------------------------------
|
|
381
|
+
|
|
382
|
+
@dataclass
|
|
383
|
+
class LogSummary:
|
|
384
|
+
versions: List[str] = field(default_factory=list)
|
|
385
|
+
startups: int = 0
|
|
386
|
+
host_info: List[str] = field(default_factory=list)
|
|
387
|
+
severity_counts: Counter = field(default_factory=Counter)
|
|
388
|
+
component_counts: Counter = field(default_factory=Counter)
|
|
389
|
+
slow_queries: int = 0
|
|
390
|
+
slowest_ms: int = 0
|
|
391
|
+
connections_accepted: int = 0
|
|
392
|
+
warnings: int = 0
|
|
393
|
+
errors: int = 0
|
|
394
|
+
|
|
395
|
+
def to_dict(self) -> dict:
|
|
396
|
+
return {
|
|
397
|
+
"startups": self.startups,
|
|
398
|
+
"versions": self.versions,
|
|
399
|
+
"hosts": self.host_info,
|
|
400
|
+
"slowQueries": self.slow_queries,
|
|
401
|
+
"slowestMs": self.slowest_ms,
|
|
402
|
+
"connectionsAccepted": self.connections_accepted,
|
|
403
|
+
"warnings": self.warnings,
|
|
404
|
+
"errors": self.errors,
|
|
405
|
+
"bySeverity": dict(self.severity_counts),
|
|
406
|
+
"byComponent": dict(self.component_counts.most_common()),
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
class SummaryAggregator:
|
|
411
|
+
def __init__(self):
|
|
412
|
+
self.summary = LogSummary()
|
|
413
|
+
|
|
414
|
+
def consume(self, entry: LogEntry):
|
|
415
|
+
s = self.summary
|
|
416
|
+
s.severity_counts[entry.severity] += 1
|
|
417
|
+
s.component_counts[entry.component] += 1
|
|
418
|
+
if entry.severity == "W":
|
|
419
|
+
s.warnings += 1
|
|
420
|
+
elif entry.severity in ("E", "F"):
|
|
421
|
+
s.errors += 1
|
|
422
|
+
if entry.msg_id == ID_STARTUP:
|
|
423
|
+
s.startups += 1
|
|
424
|
+
host = entry.attr.get("host")
|
|
425
|
+
port = entry.attr.get("port")
|
|
426
|
+
if host:
|
|
427
|
+
s.host_info.append(f"{host}:{port}" if port else str(host))
|
|
428
|
+
elif entry.msg_id == ID_BUILD_INFO:
|
|
429
|
+
version = (entry.attr.get("buildInfo") or {}).get("version")
|
|
430
|
+
if version and version not in s.versions:
|
|
431
|
+
s.versions.append(version)
|
|
432
|
+
elif entry.msg_id == ID_CONN_ACCEPTED:
|
|
433
|
+
s.connections_accepted += 1
|
|
434
|
+
if entry.is_slow_query:
|
|
435
|
+
s.slow_queries += 1
|
|
436
|
+
s.slowest_ms = max(s.slowest_ms, int(entry.attr.get("durationMillis", 0) or 0))
|
mdbkit/cli.py
ADDED
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
"""mdbkit command-line interface.
|
|
2
|
+
|
|
3
|
+
Fully offline: reads files/stdin, writes stdout. No network, no telemetry.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
from . import __version__
|
|
12
|
+
from .advisor import advise, load_indexes, load_schema
|
|
13
|
+
from .analysis import ConnectionAggregator, QueryAggregator, SummaryAggregator
|
|
14
|
+
from .filtering import Filter, parse_when
|
|
15
|
+
from .parser import PRE_44_HINT, ParseStats, iter_entries
|
|
16
|
+
from .render import (
|
|
17
|
+
dump_json,
|
|
18
|
+
render_connections,
|
|
19
|
+
render_queries,
|
|
20
|
+
render_recommendations,
|
|
21
|
+
render_summary,
|
|
22
|
+
)
|
|
23
|
+
from .scripts import INDEXES_SCRIPT, SCHEMA_SCRIPT
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _warn_if_pre44(stats: ParseStats) -> None:
|
|
27
|
+
if stats.total_lines and stats.unparsed_ratio > 0.5:
|
|
28
|
+
print(f"warning: {PRE_44_HINT}", file=sys.stderr)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def cmd_loginfo(args) -> int:
|
|
32
|
+
stats = ParseStats()
|
|
33
|
+
agg = SummaryAggregator()
|
|
34
|
+
for entry in iter_entries(args.logfile, stats):
|
|
35
|
+
agg.consume(entry)
|
|
36
|
+
_warn_if_pre44(stats)
|
|
37
|
+
if args.json:
|
|
38
|
+
out = agg.summary.to_dict()
|
|
39
|
+
out["parse"] = {"lines": stats.total_lines, "parsed": stats.parsed,
|
|
40
|
+
"unparsed": stats.unparsed}
|
|
41
|
+
print(dump_json(out))
|
|
42
|
+
else:
|
|
43
|
+
print(render_summary(agg.summary, stats))
|
|
44
|
+
return 0
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def cmd_queries(args) -> int:
|
|
48
|
+
stats = ParseStats()
|
|
49
|
+
agg = QueryAggregator(min_ms=args.min_ms)
|
|
50
|
+
for entry in iter_entries(args.logfile, stats):
|
|
51
|
+
agg.consume(entry)
|
|
52
|
+
_warn_if_pre44(stats)
|
|
53
|
+
results = agg.results(sort_by=args.sort, limit=args.limit)
|
|
54
|
+
if args.json:
|
|
55
|
+
print(dump_json([s.to_dict() for s in results]))
|
|
56
|
+
else:
|
|
57
|
+
print(render_queries(results, stats))
|
|
58
|
+
return 0
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def cmd_connections(args) -> int:
|
|
62
|
+
stats = ParseStats()
|
|
63
|
+
agg = ConnectionAggregator()
|
|
64
|
+
for entry in iter_entries(args.logfile, stats):
|
|
65
|
+
agg.consume(entry)
|
|
66
|
+
_warn_if_pre44(stats)
|
|
67
|
+
if args.json:
|
|
68
|
+
print(dump_json(agg.report.to_dict()))
|
|
69
|
+
else:
|
|
70
|
+
print(render_connections(agg.report, stats))
|
|
71
|
+
return 0
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def cmd_filter(args) -> int:
|
|
75
|
+
flt = Filter(
|
|
76
|
+
component=args.component,
|
|
77
|
+
severity=args.severity,
|
|
78
|
+
namespace=args.ns,
|
|
79
|
+
slow_ms=args.slow,
|
|
80
|
+
ts_from=parse_when(args.ts_from) if args.ts_from else None,
|
|
81
|
+
ts_to=parse_when(args.ts_to) if args.ts_to else None,
|
|
82
|
+
msg_contains=args.msg,
|
|
83
|
+
)
|
|
84
|
+
stats = ParseStats()
|
|
85
|
+
matched = 0
|
|
86
|
+
for entry in iter_entries(args.logfile, stats):
|
|
87
|
+
if flt.matches(entry):
|
|
88
|
+
print(entry.raw)
|
|
89
|
+
matched += 1
|
|
90
|
+
_warn_if_pre44(stats)
|
|
91
|
+
print(f"filter: matched {matched:,} of {stats.parsed:,} parsed lines",
|
|
92
|
+
file=sys.stderr)
|
|
93
|
+
return 0
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def cmd_advise(args) -> int:
|
|
97
|
+
stats = ParseStats()
|
|
98
|
+
agg = QueryAggregator(min_ms=args.min_ms)
|
|
99
|
+
for entry in iter_entries(args.logfile, stats):
|
|
100
|
+
agg.consume(entry)
|
|
101
|
+
_warn_if_pre44(stats)
|
|
102
|
+
indexes = load_indexes(args.indexes) if args.indexes else None
|
|
103
|
+
schema = load_schema(args.schema) if args.schema else None
|
|
104
|
+
recs = advise(agg.results(), indexes=indexes, schema=schema,
|
|
105
|
+
min_count=args.min_count)
|
|
106
|
+
if not args.indexes:
|
|
107
|
+
print("note: no --indexes file given; overlap with existing indexes was "
|
|
108
|
+
"not checked. Export with: mdbkit export-script indexes",
|
|
109
|
+
file=sys.stderr)
|
|
110
|
+
if args.json:
|
|
111
|
+
print(dump_json([r.to_dict() for r in recs]))
|
|
112
|
+
else:
|
|
113
|
+
print(render_recommendations(recs, stats))
|
|
114
|
+
return 0
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def cmd_explain(args) -> int:
|
|
118
|
+
from .explain import analyze_explain, load_explain, render_explain
|
|
119
|
+
try:
|
|
120
|
+
doc = load_explain(args.explainfile)
|
|
121
|
+
except ValueError as exc:
|
|
122
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
123
|
+
return 2
|
|
124
|
+
indexes = load_indexes(args.indexes) if args.indexes else None
|
|
125
|
+
schema = load_schema(args.schema) if args.schema else None
|
|
126
|
+
report = analyze_explain(doc, indexes=indexes, schema=schema)
|
|
127
|
+
if args.json:
|
|
128
|
+
print(dump_json(report.to_dict()))
|
|
129
|
+
else:
|
|
130
|
+
print(render_explain(report))
|
|
131
|
+
return 0
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def cmd_triage(args) -> int:
|
|
135
|
+
from .triage import render_triage, run_triage
|
|
136
|
+
findings, stats, cutoff = run_triage(
|
|
137
|
+
args.logfile, window_min=args.window, dbpath=args.dbpath,
|
|
138
|
+
no_sysprobe=args.no_sysprobe)
|
|
139
|
+
_warn_if_pre44(stats)
|
|
140
|
+
if args.json:
|
|
141
|
+
print(dump_json({
|
|
142
|
+
"window": {"from": cutoff or stats.first_ts, "to": stats.last_ts},
|
|
143
|
+
"findings": [f.to_dict() for f in findings],
|
|
144
|
+
}))
|
|
145
|
+
else:
|
|
146
|
+
print(render_triage(findings, stats, cutoff))
|
|
147
|
+
return 0
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def cmd_export_script(args) -> int:
|
|
151
|
+
print(SCHEMA_SCRIPT if args.kind == "schema" else INDEXES_SCRIPT)
|
|
152
|
+
return 0
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
156
|
+
p = argparse.ArgumentParser(
|
|
157
|
+
prog="mdbkit",
|
|
158
|
+
description=(
|
|
159
|
+
"Offline toolkit for MongoDB 4.4+ structured logs: log summaries, "
|
|
160
|
+
"slow-query analysis, connection churn, log filtering, and "
|
|
161
|
+
"deterministic candidate-index advice. A spiritual successor to "
|
|
162
|
+
"mtools' log tools. Never connects to a network."
|
|
163
|
+
),
|
|
164
|
+
)
|
|
165
|
+
p.add_argument("--version", action="version", version=f"mdbkit {__version__}")
|
|
166
|
+
sub = p.add_subparsers(dest="command", required=True)
|
|
167
|
+
|
|
168
|
+
def add_common(sp):
|
|
169
|
+
sp.add_argument("logfile", help="path to mongod/mongos log (.log or .gz), or '-' for stdin")
|
|
170
|
+
sp.add_argument("--json", action="store_true", help="machine-readable JSON output")
|
|
171
|
+
|
|
172
|
+
sp = sub.add_parser("loginfo", help="overall log summary (versions, restarts, counts)")
|
|
173
|
+
add_common(sp)
|
|
174
|
+
sp.set_defaults(func=cmd_loginfo)
|
|
175
|
+
|
|
176
|
+
sp = sub.add_parser("queries", help="group and rank slow queries by shape")
|
|
177
|
+
add_common(sp)
|
|
178
|
+
sp.add_argument("--sort", default="totalMs",
|
|
179
|
+
choices=["totalMs", "duration", "count", "mean", "max",
|
|
180
|
+
"docsExamined", "scanRatio"])
|
|
181
|
+
sp.add_argument("--limit", type=int, default=0, help="show top N shapes")
|
|
182
|
+
sp.add_argument("--min-ms", type=int, default=0,
|
|
183
|
+
help="ignore operations faster than this")
|
|
184
|
+
sp.set_defaults(func=cmd_queries)
|
|
185
|
+
|
|
186
|
+
sp = sub.add_parser("connections", help="connection churn by source IP and app")
|
|
187
|
+
add_common(sp)
|
|
188
|
+
sp.set_defaults(func=cmd_connections)
|
|
189
|
+
|
|
190
|
+
sp = sub.add_parser("filter", help="stream matching raw log lines (chainable)")
|
|
191
|
+
sp.add_argument("logfile")
|
|
192
|
+
sp.add_argument("--component", help="e.g. COMMAND, NETWORK, REPL")
|
|
193
|
+
sp.add_argument("--severity", help="I, W, E, F")
|
|
194
|
+
sp.add_argument("--ns", help="exact namespace, e.g. shop.orders")
|
|
195
|
+
sp.add_argument("--slow", type=int, help="only ops with durationMillis >= N")
|
|
196
|
+
sp.add_argument("--from", dest="ts_from", help="ISO timestamp lower bound")
|
|
197
|
+
sp.add_argument("--to", dest="ts_to", help="ISO timestamp upper bound")
|
|
198
|
+
sp.add_argument("--msg", help="substring match on the msg field")
|
|
199
|
+
sp.set_defaults(func=cmd_filter)
|
|
200
|
+
|
|
201
|
+
sp = sub.add_parser("advise", help="deterministic candidate-index recommendations")
|
|
202
|
+
add_common(sp)
|
|
203
|
+
sp.add_argument("--indexes", help="indexes.json from `mdbkit export-script indexes`")
|
|
204
|
+
sp.add_argument("--schema", help="schema.json from `mdbkit export-script schema`")
|
|
205
|
+
sp.add_argument("--min-ms", type=int, default=0)
|
|
206
|
+
sp.add_argument("--min-count", type=int, default=1,
|
|
207
|
+
help="only advise on shapes seen at least N times")
|
|
208
|
+
sp.set_defaults(func=cmd_advise)
|
|
209
|
+
|
|
210
|
+
sp = sub.add_parser("explain",
|
|
211
|
+
help="analyze a saved explain('executionStats') JSON file")
|
|
212
|
+
sp.add_argument("explainfile",
|
|
213
|
+
help="explain output saved as JSON (mongosh EJSON.stringify "
|
|
214
|
+
"or Compass export)")
|
|
215
|
+
sp.add_argument("--json", action="store_true", help="machine-readable output")
|
|
216
|
+
sp.add_argument("--indexes", help="indexes.json for overlap checks")
|
|
217
|
+
sp.add_argument("--schema", help="schema.json for field caveats")
|
|
218
|
+
sp.set_defaults(func=cmd_explain)
|
|
219
|
+
|
|
220
|
+
sp = sub.add_parser("triage",
|
|
221
|
+
help="one-command incident snapshot (beta): elections, "
|
|
222
|
+
"storms, hot collections, errors + local disk/"
|
|
223
|
+
"memory/load probes")
|
|
224
|
+
sp.add_argument("logfile", help="mongod log (.log or .gz), or '-'")
|
|
225
|
+
sp.add_argument("--window", type=int, metavar="MINUTES",
|
|
226
|
+
help="analyze only the last N minutes of log time "
|
|
227
|
+
"(default: whole file)")
|
|
228
|
+
sp.add_argument("--dbpath", help="override dbPath for the disk probe")
|
|
229
|
+
sp.add_argument("--no-sysprobe", action="store_true",
|
|
230
|
+
help="skip local disk/memory/load probes")
|
|
231
|
+
sp.add_argument("--json", action="store_true")
|
|
232
|
+
sp.set_defaults(func=cmd_triage)
|
|
233
|
+
|
|
234
|
+
sp = sub.add_parser("export-script",
|
|
235
|
+
help="print a mongosh script to export schema or indexes")
|
|
236
|
+
sp.add_argument("kind", choices=["schema", "indexes"])
|
|
237
|
+
sp.set_defaults(func=cmd_export_script)
|
|
238
|
+
|
|
239
|
+
return p
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def main(argv=None) -> int:
|
|
243
|
+
args = build_parser().parse_args(argv)
|
|
244
|
+
try:
|
|
245
|
+
return args.func(args)
|
|
246
|
+
except FileNotFoundError as exc:
|
|
247
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
248
|
+
return 2
|
|
249
|
+
except KeyboardInterrupt:
|
|
250
|
+
return 130
|
|
251
|
+
except BrokenPipeError:
|
|
252
|
+
return 0
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
if __name__ == "__main__":
|
|
256
|
+
sys.exit(main())
|