mongo-x-ray-log 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mongo_x_ray_log-2.0.0/PKG-INFO +84 -0
- mongo_x_ray_log-2.0.0/README.md +75 -0
- mongo_x_ray_log-2.0.0/pyproject.toml +60 -0
- mongo_x_ray_log-2.0.0/setup.cfg +4 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/__init__.py +11 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/ai.py +25 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/framework.py +330 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/__init__.py +9 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/base_item.py +169 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/client_meta_item.py +71 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/connection_rate_item.py +61 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/info_item.py +97 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/log_rate_item.py +67 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/slow_chart_item.py +35 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/slow_rate_item.py +54 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/state_trace_item.py +171 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/top_slow_item.py +80 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/wef_item.py +89 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/__init__.py +9 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/base_parser.py +25 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/client_meta_parser.py +117 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/connection_rate_parser.py +24 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/info_parser.py +134 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/log_rate_parser.py +21 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/slow_chart_parser.py +21 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/slow_rate_parser.py +25 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/state_trace_parser.py +21 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/top_slow_parser.py +76 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/wef_parser.py +69 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/plugin.py +133 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/query_analyzer.py +218 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/rules/__init__.py +9 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/rules/base_rule.py +38 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/rules/driver_compatibility_rule.py +182 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/full.html +1 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/full.raw.html +30 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/script.js +1 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/script.raw.js +55 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/ClientMetaParser_1.js +1 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/ClientMetaParser_1.raw.js +30 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/ClientMetaParser_2.js +1 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/ClientMetaParser_2.raw.js +30 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/ConnectionRateParser_0.js +1 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/ConnectionRateParser_0.raw.js +97 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/ConnectionRateParser_1.js +1 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/ConnectionRateParser_1.raw.js +84 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/LogRateParser_0.js +1 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/LogRateParser_0.raw.js +80 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/SlowChartParser_0.js +1 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/SlowChartParser_0.raw.js +399 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/SlowRateParser_0.js +1 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/SlowRateParser_0.raw.js +86 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/SlowRateParser_1.js +1 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/SlowRateParser_1.raw.js +34 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/SlowRateParser_2.js +1 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/SlowRateParser_2.raw.js +34 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/StateTraceParser_0.js +1 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/StateTraceParser_0.raw.js +209 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/TopSlowParser_2.js +1 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/TopSlowParser_2.raw.js +40 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/WEFParser_2.js +1 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/WEFParser_2.raw.js +47 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/style.css +1 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/style.raw.css +35 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/minify.sh +51 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log.egg-info/PKG-INFO +84 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log.egg-info/SOURCES.txt +81 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log.egg-info/dependency_links.txt +1 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log.egg-info/entry_points.txt +2 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log.egg-info/requires.txt +2 -0
- mongo_x_ray_log-2.0.0/src/mongo_x_ray_log.egg-info/top_level.txt +1 -0
- mongo_x_ray_log-2.0.0/tests/test_client_metadata.py +153 -0
- mongo_x_ray_log-2.0.0/tests/test_connection_rate.py +56 -0
- mongo_x_ray_log-2.0.0/tests/test_info.py +64 -0
- mongo_x_ray_log-2.0.0/tests/test_pdf_output.py +47 -0
- mongo_x_ray_log-2.0.0/tests/test_query_analyser.py +135 -0
- mongo_x_ray_log-2.0.0/tests/test_report_ui.py +196 -0
- mongo_x_ray_log-2.0.0/tests/test_shared.py +29 -0
- mongo_x_ray_log-2.0.0/tests/test_slow_chart.py +46 -0
- mongo_x_ray_log-2.0.0/tests/test_slow_rate.py +65 -0
- mongo_x_ray_log-2.0.0/tests/test_state_trace.py +69 -0
- mongo_x_ray_log-2.0.0/tests/test_top_slow.py +53 -0
- mongo_x_ray_log-2.0.0/tests/test_wef.py +81 -0
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mongo-x-ray-log
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: MongoDB log analysis plugin for x-ray
|
|
5
|
+
Requires-Python: <4,>=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: mongo-x-ray>=2.0.0
|
|
8
|
+
Requires-Dist: pymongo>=4.0
|
|
9
|
+
|
|
10
|
+
# mongo-x-ray-log
|
|
11
|
+
|
|
12
|
+
[](https://github.com/zhangyaoxing/mongo-x-ray-log/actions/workflows/ci.yml)
|
|
13
|
+
|
|
14
|
+
MongoDB log analysis plugin for [x-ray](https://github.com/mongodb-ps/ce-mongo-x-ray).
|
|
15
|
+
|
|
16
|
+
## Install
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
pip install mongo-x-ray mongo-x-ray-log
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## Usage
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
x-ray log /var/log/mongodb/mongod.log
|
|
26
|
+
x-ray log /var/log/mongodb/ 2026-07-20T08:00:00Z 2026-07-20T10:00:00Z
|
|
27
|
+
x-ray log /path/to/mongod.log -f html -o /path/to/output/
|
|
28
|
+
# Analyze a random 10% of a large log
|
|
29
|
+
x-ray log -r 0.1 mongodb.log
|
|
30
|
+
# Discover log folders recursively
|
|
31
|
+
x-ray log --discover /var/log/
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Compatibility
|
|
35
|
+
|
|
36
|
+
Supports MongoDB 5.0 and above on all topologies:
|
|
37
|
+
|
|
38
|
+
| Replica Set | Sharded Cluster | Standalone |
|
|
39
|
+
| :---------: | :-------------: | :--------: |
|
|
40
|
+
| ✅ | ✅ | ✅ |
|
|
41
|
+
|
|
42
|
+
## Parameters
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
x-ray log [-h] [-s CHECKSET] [-o OUTPUT] [-f {markdown,html,pdf}] [--no-browser]
|
|
46
|
+
[-r RATE] [--top TOP] [--discover] log_file [start_time] [end_time]
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
| Argument | Description | Default |
|
|
50
|
+
| --- | --- | --- |
|
|
51
|
+
| `log_file` | Path to the MongoDB log file, or a folder of log files to analyze. | required |
|
|
52
|
+
| `start_time` | Inclusive UTC start time in ISO-8601 format. | first log line |
|
|
53
|
+
| `end_time` | Inclusive UTC end time in ISO-8601 format. | last log line |
|
|
54
|
+
| `-s, --checkset` | Checkset to run. | `default` |
|
|
55
|
+
| `-o, --output` | Output folder path. | `output/` |
|
|
56
|
+
| `-f, --format` | Output format: `markdown`, `html` or `pdf` (PDF also keeps Markdown and HTML). | `html` |
|
|
57
|
+
| `--no-browser` | Do not open the generated report in the browser. | `false` |
|
|
58
|
+
| `-r, --rate` | Log sampling rate, e.g. `1` for all logs, `0.1` for 10% of logs. | `1` |
|
|
59
|
+
| `--top` | Top N slow queries to list. | `10` |
|
|
60
|
+
| `--discover` | Recursively search the given path for folders containing log files. | `false` |
|
|
61
|
+
|
|
62
|
+
## Analysis Items
|
|
63
|
+
|
|
64
|
+
| Item | Purpose |
|
|
65
|
+
| --- | --- |
|
|
66
|
+
| `InfoItem` | Basic information about the MongoDB instance. |
|
|
67
|
+
| `ClientMetaItem` | Visualize client metadata (application, driver, OS, client IPs). |
|
|
68
|
+
| `ConnectionRateItem` | Analyze the rate of connections created and ended over a time window. |
|
|
69
|
+
| `LogRateItem` | Show the rate at which different log messages (grouped by log ID) appear over time. |
|
|
70
|
+
| `SlowRateItem` | Analyze the rate of slow queries. |
|
|
71
|
+
| `SlowChartItem` | Scatter plot of slow operations over time, each point colored by namespace. |
|
|
72
|
+
| `TopSlowItem` | Identify the top N slowest operations from the log entries. |
|
|
73
|
+
| `StateTraceItem` | Visualize replica set member state changes over time. |
|
|
74
|
+
| `WEFItem` | Visualize warning, error and fatal log messages. |
|
|
75
|
+
|
|
76
|
+
## Development
|
|
77
|
+
|
|
78
|
+
Requires Python 3.10+, MongoDB 5.0 or later, and the [mongo-x-ray](https://github.com/mongodb-ps/ce-mongo-x-ray) core package.
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
make unit-test # run the unit tests
|
|
82
|
+
make lint # ruff check + ruff format --check
|
|
83
|
+
make minify # minify templates
|
|
84
|
+
```
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# mongo-x-ray-log
|
|
2
|
+
|
|
3
|
+
[](https://github.com/zhangyaoxing/mongo-x-ray-log/actions/workflows/ci.yml)
|
|
4
|
+
|
|
5
|
+
MongoDB log analysis plugin for [x-ray](https://github.com/mongodb-ps/ce-mongo-x-ray).
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install mongo-x-ray mongo-x-ray-log
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Usage
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
x-ray log /var/log/mongodb/mongod.log
|
|
17
|
+
x-ray log /var/log/mongodb/ 2026-07-20T08:00:00Z 2026-07-20T10:00:00Z
|
|
18
|
+
x-ray log /path/to/mongod.log -f html -o /path/to/output/
|
|
19
|
+
# Analyze a random 10% of a large log
|
|
20
|
+
x-ray log -r 0.1 mongodb.log
|
|
21
|
+
# Discover log folders recursively
|
|
22
|
+
x-ray log --discover /var/log/
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Compatibility
|
|
26
|
+
|
|
27
|
+
Supports MongoDB 5.0 and above on all topologies:
|
|
28
|
+
|
|
29
|
+
| Replica Set | Sharded Cluster | Standalone |
|
|
30
|
+
| :---------: | :-------------: | :--------: |
|
|
31
|
+
| ✅ | ✅ | ✅ |
|
|
32
|
+
|
|
33
|
+
## Parameters
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
x-ray log [-h] [-s CHECKSET] [-o OUTPUT] [-f {markdown,html,pdf}] [--no-browser]
|
|
37
|
+
[-r RATE] [--top TOP] [--discover] log_file [start_time] [end_time]
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
| Argument | Description | Default |
|
|
41
|
+
| --- | --- | --- |
|
|
42
|
+
| `log_file` | Path to the MongoDB log file, or a folder of log files to analyze. | required |
|
|
43
|
+
| `start_time` | Inclusive UTC start time in ISO-8601 format. | first log line |
|
|
44
|
+
| `end_time` | Inclusive UTC end time in ISO-8601 format. | last log line |
|
|
45
|
+
| `-s, --checkset` | Checkset to run. | `default` |
|
|
46
|
+
| `-o, --output` | Output folder path. | `output/` |
|
|
47
|
+
| `-f, --format` | Output format: `markdown`, `html` or `pdf` (PDF also keeps Markdown and HTML). | `html` |
|
|
48
|
+
| `--no-browser` | Do not open the generated report in the browser. | `false` |
|
|
49
|
+
| `-r, --rate` | Log sampling rate, e.g. `1` for all logs, `0.1` for 10% of logs. | `1` |
|
|
50
|
+
| `--top` | Top N slow queries to list. | `10` |
|
|
51
|
+
| `--discover` | Recursively search the given path for folders containing log files. | `false` |
|
|
52
|
+
|
|
53
|
+
## Analysis Items
|
|
54
|
+
|
|
55
|
+
| Item | Purpose |
|
|
56
|
+
| --- | --- |
|
|
57
|
+
| `InfoItem` | Basic information about the MongoDB instance. |
|
|
58
|
+
| `ClientMetaItem` | Visualize client metadata (application, driver, OS, client IPs). |
|
|
59
|
+
| `ConnectionRateItem` | Analyze the rate of connections created and ended over a time window. |
|
|
60
|
+
| `LogRateItem` | Show the rate at which different log messages (grouped by log ID) appear over time. |
|
|
61
|
+
| `SlowRateItem` | Analyze the rate of slow queries. |
|
|
62
|
+
| `SlowChartItem` | Scatter plot of slow operations over time, each point colored by namespace. |
|
|
63
|
+
| `TopSlowItem` | Identify the top N slowest operations from the log entries. |
|
|
64
|
+
| `StateTraceItem` | Visualize replica set member state changes over time. |
|
|
65
|
+
| `WEFItem` | Visualize warning, error and fatal log messages. |
|
|
66
|
+
|
|
67
|
+
## Development
|
|
68
|
+
|
|
69
|
+
Requires Python 3.10+, MongoDB 5.0 or later, and the [mongo-x-ray](https://github.com/mongodb-ps/ce-mongo-x-ray) core package.
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
make unit-test # run the unit tests
|
|
73
|
+
make lint # ruff check + ruff format --check
|
|
74
|
+
make minify # minify templates
|
|
75
|
+
```
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools==83.0.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "mongo-x-ray-log"
|
|
7
|
+
version = "2.0.0"
|
|
8
|
+
description = "MongoDB log analysis plugin for x-ray"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10,<4"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"mongo-x-ray>=2.0.0",
|
|
13
|
+
"pymongo>=4.0",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
[project.entry-points."mongo_x_ray.plugins"]
|
|
17
|
+
log = "mongo_x_ray_log.plugin:LogPlugin"
|
|
18
|
+
|
|
19
|
+
[tool.setuptools.packages.find]
|
|
20
|
+
where = ["src"]
|
|
21
|
+
include = ["mongo_x_ray_log*"]
|
|
22
|
+
|
|
23
|
+
[tool.setuptools.package-data]
|
|
24
|
+
"mongo_x_ray_log" = ["templates/**/*"]
|
|
25
|
+
|
|
26
|
+
[tool.pytest.ini_options]
|
|
27
|
+
testpaths = ["tests"]
|
|
28
|
+
pythonpath = ["src"]
|
|
29
|
+
python_files = ["test_*.py"]
|
|
30
|
+
python_classes = ["Test*"]
|
|
31
|
+
python_functions = ["test_*"]
|
|
32
|
+
markers = [
|
|
33
|
+
"slow: marks tests as slow",
|
|
34
|
+
"integration: marks tests as integration tests",
|
|
35
|
+
"unit: marks tests as unit tests",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
[tool.ruff]
|
|
40
|
+
line-length = 120
|
|
41
|
+
target-version = "py310"
|
|
42
|
+
exclude = [".venv", "build", "dist"]
|
|
43
|
+
|
|
44
|
+
[tool.ruff.lint]
|
|
45
|
+
select = ["E", "F", "I"]
|
|
46
|
+
ignore = ["E501"]
|
|
47
|
+
|
|
48
|
+
[tool.ruff.lint.isort]
|
|
49
|
+
known-first-party = ["mongo_x_ray", "mongo_x_ray_ftdc", "mongo_x_ray_gmd", "mongo_x_ray_hc", "mongo_x_ray_log", "mongo_x_ray_risk"]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
[tool.pyright]
|
|
53
|
+
pythonVersion = "3.10"
|
|
54
|
+
venvPath = "../ce-mongo-x-ray"
|
|
55
|
+
venv = ".venv"
|
|
56
|
+
typeCheckingMode = "basic"
|
|
57
|
+
extraPaths = [
|
|
58
|
+
"src",
|
|
59
|
+
"../ce-mongo-x-ray/src",
|
|
60
|
+
]
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Copyright (c) 2026 MongoDB Inc.
|
|
3
|
+
|
|
4
|
+
DISCLAIMER: THESE CODE SAMPLES ARE PROVIDED FOR EDUCATIONAL AND ILLUSTRATIVE PURPOSES ONLY,
|
|
5
|
+
TO DEMONSTRATE THE FUNCTIONALITY OF SPECIFIC MONGODB FEATURES.
|
|
6
|
+
THEY ARE NOT PRODUCTION-READY AND MAY LACK THE SECURITY HARDENING, ERROR HANDLING, AND TESTING REQUIRED FOR A LIVE ENVIRONMENT.
|
|
7
|
+
YOU ARE RESPONSIBLE FOR TESTING, VALIDATING, AND SECURING THIS CODE WITHIN YOUR OWN ENVIRONMENT BEFORE IMPLEMENTATION.
|
|
8
|
+
THIS MATERIAL IS PROVIDED "AS IS" WITHOUT WARRANTY OR LIABILITY.
|
|
9
|
+
|
|
10
|
+
MongoDB log analysis package.
|
|
11
|
+
"""
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Copyright (c) 2026 MongoDB Inc.
|
|
3
|
+
|
|
4
|
+
DISCLAIMER: THESE CODE SAMPLES ARE PROVIDED FOR EDUCATIONAL AND ILLUSTRATIVE PURPOSES ONLY,
|
|
5
|
+
TO DEMONSTRATE THE FUNCTIONALITY OF SPECIFIC MONGODB FEATURES.
|
|
6
|
+
THEY ARE NOT PRODUCTION-READY AND MAY LACK THE SECURITY HARDENING, ERROR HANDLING, AND TESTING REQUIRED FOR A LIVE ENVIRONMENT.
|
|
7
|
+
YOU ARE RESPONSIBLE FOR TESTING, VALIDATING, AND SECURING THIS CODE WITHIN YOUR OWN ENVIRONMENT BEFORE IMPLEMENTATION.
|
|
8
|
+
THIS MATERIAL IS PROVIDED "AS IS" WITHOUT WARRANTY OR LIABILITY.
|
|
9
|
+
|
|
10
|
+
AI analysis for the log plugin (warning / error / fatal log lines).
|
|
11
|
+
|
|
12
|
+
Uses the shared client in :mod:`mongo_x_ray.ai_client`.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from mongo_x_ray.ai_client import complete
|
|
18
|
+
|
|
19
|
+
_SYSTEM_PROMPT = "You are a MongoDB expert. Analyze MongoDB log messages and tell me the reason in max 200 words."
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def analyze_log_line_gpt(log_line: dict) -> str:
|
|
23
|
+
"""Analyze a MongoDB log line using the shared AI client."""
|
|
24
|
+
result = complete(str(log_line), system=_SYSTEM_PROMPT)
|
|
25
|
+
return (result or "").strip()
|
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Copyright (c) 2026 MongoDB Inc.
|
|
3
|
+
|
|
4
|
+
DISCLAIMER: THESE CODE SAMPLES ARE PROVIDED FOR EDUCATIONAL AND ILLUSTRATIVE PURPOSES ONLY,
|
|
5
|
+
TO DEMONSTRATE THE FUNCTIONALITY OF SPECIFIC MONGODB FEATURES.
|
|
6
|
+
THEY ARE NOT PRODUCTION-READY AND MAY LACK THE SECURITY HARDENING, ERROR HANDLING, AND TESTING REQUIRED FOR A LIVE ENVIRONMENT.
|
|
7
|
+
YOU ARE RESPONSIBLE FOR TESTING, VALIDATING, AND SECURING THIS CODE WITHIN YOUR OWN ENVIRONMENT BEFORE IMPLEMENTATION.
|
|
8
|
+
THIS MATERIAL IS PROVIDED "AS IS" WITHOUT WARRANTY OR LIABILITY.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import logging
|
|
12
|
+
import random
|
|
13
|
+
import re
|
|
14
|
+
from datetime import datetime, timezone
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Optional, TextIO
|
|
17
|
+
|
|
18
|
+
from bson import json_util
|
|
19
|
+
|
|
20
|
+
from mongo_x_ray.framework import BaseFramework
|
|
21
|
+
from mongo_x_ray.shared import str_to_md_id, to_json
|
|
22
|
+
from mongo_x_ray.utils import bold, cyan, env, green, load_classes, yellow
|
|
23
|
+
from mongo_x_ray_log.log_items.info_item import InfoItem
|
|
24
|
+
from mongo_x_ray_log.log_items.state_trace_item import StateTraceItem
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger(__name__)
|
|
27
|
+
LOG_CLASSES = load_classes("mongo_x_ray_log.log_items")
|
|
28
|
+
SKIP_LINE_MSG = "HEADER INCLUDED, NOW SKIPPING 64728 LINES ACCORDING TO REQUESTED SIZE LIMIT"
|
|
29
|
+
_SANITIZE_DATE_RE = re.compile(r'\{\s*"\$date"\s*:\s*\{\s*"\$numberLong"\s*:\s*"-?\d{16,}"\s*\}\s*\}')
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _sanitize_date_numberlong(line: str) -> str:
|
|
33
|
+
"""Replace out-of-range $date.$numberLong sentinel values with null."""
|
|
34
|
+
return _SANITIZE_DATE_RE.sub("null", line)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _safe_json_loads(line: str) -> dict:
|
|
38
|
+
"""Parse a JSON log line, sanitising out-of-range dates on failure only.
|
|
39
|
+
Also ensures all datetime values are timezone-aware (UTC).
|
|
40
|
+
"""
|
|
41
|
+
try:
|
|
42
|
+
parsed = json_util.loads(line)
|
|
43
|
+
_normalise_datetimes(parsed)
|
|
44
|
+
return parsed
|
|
45
|
+
except Exception as exc:
|
|
46
|
+
try:
|
|
47
|
+
parsed = json_util.loads(_sanitize_date_numberlong(line))
|
|
48
|
+
_normalise_datetimes(parsed)
|
|
49
|
+
return parsed
|
|
50
|
+
except Exception:
|
|
51
|
+
logger.debug("JSON parse failed (first error: %s): %s", exc, line.strip()[:200])
|
|
52
|
+
return {}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _normalise_datetimes(obj: dict) -> None:
|
|
56
|
+
"""Ensure all datetime values in *obj* are UTC-aware (in place)."""
|
|
57
|
+
for key, value in obj.items():
|
|
58
|
+
if isinstance(value, datetime) and value.tzinfo is None:
|
|
59
|
+
obj[key] = value.replace(tzinfo=timezone.utc)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class Framework(BaseFramework):
|
|
63
|
+
template_module = "log"
|
|
64
|
+
template_package = "mongo_x_ray_log"
|
|
65
|
+
|
|
66
|
+
def __init__(
|
|
67
|
+
self,
|
|
68
|
+
file_path: str,
|
|
69
|
+
config: dict,
|
|
70
|
+
start_time: Optional[datetime] = None,
|
|
71
|
+
end_time: Optional[datetime] = None,
|
|
72
|
+
):
|
|
73
|
+
super().__init__(config)
|
|
74
|
+
self._file_path = file_path
|
|
75
|
+
self._start_time = start_time
|
|
76
|
+
self._end_time = end_time
|
|
77
|
+
self._logger.debug(to_json(self._config))
|
|
78
|
+
self._log_start: Optional[datetime] = None
|
|
79
|
+
self._log_end: Optional[datetime] = None
|
|
80
|
+
self._hostname: Optional[str] = None
|
|
81
|
+
if env == "development":
|
|
82
|
+
self._logger.info(yellow("Running in development mode."))
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def hostname(self) -> Optional[str]:
|
|
86
|
+
"""The hostname from the Process Info log item, or from log lines."""
|
|
87
|
+
for item in self._items:
|
|
88
|
+
if isinstance(item, InfoItem):
|
|
89
|
+
host = item._cache.get("process", {}).get("host")
|
|
90
|
+
if host and host != "Unknown":
|
|
91
|
+
return host
|
|
92
|
+
break
|
|
93
|
+
return self._hostname
|
|
94
|
+
|
|
95
|
+
def _log_files(self) -> list[Path]:
|
|
96
|
+
"""Return a sorted list of log files to process."""
|
|
97
|
+
path = Path(self._file_path)
|
|
98
|
+
if path.is_file():
|
|
99
|
+
return [path]
|
|
100
|
+
# Match mongod.log, mongod.log.2026-06-10T01-58-56, etc.
|
|
101
|
+
files = sorted(path.glob("*.log*"))
|
|
102
|
+
if not files:
|
|
103
|
+
files = sorted(path.glob("*"))
|
|
104
|
+
return files
|
|
105
|
+
|
|
106
|
+
@staticmethod
|
|
107
|
+
def _file_time_range(file_path: Path) -> tuple:
|
|
108
|
+
"""Read the first and last valid JSON log line to get the file's time range."""
|
|
109
|
+
first_ts = None
|
|
110
|
+
last_ts = None
|
|
111
|
+
try:
|
|
112
|
+
with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
|
|
113
|
+
# Skip preamble lines (e.g. Atlas download header)
|
|
114
|
+
for line in f:
|
|
115
|
+
parsed = _safe_json_loads(line)
|
|
116
|
+
if parsed:
|
|
117
|
+
first_ts = parsed.get("t")
|
|
118
|
+
break
|
|
119
|
+
# Scan backwards from end for the last non-empty line
|
|
120
|
+
f.seek(0, 2)
|
|
121
|
+
pos = f.tell()
|
|
122
|
+
last_line = ""
|
|
123
|
+
while pos > 0:
|
|
124
|
+
pos -= 1
|
|
125
|
+
f.seek(pos)
|
|
126
|
+
if f.read(1) == "\n":
|
|
127
|
+
candidate = f.readline().strip()
|
|
128
|
+
if candidate:
|
|
129
|
+
last_line = candidate
|
|
130
|
+
break
|
|
131
|
+
if last_line:
|
|
132
|
+
last_ts = _safe_json_loads(last_line).get("t")
|
|
133
|
+
except Exception as exc:
|
|
134
|
+
logger.warning("Failed to read time range from %s: %s", file_path.name, exc)
|
|
135
|
+
logger.debug(
|
|
136
|
+
"File %s time range: %s – %s",
|
|
137
|
+
file_path.name,
|
|
138
|
+
first_ts.isoformat() if first_ts else "?",
|
|
139
|
+
last_ts.isoformat() if last_ts else "?",
|
|
140
|
+
)
|
|
141
|
+
return first_ts, last_ts
|
|
142
|
+
|
|
143
|
+
def _file_overlaps_range(self, file_path: Path) -> bool:
|
|
144
|
+
"""Return False if the file's time range is entirely outside the requested range."""
|
|
145
|
+
if self._start_time is None and self._end_time is None:
|
|
146
|
+
return True
|
|
147
|
+
first_ts, last_ts = self._file_time_range(file_path)
|
|
148
|
+
if first_ts is None or last_ts is None:
|
|
149
|
+
return True # can't determine, process anyway
|
|
150
|
+
if self._end_time is not None and first_ts > self._end_time:
|
|
151
|
+
return False
|
|
152
|
+
if self._start_time is not None and last_ts < self._start_time:
|
|
153
|
+
return False
|
|
154
|
+
return True
|
|
155
|
+
|
|
156
|
+
def _any_file_fully_covered(self, files: list[Path]) -> bool:
|
|
157
|
+
"""Return True if at least one file is fully within [start_time, end_time]."""
|
|
158
|
+
if self._start_time is None and self._end_time is None:
|
|
159
|
+
return True
|
|
160
|
+
for fp in files:
|
|
161
|
+
first_ts, last_ts = self._file_time_range(fp)
|
|
162
|
+
if first_ts is None or last_ts is None:
|
|
163
|
+
continue
|
|
164
|
+
if (self._start_time is None or first_ts >= self._start_time) and (
|
|
165
|
+
self._end_time is None or last_ts <= self._end_time
|
|
166
|
+
):
|
|
167
|
+
return True
|
|
168
|
+
return False
|
|
169
|
+
|
|
170
|
+
def run_logs_analysis(self, logset_name: str, *_args, **kwargs):
|
|
171
|
+
self._set_name = logset_name
|
|
172
|
+
# Create output folder if it doesn't exist
|
|
173
|
+
output_folder = kwargs.get("output_folder", "output/")
|
|
174
|
+
batch_folder = self._get_output_folder(output_folder)
|
|
175
|
+
# Dynamically load the log checkset based on the name
|
|
176
|
+
logsets = self._config.get("logsets", {})
|
|
177
|
+
if logset_name not in logsets:
|
|
178
|
+
self._logger.warning(
|
|
179
|
+
yellow(f"Log checkset '{logset_name}' not found in configuration. Using default logset.")
|
|
180
|
+
)
|
|
181
|
+
logset_name = "default"
|
|
182
|
+
ls = logsets[logset_name]
|
|
183
|
+
self._logger.info("Running log checkset: %s", bold(cyan(logset_name)))
|
|
184
|
+
|
|
185
|
+
self._items = []
|
|
186
|
+
for item_name in ls.get("items", []):
|
|
187
|
+
item_cls = LOG_CLASSES.get(item_name)
|
|
188
|
+
if not item_cls:
|
|
189
|
+
self._logger.warning(yellow(f"Log item '{item_name}' not found. Skipping."))
|
|
190
|
+
continue
|
|
191
|
+
item_config = self._config.get("item_config", {}).get(item_name, {})
|
|
192
|
+
item = item_cls(str(batch_folder), item_config)
|
|
193
|
+
self._items.append(item)
|
|
194
|
+
self._logger.info("Log analyze item loaded: %s", bold(cyan(item_name)))
|
|
195
|
+
|
|
196
|
+
rate = self._config.get("sample_rate", 1.0)
|
|
197
|
+
log_files = self._log_files()
|
|
198
|
+
partial_only = (
|
|
199
|
+
self._start_time is not None or self._end_time is not None
|
|
200
|
+
) and not self._any_file_fully_covered(log_files)
|
|
201
|
+
if partial_only:
|
|
202
|
+
self._logger.info(
|
|
203
|
+
"No log file is fully covered by the requested time range. "
|
|
204
|
+
"InfoItem and StateTraceItem will receive all lines."
|
|
205
|
+
)
|
|
206
|
+
log_line: dict = {}
|
|
207
|
+
global_counter: int = 0
|
|
208
|
+
|
|
209
|
+
for lf in log_files:
|
|
210
|
+
if not self._file_overlaps_range(lf):
|
|
211
|
+
self._logger.info(
|
|
212
|
+
"Skipping %s (outside time range %s – %s)",
|
|
213
|
+
lf.name,
|
|
214
|
+
self._start_time.isoformat() if self._start_time else "…",
|
|
215
|
+
self._end_time.isoformat() if self._end_time else "…",
|
|
216
|
+
)
|
|
217
|
+
continue
|
|
218
|
+
self._logger.info("Processing %s", green(str(lf)))
|
|
219
|
+
|
|
220
|
+
with open(lf, "r", encoding="utf-8", errors="ignore") as f:
|
|
221
|
+
counter: int = 0
|
|
222
|
+
for line in f:
|
|
223
|
+
counter += 1
|
|
224
|
+
global_counter += 1
|
|
225
|
+
if global_counter % 10000 == 0:
|
|
226
|
+
self._logger.info("%s lines ingested...", green(str(global_counter)))
|
|
227
|
+
if random.random() > rate:
|
|
228
|
+
continue
|
|
229
|
+
try:
|
|
230
|
+
if counter == 101 and line.startswith(SKIP_LINE_MSG):
|
|
231
|
+
self._logger.debug("Some lines are skipped due to the size limit. This is expected.")
|
|
232
|
+
continue
|
|
233
|
+
log_line = _safe_json_loads(line)
|
|
234
|
+
if not log_line:
|
|
235
|
+
self._logger.warning(yellow(f"Failed to parse log line as JSON: {line.strip()}"))
|
|
236
|
+
continue
|
|
237
|
+
if self._hostname is None:
|
|
238
|
+
hostname = log_line.get("hostname")
|
|
239
|
+
if isinstance(hostname, str) and hostname.strip():
|
|
240
|
+
self._hostname = hostname.strip()
|
|
241
|
+
line_ts = log_line.get("t")
|
|
242
|
+
out_of_range = False
|
|
243
|
+
if line_ts is not None:
|
|
244
|
+
if self._start_time is not None and line_ts < self._start_time:
|
|
245
|
+
out_of_range = True
|
|
246
|
+
elif self._end_time is not None and line_ts > self._end_time:
|
|
247
|
+
out_of_range = True
|
|
248
|
+
|
|
249
|
+
if out_of_range and not partial_only:
|
|
250
|
+
continue
|
|
251
|
+
|
|
252
|
+
if self._log_start is None:
|
|
253
|
+
self._log_start = line_ts
|
|
254
|
+
|
|
255
|
+
if out_of_range:
|
|
256
|
+
# partial_only: dispatch only to InfoItem and StateTraceItem
|
|
257
|
+
for item in self._items:
|
|
258
|
+
if isinstance(item, (InfoItem, StateTraceItem)):
|
|
259
|
+
try:
|
|
260
|
+
item.analyze(log_line)
|
|
261
|
+
except Exception as e:
|
|
262
|
+
self._logger.warning(yellow(f"Log analysis item '{item.name}' failed: {e}"))
|
|
263
|
+
continue
|
|
264
|
+
|
|
265
|
+
for item in self._items:
|
|
266
|
+
try:
|
|
267
|
+
item.analyze(log_line)
|
|
268
|
+
except Exception as e:
|
|
269
|
+
self._logger.warning(yellow(f"Log analysis item '{item.name}' failed: {e}"))
|
|
270
|
+
continue
|
|
271
|
+
except Exception as exc:
|
|
272
|
+
self._logger.warning(yellow(f"Unexpected error processing log line: {exc}"))
|
|
273
|
+
continue
|
|
274
|
+
|
|
275
|
+
self._log_end = log_line.get("t", None) if log_line else None
|
|
276
|
+
for item in self._items:
|
|
277
|
+
item._hostname = self._hostname
|
|
278
|
+
try:
|
|
279
|
+
item.finalize_analysis()
|
|
280
|
+
except Exception as e:
|
|
281
|
+
self._logger.warning(yellow(f"Log analysis item '{item.name}' finalize failed: {e}"))
|
|
282
|
+
continue
|
|
283
|
+
|
|
284
|
+
def _render_markdown(self, output: TextIO) -> None:
|
|
285
|
+
assert self._log_start is not None and self._log_end is not None, (
|
|
286
|
+
"Log start and end time should be set after analysis."
|
|
287
|
+
)
|
|
288
|
+
output.write("# Log Analysis Report\n")
|
|
289
|
+
output.write(f"Generated at: `{str(datetime.now(tz=timezone.utc))} UTC`\n\n")
|
|
290
|
+
output.write(f"Log path: `{self._file_path}`\n\n")
|
|
291
|
+
if self._start_time or self._end_time:
|
|
292
|
+
start_str = self._start_time.isoformat() if self._start_time else "…"
|
|
293
|
+
end_str = self._end_time.isoformat() if self._end_time else "…"
|
|
294
|
+
output.write(f"Requested time range: `{start_str}` – `{end_str}`\n\n")
|
|
295
|
+
output.write(f"Log analysis period: `{self._log_start.isoformat()}` to `{self._log_end.isoformat()}`\n\n")
|
|
296
|
+
output.write("Histogram chart instructions:\n\n")
|
|
297
|
+
output.write("- **zoom in/out:** _ctrl+wheel, or pinch_\n")
|
|
298
|
+
output.write("- **pan:** _shift+drag_\n")
|
|
299
|
+
output.write("- **select time frame:** _drag_\n\n")
|
|
300
|
+
|
|
301
|
+
output.write("## 1 Review Test Results\n\n")
|
|
302
|
+
for i, item in enumerate(self._items):
|
|
303
|
+
title = f"1.{i + 1} {item.name}"
|
|
304
|
+
review_title = f"2.{i + 1} Review {item.name}"
|
|
305
|
+
review_title_id = str_to_md_id(review_title)
|
|
306
|
+
output.write(f"### {title}\n\n")
|
|
307
|
+
output.write(f"{item.description}\n\n")
|
|
308
|
+
output.write(f"[Review Raw Results →](#{review_title_id})\n\n")
|
|
309
|
+
try:
|
|
310
|
+
item.test_result_markdown(output)
|
|
311
|
+
except Exception as e:
|
|
312
|
+
self._logger.warning(yellow(f"Failed to generate test results for log item '{item.name}': {e}"))
|
|
313
|
+
continue
|
|
314
|
+
|
|
315
|
+
output.write("## 2 Review Raw Results\n\n")
|
|
316
|
+
for i, item in enumerate(self._items):
|
|
317
|
+
title = f"1.{i + 1} {item.name}"
|
|
318
|
+
title_id = str_to_md_id(title)
|
|
319
|
+
review_title = f"2.{i + 1} Review {item.name}"
|
|
320
|
+
output.write(f"### {review_title}\n\n")
|
|
321
|
+
output.write(f"[← Review Test Results](#{title_id})\n\n")
|
|
322
|
+
if getattr(item, "_show_reset", False):
|
|
323
|
+
output.write(
|
|
324
|
+
f'<input type="button" id="reset_{item.__class__.__name__}" class="table-copy-button" value="Reset">\n\n'
|
|
325
|
+
)
|
|
326
|
+
try:
|
|
327
|
+
item.review_results_markdown(output)
|
|
328
|
+
except Exception as e:
|
|
329
|
+
self._logger.warning(yellow(f"Failed to generate markdown for log item '{item.name}': {e}"))
|
|
330
|
+
continue
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Copyright (c) 2026 MongoDB Inc.
|
|
3
|
+
|
|
4
|
+
DISCLAIMER: THESE CODE SAMPLES ARE PROVIDED FOR EDUCATIONAL AND ILLUSTRATIVE PURPOSES ONLY,
|
|
5
|
+
TO DEMONSTRATE THE FUNCTIONALITY OF SPECIFIC MONGODB FEATURES.
|
|
6
|
+
THEY ARE NOT PRODUCTION-READY AND MAY LACK THE SECURITY HARDENING, ERROR HANDLING, AND TESTING REQUIRED FOR A LIVE ENVIRONMENT.
|
|
7
|
+
YOU ARE RESPONSIBLE FOR TESTING, VALIDATING, AND SECURING THIS CODE WITHIN YOUR OWN ENVIRONMENT BEFORE IMPLEMENTATION.
|
|
8
|
+
THIS MATERIAL IS PROVIDED "AS IS" WITHOUT WARRANTY OR LIABILITY.
|
|
9
|
+
"""
|