mongo-x-ray-log 2.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. mongo_x_ray_log-2.0.0/PKG-INFO +84 -0
  2. mongo_x_ray_log-2.0.0/README.md +75 -0
  3. mongo_x_ray_log-2.0.0/pyproject.toml +60 -0
  4. mongo_x_ray_log-2.0.0/setup.cfg +4 -0
  5. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/__init__.py +11 -0
  6. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/ai.py +25 -0
  7. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/framework.py +330 -0
  8. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/__init__.py +9 -0
  9. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/base_item.py +169 -0
  10. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/client_meta_item.py +71 -0
  11. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/connection_rate_item.py +61 -0
  12. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/info_item.py +97 -0
  13. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/log_rate_item.py +67 -0
  14. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/slow_chart_item.py +35 -0
  15. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/slow_rate_item.py +54 -0
  16. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/state_trace_item.py +171 -0
  17. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/top_slow_item.py +80 -0
  18. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/log_items/wef_item.py +89 -0
  19. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/__init__.py +9 -0
  20. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/base_parser.py +25 -0
  21. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/client_meta_parser.py +117 -0
  22. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/connection_rate_parser.py +24 -0
  23. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/info_parser.py +134 -0
  24. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/log_rate_parser.py +21 -0
  25. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/slow_chart_parser.py +21 -0
  26. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/slow_rate_parser.py +25 -0
  27. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/state_trace_parser.py +21 -0
  28. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/top_slow_parser.py +76 -0
  29. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/parsers/wef_parser.py +69 -0
  30. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/plugin.py +133 -0
  31. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/query_analyzer.py +218 -0
  32. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/rules/__init__.py +9 -0
  33. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/rules/base_rule.py +38 -0
  34. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/rules/driver_compatibility_rule.py +182 -0
  35. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/full.html +1 -0
  36. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/full.raw.html +30 -0
  37. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/script.js +1 -0
  38. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/script.raw.js +55 -0
  39. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/ClientMetaParser_1.js +1 -0
  40. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/ClientMetaParser_1.raw.js +30 -0
  41. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/ClientMetaParser_2.js +1 -0
  42. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/ClientMetaParser_2.raw.js +30 -0
  43. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/ConnectionRateParser_0.js +1 -0
  44. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/ConnectionRateParser_0.raw.js +97 -0
  45. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/ConnectionRateParser_1.js +1 -0
  46. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/ConnectionRateParser_1.raw.js +84 -0
  47. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/LogRateParser_0.js +1 -0
  48. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/LogRateParser_0.raw.js +80 -0
  49. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/SlowChartParser_0.js +1 -0
  50. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/SlowChartParser_0.raw.js +399 -0
  51. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/SlowRateParser_0.js +1 -0
  52. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/SlowRateParser_0.raw.js +86 -0
  53. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/SlowRateParser_1.js +1 -0
  54. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/SlowRateParser_1.raw.js +34 -0
  55. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/SlowRateParser_2.js +1 -0
  56. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/SlowRateParser_2.raw.js +34 -0
  57. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/StateTraceParser_0.js +1 -0
  58. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/StateTraceParser_0.raw.js +209 -0
  59. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/TopSlowParser_2.js +1 -0
  60. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/TopSlowParser_2.raw.js +40 -0
  61. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/WEFParser_2.js +1 -0
  62. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/snippets/WEFParser_2.raw.js +47 -0
  63. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/style.css +1 -0
  64. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/log/style.raw.css +35 -0
  65. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log/templates/minify.sh +51 -0
  66. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log.egg-info/PKG-INFO +84 -0
  67. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log.egg-info/SOURCES.txt +81 -0
  68. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log.egg-info/dependency_links.txt +1 -0
  69. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log.egg-info/entry_points.txt +2 -0
  70. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log.egg-info/requires.txt +2 -0
  71. mongo_x_ray_log-2.0.0/src/mongo_x_ray_log.egg-info/top_level.txt +1 -0
  72. mongo_x_ray_log-2.0.0/tests/test_client_metadata.py +153 -0
  73. mongo_x_ray_log-2.0.0/tests/test_connection_rate.py +56 -0
  74. mongo_x_ray_log-2.0.0/tests/test_info.py +64 -0
  75. mongo_x_ray_log-2.0.0/tests/test_pdf_output.py +47 -0
  76. mongo_x_ray_log-2.0.0/tests/test_query_analyser.py +135 -0
  77. mongo_x_ray_log-2.0.0/tests/test_report_ui.py +196 -0
  78. mongo_x_ray_log-2.0.0/tests/test_shared.py +29 -0
  79. mongo_x_ray_log-2.0.0/tests/test_slow_chart.py +46 -0
  80. mongo_x_ray_log-2.0.0/tests/test_slow_rate.py +65 -0
  81. mongo_x_ray_log-2.0.0/tests/test_state_trace.py +69 -0
  82. mongo_x_ray_log-2.0.0/tests/test_top_slow.py +53 -0
  83. mongo_x_ray_log-2.0.0/tests/test_wef.py +81 -0
@@ -0,0 +1,84 @@
1
+ Metadata-Version: 2.4
2
+ Name: mongo-x-ray-log
3
+ Version: 2.0.0
4
+ Summary: MongoDB log analysis plugin for x-ray
5
+ Requires-Python: <4,>=3.10
6
+ Description-Content-Type: text/markdown
7
+ Requires-Dist: mongo-x-ray>=2.0.0
8
+ Requires-Dist: pymongo>=4.0
9
+
10
+ # mongo-x-ray-log
11
+
12
+ [![CI](https://github.com/zhangyaoxing/mongo-x-ray-log/actions/workflows/ci.yml/badge.svg)](https://github.com/zhangyaoxing/mongo-x-ray-log/actions/workflows/ci.yml)
13
+
14
+ MongoDB log analysis plugin for [x-ray](https://github.com/mongodb-ps/ce-mongo-x-ray).
15
+
16
+ ## Install
17
+
18
+ ```bash
19
+ pip install mongo-x-ray mongo-x-ray-log
20
+ ```
21
+
22
+ ## Usage
23
+
24
+ ```bash
25
+ x-ray log /var/log/mongodb/mongod.log
26
+ x-ray log /var/log/mongodb/ 2026-07-20T08:00:00Z 2026-07-20T10:00:00Z
27
+ x-ray log /path/to/mongod.log -f html -o /path/to/output/
28
+ # Analyze a random 10% of a large log
29
+ x-ray log -r 0.1 mongodb.log
30
+ # Discover log folders recursively
31
+ x-ray log --discover /var/log/
32
+ ```
33
+
34
+ ## Compatibility
35
+
36
+ Supports MongoDB 5.0 and above on all topologies:
37
+
38
+ | Replica Set | Sharded Cluster | Standalone |
39
+ | :---------: | :-------------: | :--------: |
40
+ | ✅ | ✅ | ✅ |
41
+
42
+ ## Parameters
43
+
44
+ ```bash
45
+ x-ray log [-h] [-s CHECKSET] [-o OUTPUT] [-f {markdown,html,pdf}] [--no-browser]
46
+ [-r RATE] [--top TOP] [--discover] log_file [start_time] [end_time]
47
+ ```
48
+
49
+ | Argument | Description | Default |
50
+ | --- | --- | --- |
51
+ | `log_file` | Path to the MongoDB log file, or a folder of log files to analyze. | required |
52
+ | `start_time` | Inclusive UTC start time in ISO-8601 format. | first log line |
53
+ | `end_time` | Inclusive UTC end time in ISO-8601 format. | last log line |
54
+ | `-s, --checkset` | Checkset to run. | `default` |
55
+ | `-o, --output` | Output folder path. | `output/` |
56
+ | `-f, --format` | Output format: `markdown`, `html` or `pdf` (PDF also keeps Markdown and HTML). | `html` |
57
+ | `--no-browser` | Do not open the generated report in the browser. | `false` |
58
+ | `-r, --rate` | Log sampling rate, e.g. `1` for all logs, `0.1` for 10% of logs. | `1` |
59
+ | `--top` | Top N slow queries to list. | `10` |
60
+ | `--discover` | Recursively search the given path for folders containing log files. | `false` |
61
+
62
+ ## Analysis Items
63
+
64
+ | Item | Purpose |
65
+ | --- | --- |
66
+ | `InfoItem` | Basic information about the MongoDB instance. |
67
+ | `ClientMetaItem` | Visualize client metadata (application, driver, OS, client IPs). |
68
+ | `ConnectionRateItem` | Analyze the rate of connections created and ended over a time window. |
69
+ | `LogRateItem` | Show the rate at which different log messages (grouped by log ID) appear over time. |
70
+ | `SlowRateItem` | Analyze the rate of slow queries. |
71
+ | `SlowChartItem` | Scatter plot of slow operations over time, each point colored by namespace. |
72
+ | `TopSlowItem` | Identify the top N slowest operations from the log entries. |
73
+ | `StateTraceItem` | Visualize replica set member state changes over time. |
74
+ | `WEFItem` | Visualize warning, error and fatal log messages. |
75
+
76
+ ## Development
77
+
78
+ Requires Python 3.10+, MongoDB 5.0 or later, and the [mongo-x-ray](https://github.com/mongodb-ps/ce-mongo-x-ray) core package.
79
+
80
+ ```bash
81
+ make unit-test # run the unit tests
82
+ make lint # ruff check + ruff format --check
83
+ make minify # minify templates
84
+ ```
@@ -0,0 +1,75 @@
1
+ # mongo-x-ray-log
2
+
3
+ [![CI](https://github.com/zhangyaoxing/mongo-x-ray-log/actions/workflows/ci.yml/badge.svg)](https://github.com/zhangyaoxing/mongo-x-ray-log/actions/workflows/ci.yml)
4
+
5
+ MongoDB log analysis plugin for [x-ray](https://github.com/mongodb-ps/ce-mongo-x-ray).
6
+
7
+ ## Install
8
+
9
+ ```bash
10
+ pip install mongo-x-ray mongo-x-ray-log
11
+ ```
12
+
13
+ ## Usage
14
+
15
+ ```bash
16
+ x-ray log /var/log/mongodb/mongod.log
17
+ x-ray log /var/log/mongodb/ 2026-07-20T08:00:00Z 2026-07-20T10:00:00Z
18
+ x-ray log /path/to/mongod.log -f html -o /path/to/output/
19
+ # Analyze a random 10% of a large log
20
+ x-ray log -r 0.1 mongodb.log
21
+ # Discover log folders recursively
22
+ x-ray log --discover /var/log/
23
+ ```
24
+
25
+ ## Compatibility
26
+
27
+ Supports MongoDB 5.0 and above on all topologies:
28
+
29
+ | Replica Set | Sharded Cluster | Standalone |
30
+ | :---------: | :-------------: | :--------: |
31
+ | ✅ | ✅ | ✅ |
32
+
33
+ ## Parameters
34
+
35
+ ```bash
36
+ x-ray log [-h] [-s CHECKSET] [-o OUTPUT] [-f {markdown,html,pdf}] [--no-browser]
37
+ [-r RATE] [--top TOP] [--discover] log_file [start_time] [end_time]
38
+ ```
39
+
40
+ | Argument | Description | Default |
41
+ | --- | --- | --- |
42
+ | `log_file` | Path to the MongoDB log file, or a folder of log files to analyze. | required |
43
+ | `start_time` | Inclusive UTC start time in ISO-8601 format. | first log line |
44
+ | `end_time` | Inclusive UTC end time in ISO-8601 format. | last log line |
45
+ | `-s, --checkset` | Checkset to run. | `default` |
46
+ | `-o, --output` | Output folder path. | `output/` |
47
+ | `-f, --format` | Output format: `markdown`, `html` or `pdf` (PDF also keeps Markdown and HTML). | `html` |
48
+ | `--no-browser` | Do not open the generated report in the browser. | `false` |
49
+ | `-r, --rate` | Log sampling rate, e.g. `1` for all logs, `0.1` for 10% of logs. | `1` |
50
+ | `--top` | Top N slow queries to list. | `10` |
51
+ | `--discover` | Recursively search the given path for folders containing log files. | `false` |
52
+
53
+ ## Analysis Items
54
+
55
+ | Item | Purpose |
56
+ | --- | --- |
57
+ | `InfoItem` | Basic information about the MongoDB instance. |
58
+ | `ClientMetaItem` | Visualize client metadata (application, driver, OS, client IPs). |
59
+ | `ConnectionRateItem` | Analyze the rate of connections created and ended over a time window. |
60
+ | `LogRateItem` | Show the rate at which different log messages (grouped by log ID) appear over time. |
61
+ | `SlowRateItem` | Analyze the rate of slow queries. |
62
+ | `SlowChartItem` | Scatter plot of slow operations over time, each point colored by namespace. |
63
+ | `TopSlowItem` | Identify the top N slowest operations from the log entries. |
64
+ | `StateTraceItem` | Visualize replica set member state changes over time. |
65
+ | `WEFItem` | Visualize warning, error and fatal log messages. |
66
+
67
+ ## Development
68
+
69
+ Requires Python 3.10+, MongoDB 5.0 or later, and the [mongo-x-ray](https://github.com/mongodb-ps/ce-mongo-x-ray) core package.
70
+
71
+ ```bash
72
+ make unit-test # run the unit tests
73
+ make lint # ruff check + ruff format --check
74
+ make minify # minify templates
75
+ ```
@@ -0,0 +1,60 @@
1
+ [build-system]
2
+ requires = ["setuptools==83.0.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "mongo-x-ray-log"
7
+ version = "2.0.0"
8
+ description = "MongoDB log analysis plugin for x-ray"
9
+ readme = "README.md"
10
+ requires-python = ">=3.10,<4"
11
+ dependencies = [
12
+ "mongo-x-ray>=2.0.0",
13
+ "pymongo>=4.0",
14
+ ]
15
+
16
+ [project.entry-points."mongo_x_ray.plugins"]
17
+ log = "mongo_x_ray_log.plugin:LogPlugin"
18
+
19
+ [tool.setuptools.packages.find]
20
+ where = ["src"]
21
+ include = ["mongo_x_ray_log*"]
22
+
23
+ [tool.setuptools.package-data]
24
+ "mongo_x_ray_log" = ["templates/**/*"]
25
+
26
+ [tool.pytest.ini_options]
27
+ testpaths = ["tests"]
28
+ pythonpath = ["src"]
29
+ python_files = ["test_*.py"]
30
+ python_classes = ["Test*"]
31
+ python_functions = ["test_*"]
32
+ markers = [
33
+ "slow: marks tests as slow",
34
+ "integration: marks tests as integration tests",
35
+ "unit: marks tests as unit tests",
36
+ ]
37
+
38
+
39
+ [tool.ruff]
40
+ line-length = 120
41
+ target-version = "py310"
42
+ exclude = [".venv", "build", "dist"]
43
+
44
+ [tool.ruff.lint]
45
+ select = ["E", "F", "I"]
46
+ ignore = ["E501"]
47
+
48
+ [tool.ruff.lint.isort]
49
+ known-first-party = ["mongo_x_ray", "mongo_x_ray_ftdc", "mongo_x_ray_gmd", "mongo_x_ray_hc", "mongo_x_ray_log", "mongo_x_ray_risk"]
50
+
51
+
52
+ [tool.pyright]
53
+ pythonVersion = "3.10"
54
+ venvPath = "../ce-mongo-x-ray"
55
+ venv = ".venv"
56
+ typeCheckingMode = "basic"
57
+ extraPaths = [
58
+ "src",
59
+ "../ce-mongo-x-ray/src",
60
+ ]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,11 @@
1
+ """
2
+ Copyright (c) 2026 MongoDB Inc.
3
+
4
+ DISCLAIMER: THESE CODE SAMPLES ARE PROVIDED FOR EDUCATIONAL AND ILLUSTRATIVE PURPOSES ONLY,
5
+ TO DEMONSTRATE THE FUNCTIONALITY OF SPECIFIC MONGODB FEATURES.
6
+ THEY ARE NOT PRODUCTION-READY AND MAY LACK THE SECURITY HARDENING, ERROR HANDLING, AND TESTING REQUIRED FOR A LIVE ENVIRONMENT.
7
+ YOU ARE RESPONSIBLE FOR TESTING, VALIDATING, AND SECURING THIS CODE WITHIN YOUR OWN ENVIRONMENT BEFORE IMPLEMENTATION.
8
+ THIS MATERIAL IS PROVIDED "AS IS" WITHOUT WARRANTY OR LIABILITY.
9
+
10
+ MongoDB log analysis package.
11
+ """
@@ -0,0 +1,25 @@
1
+ """
2
+ Copyright (c) 2026 MongoDB Inc.
3
+
4
+ DISCLAIMER: THESE CODE SAMPLES ARE PROVIDED FOR EDUCATIONAL AND ILLUSTRATIVE PURPOSES ONLY,
5
+ TO DEMONSTRATE THE FUNCTIONALITY OF SPECIFIC MONGODB FEATURES.
6
+ THEY ARE NOT PRODUCTION-READY AND MAY LACK THE SECURITY HARDENING, ERROR HANDLING, AND TESTING REQUIRED FOR A LIVE ENVIRONMENT.
7
+ YOU ARE RESPONSIBLE FOR TESTING, VALIDATING, AND SECURING THIS CODE WITHIN YOUR OWN ENVIRONMENT BEFORE IMPLEMENTATION.
8
+ THIS MATERIAL IS PROVIDED "AS IS" WITHOUT WARRANTY OR LIABILITY.
9
+
10
+ AI analysis for the log plugin (warning / error / fatal log lines).
11
+
12
+ Uses the shared client in :mod:`mongo_x_ray.ai_client`.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from mongo_x_ray.ai_client import complete
18
+
19
+ _SYSTEM_PROMPT = "You are a MongoDB expert. Analyze MongoDB log messages and tell me the reason in max 200 words."
20
+
21
+
22
+ def analyze_log_line_gpt(log_line: dict) -> str:
23
+ """Analyze a MongoDB log line using the shared AI client."""
24
+ result = complete(str(log_line), system=_SYSTEM_PROMPT)
25
+ return (result or "").strip()
@@ -0,0 +1,330 @@
1
+ """
2
+ Copyright (c) 2026 MongoDB Inc.
3
+
4
+ DISCLAIMER: THESE CODE SAMPLES ARE PROVIDED FOR EDUCATIONAL AND ILLUSTRATIVE PURPOSES ONLY,
5
+ TO DEMONSTRATE THE FUNCTIONALITY OF SPECIFIC MONGODB FEATURES.
6
+ THEY ARE NOT PRODUCTION-READY AND MAY LACK THE SECURITY HARDENING, ERROR HANDLING, AND TESTING REQUIRED FOR A LIVE ENVIRONMENT.
7
+ YOU ARE RESPONSIBLE FOR TESTING, VALIDATING, AND SECURING THIS CODE WITHIN YOUR OWN ENVIRONMENT BEFORE IMPLEMENTATION.
8
+ THIS MATERIAL IS PROVIDED "AS IS" WITHOUT WARRANTY OR LIABILITY.
9
+ """
10
+
11
+ import logging
12
+ import random
13
+ import re
14
+ from datetime import datetime, timezone
15
+ from pathlib import Path
16
+ from typing import Optional, TextIO
17
+
18
+ from bson import json_util
19
+
20
+ from mongo_x_ray.framework import BaseFramework
21
+ from mongo_x_ray.shared import str_to_md_id, to_json
22
+ from mongo_x_ray.utils import bold, cyan, env, green, load_classes, yellow
23
+ from mongo_x_ray_log.log_items.info_item import InfoItem
24
+ from mongo_x_ray_log.log_items.state_trace_item import StateTraceItem
25
+
26
+ logger = logging.getLogger(__name__)
27
+ LOG_CLASSES = load_classes("mongo_x_ray_log.log_items")
28
+ SKIP_LINE_MSG = "HEADER INCLUDED, NOW SKIPPING 64728 LINES ACCORDING TO REQUESTED SIZE LIMIT"
29
+ _SANITIZE_DATE_RE = re.compile(r'\{\s*"\$date"\s*:\s*\{\s*"\$numberLong"\s*:\s*"-?\d{16,}"\s*\}\s*\}')
30
+
31
+
32
+ def _sanitize_date_numberlong(line: str) -> str:
33
+ """Replace out-of-range $date.$numberLong sentinel values with null."""
34
+ return _SANITIZE_DATE_RE.sub("null", line)
35
+
36
+
37
+ def _safe_json_loads(line: str) -> dict:
38
+ """Parse a JSON log line, sanitising out-of-range dates on failure only.
39
+ Also ensures all datetime values are timezone-aware (UTC).
40
+ """
41
+ try:
42
+ parsed = json_util.loads(line)
43
+ _normalise_datetimes(parsed)
44
+ return parsed
45
+ except Exception as exc:
46
+ try:
47
+ parsed = json_util.loads(_sanitize_date_numberlong(line))
48
+ _normalise_datetimes(parsed)
49
+ return parsed
50
+ except Exception:
51
+ logger.debug("JSON parse failed (first error: %s): %s", exc, line.strip()[:200])
52
+ return {}
53
+
54
+
55
+ def _normalise_datetimes(obj: dict) -> None:
56
+ """Ensure all datetime values in *obj* are UTC-aware (in place)."""
57
+ for key, value in obj.items():
58
+ if isinstance(value, datetime) and value.tzinfo is None:
59
+ obj[key] = value.replace(tzinfo=timezone.utc)
60
+
61
+
62
+ class Framework(BaseFramework):
63
+ template_module = "log"
64
+ template_package = "mongo_x_ray_log"
65
+
66
+ def __init__(
67
+ self,
68
+ file_path: str,
69
+ config: dict,
70
+ start_time: Optional[datetime] = None,
71
+ end_time: Optional[datetime] = None,
72
+ ):
73
+ super().__init__(config)
74
+ self._file_path = file_path
75
+ self._start_time = start_time
76
+ self._end_time = end_time
77
+ self._logger.debug(to_json(self._config))
78
+ self._log_start: Optional[datetime] = None
79
+ self._log_end: Optional[datetime] = None
80
+ self._hostname: Optional[str] = None
81
+ if env == "development":
82
+ self._logger.info(yellow("Running in development mode."))
83
+
84
+ @property
85
+ def hostname(self) -> Optional[str]:
86
+ """The hostname from the Process Info log item, or from log lines."""
87
+ for item in self._items:
88
+ if isinstance(item, InfoItem):
89
+ host = item._cache.get("process", {}).get("host")
90
+ if host and host != "Unknown":
91
+ return host
92
+ break
93
+ return self._hostname
94
+
95
+ def _log_files(self) -> list[Path]:
96
+ """Return a sorted list of log files to process."""
97
+ path = Path(self._file_path)
98
+ if path.is_file():
99
+ return [path]
100
+ # Match mongod.log, mongod.log.2026-06-10T01-58-56, etc.
101
+ files = sorted(path.glob("*.log*"))
102
+ if not files:
103
+ files = sorted(path.glob("*"))
104
+ return files
105
+
106
+ @staticmethod
107
+ def _file_time_range(file_path: Path) -> tuple:
108
+ """Read the first and last valid JSON log line to get the file's time range."""
109
+ first_ts = None
110
+ last_ts = None
111
+ try:
112
+ with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
113
+ # Skip preamble lines (e.g. Atlas download header)
114
+ for line in f:
115
+ parsed = _safe_json_loads(line)
116
+ if parsed:
117
+ first_ts = parsed.get("t")
118
+ break
119
+ # Scan backwards from end for the last non-empty line
120
+ f.seek(0, 2)
121
+ pos = f.tell()
122
+ last_line = ""
123
+ while pos > 0:
124
+ pos -= 1
125
+ f.seek(pos)
126
+ if f.read(1) == "\n":
127
+ candidate = f.readline().strip()
128
+ if candidate:
129
+ last_line = candidate
130
+ break
131
+ if last_line:
132
+ last_ts = _safe_json_loads(last_line).get("t")
133
+ except Exception as exc:
134
+ logger.warning("Failed to read time range from %s: %s", file_path.name, exc)
135
+ logger.debug(
136
+ "File %s time range: %s – %s",
137
+ file_path.name,
138
+ first_ts.isoformat() if first_ts else "?",
139
+ last_ts.isoformat() if last_ts else "?",
140
+ )
141
+ return first_ts, last_ts
142
+
143
+ def _file_overlaps_range(self, file_path: Path) -> bool:
144
+ """Return False if the file's time range is entirely outside the requested range."""
145
+ if self._start_time is None and self._end_time is None:
146
+ return True
147
+ first_ts, last_ts = self._file_time_range(file_path)
148
+ if first_ts is None or last_ts is None:
149
+ return True # can't determine, process anyway
150
+ if self._end_time is not None and first_ts > self._end_time:
151
+ return False
152
+ if self._start_time is not None and last_ts < self._start_time:
153
+ return False
154
+ return True
155
+
156
+ def _any_file_fully_covered(self, files: list[Path]) -> bool:
157
+ """Return True if at least one file is fully within [start_time, end_time]."""
158
+ if self._start_time is None and self._end_time is None:
159
+ return True
160
+ for fp in files:
161
+ first_ts, last_ts = self._file_time_range(fp)
162
+ if first_ts is None or last_ts is None:
163
+ continue
164
+ if (self._start_time is None or first_ts >= self._start_time) and (
165
+ self._end_time is None or last_ts <= self._end_time
166
+ ):
167
+ return True
168
+ return False
169
+
170
+ def run_logs_analysis(self, logset_name: str, *_args, **kwargs):
171
+ self._set_name = logset_name
172
+ # Create output folder if it doesn't exist
173
+ output_folder = kwargs.get("output_folder", "output/")
174
+ batch_folder = self._get_output_folder(output_folder)
175
+ # Dynamically load the log checkset based on the name
176
+ logsets = self._config.get("logsets", {})
177
+ if logset_name not in logsets:
178
+ self._logger.warning(
179
+ yellow(f"Log checkset '{logset_name}' not found in configuration. Using default logset.")
180
+ )
181
+ logset_name = "default"
182
+ ls = logsets[logset_name]
183
+ self._logger.info("Running log checkset: %s", bold(cyan(logset_name)))
184
+
185
+ self._items = []
186
+ for item_name in ls.get("items", []):
187
+ item_cls = LOG_CLASSES.get(item_name)
188
+ if not item_cls:
189
+ self._logger.warning(yellow(f"Log item '{item_name}' not found. Skipping."))
190
+ continue
191
+ item_config = self._config.get("item_config", {}).get(item_name, {})
192
+ item = item_cls(str(batch_folder), item_config)
193
+ self._items.append(item)
194
+ self._logger.info("Log analyze item loaded: %s", bold(cyan(item_name)))
195
+
196
+ rate = self._config.get("sample_rate", 1.0)
197
+ log_files = self._log_files()
198
+ partial_only = (
199
+ self._start_time is not None or self._end_time is not None
200
+ ) and not self._any_file_fully_covered(log_files)
201
+ if partial_only:
202
+ self._logger.info(
203
+ "No log file is fully covered by the requested time range. "
204
+ "InfoItem and StateTraceItem will receive all lines."
205
+ )
206
+ log_line: dict = {}
207
+ global_counter: int = 0
208
+
209
+ for lf in log_files:
210
+ if not self._file_overlaps_range(lf):
211
+ self._logger.info(
212
+ "Skipping %s (outside time range %s – %s)",
213
+ lf.name,
214
+ self._start_time.isoformat() if self._start_time else "…",
215
+ self._end_time.isoformat() if self._end_time else "…",
216
+ )
217
+ continue
218
+ self._logger.info("Processing %s", green(str(lf)))
219
+
220
+ with open(lf, "r", encoding="utf-8", errors="ignore") as f:
221
+ counter: int = 0
222
+ for line in f:
223
+ counter += 1
224
+ global_counter += 1
225
+ if global_counter % 10000 == 0:
226
+ self._logger.info("%s lines ingested...", green(str(global_counter)))
227
+ if random.random() > rate:
228
+ continue
229
+ try:
230
+ if counter == 101 and line.startswith(SKIP_LINE_MSG):
231
+ self._logger.debug("Some lines are skipped due to the size limit. This is expected.")
232
+ continue
233
+ log_line = _safe_json_loads(line)
234
+ if not log_line:
235
+ self._logger.warning(yellow(f"Failed to parse log line as JSON: {line.strip()}"))
236
+ continue
237
+ if self._hostname is None:
238
+ hostname = log_line.get("hostname")
239
+ if isinstance(hostname, str) and hostname.strip():
240
+ self._hostname = hostname.strip()
241
+ line_ts = log_line.get("t")
242
+ out_of_range = False
243
+ if line_ts is not None:
244
+ if self._start_time is not None and line_ts < self._start_time:
245
+ out_of_range = True
246
+ elif self._end_time is not None and line_ts > self._end_time:
247
+ out_of_range = True
248
+
249
+ if out_of_range and not partial_only:
250
+ continue
251
+
252
+ if self._log_start is None:
253
+ self._log_start = line_ts
254
+
255
+ if out_of_range:
256
+ # partial_only: dispatch only to InfoItem and StateTraceItem
257
+ for item in self._items:
258
+ if isinstance(item, (InfoItem, StateTraceItem)):
259
+ try:
260
+ item.analyze(log_line)
261
+ except Exception as e:
262
+ self._logger.warning(yellow(f"Log analysis item '{item.name}' failed: {e}"))
263
+ continue
264
+
265
+ for item in self._items:
266
+ try:
267
+ item.analyze(log_line)
268
+ except Exception as e:
269
+ self._logger.warning(yellow(f"Log analysis item '{item.name}' failed: {e}"))
270
+ continue
271
+ except Exception as exc:
272
+ self._logger.warning(yellow(f"Unexpected error processing log line: {exc}"))
273
+ continue
274
+
275
+ self._log_end = log_line.get("t", None) if log_line else None
276
+ for item in self._items:
277
+ item._hostname = self._hostname
278
+ try:
279
+ item.finalize_analysis()
280
+ except Exception as e:
281
+ self._logger.warning(yellow(f"Log analysis item '{item.name}' finalize failed: {e}"))
282
+ continue
283
+
284
+ def _render_markdown(self, output: TextIO) -> None:
285
+ assert self._log_start is not None and self._log_end is not None, (
286
+ "Log start and end time should be set after analysis."
287
+ )
288
+ output.write("# Log Analysis Report\n")
289
+ output.write(f"Generated at: `{str(datetime.now(tz=timezone.utc))} UTC`\n\n")
290
+ output.write(f"Log path: `{self._file_path}`\n\n")
291
+ if self._start_time or self._end_time:
292
+ start_str = self._start_time.isoformat() if self._start_time else "…"
293
+ end_str = self._end_time.isoformat() if self._end_time else "…"
294
+ output.write(f"Requested time range: `{start_str}` – `{end_str}`\n\n")
295
+ output.write(f"Log analysis period: `{self._log_start.isoformat()}` to `{self._log_end.isoformat()}`\n\n")
296
+ output.write("Histogram chart instructions:\n\n")
297
+ output.write("- **zoom in/out:** _ctrl+wheel, or pinch_\n")
298
+ output.write("- **pan:** _shift+drag_\n")
299
+ output.write("- **select time frame:** _drag_\n\n")
300
+
301
+ output.write("## 1 Review Test Results\n\n")
302
+ for i, item in enumerate(self._items):
303
+ title = f"1.{i + 1} {item.name}"
304
+ review_title = f"2.{i + 1} Review {item.name}"
305
+ review_title_id = str_to_md_id(review_title)
306
+ output.write(f"### {title}\n\n")
307
+ output.write(f"{item.description}\n\n")
308
+ output.write(f"[Review Raw Results &rarr;](#{review_title_id})\n\n")
309
+ try:
310
+ item.test_result_markdown(output)
311
+ except Exception as e:
312
+ self._logger.warning(yellow(f"Failed to generate test results for log item '{item.name}': {e}"))
313
+ continue
314
+
315
+ output.write("## 2 Review Raw Results\n\n")
316
+ for i, item in enumerate(self._items):
317
+ title = f"1.{i + 1} {item.name}"
318
+ title_id = str_to_md_id(title)
319
+ review_title = f"2.{i + 1} Review {item.name}"
320
+ output.write(f"### {review_title}\n\n")
321
+ output.write(f"[&larr; Review Test Results](#{title_id})\n\n")
322
+ if getattr(item, "_show_reset", False):
323
+ output.write(
324
+ f'<input type="button" id="reset_{item.__class__.__name__}" class="table-copy-button" value="Reset">\n\n'
325
+ )
326
+ try:
327
+ item.review_results_markdown(output)
328
+ except Exception as e:
329
+ self._logger.warning(yellow(f"Failed to generate markdown for log item '{item.name}': {e}"))
330
+ continue
@@ -0,0 +1,9 @@
1
+ """
2
+ Copyright (c) 2026 MongoDB Inc.
3
+
4
+ DISCLAIMER: THESE CODE SAMPLES ARE PROVIDED FOR EDUCATIONAL AND ILLUSTRATIVE PURPOSES ONLY,
5
+ TO DEMONSTRATE THE FUNCTIONALITY OF SPECIFIC MONGODB FEATURES.
6
+ THEY ARE NOT PRODUCTION-READY AND MAY LACK THE SECURITY HARDENING, ERROR HANDLING, AND TESTING REQUIRED FOR A LIVE ENVIRONMENT.
7
+ YOU ARE RESPONSIBLE FOR TESTING, VALIDATING, AND SECURING THIS CODE WITHIN YOUR OWN ENVIRONMENT BEFORE IMPLEMENTATION.
8
+ THIS MATERIAL IS PROVIDED "AS IS" WITHOUT WARRANTY OR LIABILITY.
9
+ """