mongo-x-ray 1.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. mongo_x_ray-1.2.0/LICENSE +21 -0
  2. mongo_x_ray-1.2.0/PKG-INFO +265 -0
  3. mongo_x_ray-1.2.0/README.md +155 -0
  4. mongo_x_ray-1.2.0/libs/__init__.py +1 -0
  5. mongo_x_ray-1.2.0/libs/ai.py +75 -0
  6. mongo_x_ray-1.2.0/libs/compatibility_matrix.json +109 -0
  7. mongo_x_ray-1.2.0/libs/config.json +96 -0
  8. mongo_x_ray-1.2.0/libs/gmd/framework.py +0 -0
  9. mongo_x_ray-1.2.0/libs/healthcheck/__init__.py +1 -0
  10. mongo_x_ray-1.2.0/libs/healthcheck/check_items/__init__.py +1 -0
  11. mongo_x_ray-1.2.0/libs/healthcheck/check_items/base_item.py +152 -0
  12. mongo_x_ray-1.2.0/libs/healthcheck/check_items/build_info_item.py +99 -0
  13. mongo_x_ray-1.2.0/libs/healthcheck/check_items/cluster_item.py +290 -0
  14. mongo_x_ray-1.2.0/libs/healthcheck/check_items/coll_info_item.py +259 -0
  15. mongo_x_ray-1.2.0/libs/healthcheck/check_items/host_info_item.py +96 -0
  16. mongo_x_ray-1.2.0/libs/healthcheck/check_items/index_info_item.py +167 -0
  17. mongo_x_ray-1.2.0/libs/healthcheck/check_items/security_item.py +130 -0
  18. mongo_x_ray-1.2.0/libs/healthcheck/check_items/server_status_item.py +428 -0
  19. mongo_x_ray-1.2.0/libs/healthcheck/check_items/shard_key_item.py +118 -0
  20. mongo_x_ray-1.2.0/libs/healthcheck/framework.py +138 -0
  21. mongo_x_ray-1.2.0/libs/healthcheck/issues.py +385 -0
  22. mongo_x_ray-1.2.0/libs/healthcheck/rules/__init__.py +1 -0
  23. mongo_x_ray-1.2.0/libs/healthcheck/rules/base_rule.py +12 -0
  24. mongo_x_ray-1.2.0/libs/healthcheck/rules/cache_rule.py +103 -0
  25. mongo_x_ray-1.2.0/libs/healthcheck/rules/connections_rule.py +37 -0
  26. mongo_x_ray-1.2.0/libs/healthcheck/rules/data_size_rule.py +51 -0
  27. mongo_x_ray-1.2.0/libs/healthcheck/rules/fragmentation_rule.py +59 -0
  28. mongo_x_ray-1.2.0/libs/healthcheck/rules/index_rule.py +103 -0
  29. mongo_x_ray-1.2.0/libs/healthcheck/rules/op_latency_rule.py +95 -0
  30. mongo_x_ray-1.2.0/libs/healthcheck/rules/oplog_window_rule.py +42 -0
  31. mongo_x_ray-1.2.0/libs/healthcheck/rules/query_targeting_rule.py +55 -0
  32. mongo_x_ray-1.2.0/libs/healthcheck/rules/rs_config_rule.py +72 -0
  33. mongo_x_ray-1.2.0/libs/healthcheck/rules/rs_status_rule.py +60 -0
  34. mongo_x_ray-1.2.0/libs/healthcheck/rules/security_rule.py +57 -0
  35. mongo_x_ray-1.2.0/libs/healthcheck/rules/shard_balance_rule.py +51 -0
  36. mongo_x_ray-1.2.0/libs/healthcheck/rules/shard_key_rule.py +29 -0
  37. mongo_x_ray-1.2.0/libs/healthcheck/rules/shard_mongos_rule.py +35 -0
  38. mongo_x_ray-1.2.0/libs/healthcheck/rules/version_eol_rule.py +36 -0
  39. mongo_x_ray-1.2.0/libs/healthcheck/shared.py +367 -0
  40. mongo_x_ray-1.2.0/libs/log_analysis/__init__.py +1 -0
  41. mongo_x_ray-1.2.0/libs/log_analysis/framework.py +135 -0
  42. mongo_x_ray-1.2.0/libs/log_analysis/log_items/__init__.py +1 -0
  43. mongo_x_ray-1.2.0/libs/log_analysis/log_items/base_item.py +117 -0
  44. mongo_x_ray-1.2.0/libs/log_analysis/log_items/client_meta_item.py +174 -0
  45. mongo_x_ray-1.2.0/libs/log_analysis/log_items/connection_rate_item.py +50 -0
  46. mongo_x_ray-1.2.0/libs/log_analysis/log_items/info_item.py +146 -0
  47. mongo_x_ray-1.2.0/libs/log_analysis/log_items/slow_chart_item.py +39 -0
  48. mongo_x_ray-1.2.0/libs/log_analysis/log_items/slow_rate_item.py +46 -0
  49. mongo_x_ray-1.2.0/libs/log_analysis/log_items/state_trace_item.py +157 -0
  50. mongo_x_ray-1.2.0/libs/log_analysis/log_items/top_slow_item.py +114 -0
  51. mongo_x_ray-1.2.0/libs/log_analysis/log_items/wef_item.py +98 -0
  52. mongo_x_ray-1.2.0/libs/log_analysis/query_analyzer.py +205 -0
  53. mongo_x_ray-1.2.0/libs/log_analysis/shared.py +7 -0
  54. mongo_x_ray-1.2.0/libs/utils.py +260 -0
  55. mongo_x_ray-1.2.0/libs/version.py +94 -0
  56. mongo_x_ray-1.2.0/mongo_x_ray.egg-info/PKG-INFO +265 -0
  57. mongo_x_ray-1.2.0/mongo_x_ray.egg-info/SOURCES.txt +86 -0
  58. mongo_x_ray-1.2.0/mongo_x_ray.egg-info/dependency_links.txt +1 -0
  59. mongo_x_ray-1.2.0/mongo_x_ray.egg-info/entry_points.txt +2 -0
  60. mongo_x_ray-1.2.0/mongo_x_ray.egg-info/requires.txt +64 -0
  61. mongo_x_ray-1.2.0/mongo_x_ray.egg-info/top_level.txt +2 -0
  62. mongo_x_ray-1.2.0/pyproject.toml +138 -0
  63. mongo_x_ray-1.2.0/setup.cfg +4 -0
  64. mongo_x_ray-1.2.0/templates/healthcheck/full.html +1 -0
  65. mongo_x_ray-1.2.0/templates/healthcheck/full.raw.html +171 -0
  66. mongo_x_ray-1.2.0/templates/healthcheck/no-network.html +1 -0
  67. mongo_x_ray-1.2.0/templates/healthcheck/no-network.raw.html +30 -0
  68. mongo_x_ray-1.2.0/templates/healthcheck/snippets/BuildInfoItem_1.js +30 -0
  69. mongo_x_ray-1.2.0/templates/healthcheck/snippets/CollInfoItem_1.js +84 -0
  70. mongo_x_ray-1.2.0/templates/healthcheck/snippets/CollInfoItem_3.js +139 -0
  71. mongo_x_ray-1.2.0/templates/healthcheck/snippets/CollInfoItem_5.js +82 -0
  72. mongo_x_ray-1.2.0/templates/healthcheck/snippets/ServerStatusItem_1.js +115 -0
  73. mongo_x_ray-1.2.0/templates/healthcheck/snippets/ServerStatusItem_3.js +64 -0
  74. mongo_x_ray-1.2.0/templates/healthcheck/snippets/ServerStatusItem_5.js +139 -0
  75. mongo_x_ray-1.2.0/templates/healthcheck/standard.html +1 -0
  76. mongo_x_ray-1.2.0/templates/healthcheck/standard.raw.html +30 -0
  77. mongo_x_ray-1.2.0/templates/log/full.html +1 -0
  78. mongo_x_ray-1.2.0/templates/log/full.raw.html +281 -0
  79. mongo_x_ray-1.2.0/templates/log/snippets/ClientMetaItem.js +80 -0
  80. mongo_x_ray-1.2.0/templates/log/snippets/ConnectionRateItem.js +149 -0
  81. mongo_x_ray-1.2.0/templates/log/snippets/InfoItem.js +3 -0
  82. mongo_x_ray-1.2.0/templates/log/snippets/SlowChartItem.js +358 -0
  83. mongo_x_ray-1.2.0/templates/log/snippets/SlowRateItem.js +143 -0
  84. mongo_x_ray-1.2.0/templates/log/snippets/StateTraceItem.js +191 -0
  85. mongo_x_ray-1.2.0/templates/log/snippets/TopSlowItem.js +23 -0
  86. mongo_x_ray-1.2.0/templates/log/snippets/WEFItem.js +30 -0
  87. mongo_x_ray-1.2.0/tests/test_utils.py +98 -0
  88. mongo_x_ray-1.2.0/x_ray.py +254 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Yaoxing Zhang
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,265 @@
1
+ Metadata-Version: 2.1
2
+ Name: mongo-x-ray
3
+ Version: 1.2.0
4
+ Summary: MongoDB diagnostics toolkit with health checks and log analysis
5
+ Author: Yaoxing Zhang
6
+ License: MIT License
7
+
8
+ Copyright (c) 2025 Yaoxing Zhang
9
+
10
+ Permission is hereby granted, free of charge, to any person obtaining a copy
11
+ of this software and associated documentation files (the "Software"), to deal
12
+ in the Software without restriction, including without limitation the rights
13
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14
+ copies of the Software, and to permit persons to whom the Software is
15
+ furnished to do so, subject to the following conditions:
16
+
17
+ The above copyright notice and this permission notice shall be included in all
18
+ copies or substantial portions of the Software.
19
+
20
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26
+ SOFTWARE.
27
+
28
+ Project-URL: Homepage, https://github.com/zhangyaoxing/x-ray
29
+ Project-URL: Repository, https://github.com/zhangyaoxing/x-ray
30
+ Project-URL: Issues, https://github.com/zhangyaoxing/x-ray/issues
31
+ Keywords: mongodb,diagnostics,healthcheck,log-analysis
32
+ Classifier: Development Status :: 4 - Beta
33
+ Classifier: Intended Audience :: System Administrators
34
+ Classifier: Intended Audience :: Developers
35
+ Classifier: License :: OSI Approved :: MIT License
36
+ Classifier: Operating System :: OS Independent
37
+ Classifier: Programming Language :: Python
38
+ Classifier: Programming Language :: Python :: 3
39
+ Classifier: Programming Language :: Python :: 3.9
40
+ Classifier: Programming Language :: Python :: 3.10
41
+ Classifier: Programming Language :: Python :: 3.11
42
+ Classifier: Programming Language :: Python :: 3.12
43
+ Classifier: Topic :: Database
44
+ Classifier: Topic :: System :: Monitoring
45
+ Requires-Python: <4,>=3.9
46
+ Description-Content-Type: text/markdown
47
+ License-File: LICENSE
48
+ Requires-Dist: pymongo==4.15.4
49
+ Requires-Dist: Jinja2==3.1.6
50
+ Requires-Dist: Markdown==3.9
51
+ Requires-Dist: MarkupSafe==3.0.3
52
+ Requires-Dist: packaging==25
53
+ Requires-Dist: Pygments==2.19.2
54
+ Requires-Dist: PyYAML==6.0.3
55
+ Requires-Dist: requests==2.32.5
56
+ Requires-Dist: urllib3==2.5.0
57
+ Requires-Dist: openai==2.8.1
58
+ Requires-Dist: httpx==0.28.1
59
+ Requires-Dist: httpcore==1.0.9
60
+ Requires-Dist: h11==0.16.0
61
+ Requires-Dist: certifi==2025.11.12
62
+ Requires-Dist: idna==3.11
63
+ Requires-Dist: sniffio==1.3.1
64
+ Requires-Dist: anyio==4.11.0
65
+ Provides-Extra: dev
66
+ Requires-Dist: black==25.11.0; extra == "dev"
67
+ Requires-Dist: pylint==3.3.9; extra == "dev"
68
+ Requires-Dist: flake8==7.3.0; extra == "dev"
69
+ Requires-Dist: pytest==8.4.2; extra == "dev"
70
+ Requires-Dist: pluggy==1.6.0; extra == "dev"
71
+ Requires-Dist: iniconfig==2.1.0; extra == "dev"
72
+ Requires-Dist: tomli==2.3.0; extra == "dev"
73
+ Requires-Dist: pyinstaller==6.16.0; extra == "dev"
74
+ Requires-Dist: pyinstaller-hooks-contrib==2025.10; extra == "dev"
75
+ Requires-Dist: macholib==1.16.4; extra == "dev"
76
+ Requires-Dist: altgraph==0.17.5; extra == "dev"
77
+ Provides-Extra: ai
78
+ Requires-Dist: torch==2.8.0; extra == "ai"
79
+ Requires-Dist: torchvision==0.23.0; extra == "ai"
80
+ Requires-Dist: transformers==4.57.1; extra == "ai"
81
+ Requires-Dist: accelerate==1.10.1; extra == "ai"
82
+ Requires-Dist: bitsandbytes==0.48.2; extra == "ai"
83
+ Requires-Dist: huggingface-hub==1.1.5; extra == "ai"
84
+ Requires-Dist: safetensors==0.7.0; extra == "ai"
85
+ Requires-Dist: tokenizers==0.22.1; extra == "ai"
86
+ Requires-Dist: filelock==3.19.1; extra == "ai"
87
+ Requires-Dist: fsspec==2025.10.0; extra == "ai"
88
+ Requires-Dist: hf-xet==1.2.0; extra == "ai"
89
+ Requires-Dist: numpy==2.0.2; extra == "ai"
90
+ Requires-Dist: scipy==1.13.1; extra == "ai"
91
+ Requires-Dist: pillow==11.3.0; extra == "ai"
92
+ Requires-Dist: regex==2025.11.3; extra == "ai"
93
+ Requires-Dist: tqdm==4.67.1; extra == "ai"
94
+ Requires-Dist: pydantic==2.12.4; extra == "ai"
95
+ Requires-Dist: pydantic_core==2.41.5; extra == "ai"
96
+ Requires-Dist: annotated-types==0.7.0; extra == "ai"
97
+ Requires-Dist: typing_extensions==4.15.0; extra == "ai"
98
+ Requires-Dist: jiter==0.12.0; extra == "ai"
99
+ Requires-Dist: psutil==7.1.3; extra == "ai"
100
+ Requires-Dist: networkx==3.2.1; extra == "ai"
101
+ Requires-Dist: sympy==1.14.0; extra == "ai"
102
+ Requires-Dist: mpmath==1.3.0; extra == "ai"
103
+ Requires-Dist: distro==1.9.0; extra == "ai"
104
+ Requires-Dist: dnspython==2.7.0; extra == "ai"
105
+ Requires-Dist: charset-normalizer==3.4.4; extra == "ai"
106
+ Requires-Dist: exceptiongroup==1.3.1; extra == "ai"
107
+ Requires-Dist: importlib_metadata==8.7.0; extra == "ai"
108
+ Requires-Dist: typing-inspection==0.4.2; extra == "ai"
109
+ Requires-Dist: zipp==3.23.0; extra == "ai"
110
+
111
+ # x-ray
112
+ [![Makefile](https://github.com/zhangyaoxing/x-ray/actions/workflows/makefile.yml/badge.svg)](https://github.com/zhangyaoxing/x-ray/actions/workflows/makefile.yml)
113
+ [![Release](https://github.com/zhangyaoxing/x-ray/actions/workflows/release.yml/badge.svg)](https://github.com/zhangyaoxing/x-ray/actions/workflows/release.yml)
114
+ [![PyPI](https://img.shields.io/pypi/v/mongo-x-ray.svg)](https://pypi.org/project/mongo-x-ray/)
115
+
116
+
117
+ This project aims to create tools for MongoDB analysis and diagnosis. So far 3 modules are built:
118
+ - Health check module.
119
+ - Log analysis module.
120
+ - `getMongoData` visualization module (Under construction).
121
+
122
+ ## 1 Compatibility Matrix
123
+ ### Health Check
124
+ | Replica Set | Sharded Cluster | Standalone |
125
+ | :-----------: | :-------------: | :--------: |
126
+ | >=4.2 &check; | >=4.2 &check; | &cross; |
127
+
128
+ Older versions are not tested.
129
+
130
+ ### Log Analysis
131
+ Log analysis requires JSON format logs, which is supported since 4.4.
132
+ | Replica Set | Sharded Cluster | Standalone |
133
+ | :-----------: | :-------------: | :-----------: |
134
+ | >=4.4 &check; | >=4.4 &check; | >=4.4 &check; |
135
+
136
+ ## 2 How to Install
137
+ ### 2.1 PyPi
138
+ #### 2.1.1 Install with Pip
139
+ The easiest and recommended way to install x-ray is to use `pip`:
140
+ ```bash
141
+ pip install mongo-x-ray
142
+ ```
143
+
144
+ #### 2.1.2 Build from Source
145
+ ```bash
146
+ git clone https://github.com/zhangyaoxing/x-ray
147
+ cd x-ray
148
+ pip install .
149
+ ```
150
+
151
+ ### 2.2 PyInstaller
152
+ #### 2.2.1 Prebuilt Binaries
153
+ Currently the prebuilt binaries are built on 3 platforms:
154
+ - Ubuntu Latest (AMD64)
155
+ - MacOS Latest (ARM64)
156
+ - Windows Latest (AMD64)
157
+
158
+ Download them from [Releases](https://github.com/zhangyaoxing/x-ray/releases).
159
+
160
+ #### 2.2.2 Build from Source
161
+ x-ray is tested on `Python 3.9.22`. On MacOS or Linux distributions, you can use the `make` command to build the binary:
162
+ ```bash
163
+ git clone https://github.com/zhangyaoxing/x-ray
164
+ cd x-ray
165
+ make deps # if it's the first time you build the project
166
+ make # equal to `make build` and `make build-lite`
167
+ ```
168
+
169
+ There are other make targets. Use `make help` to find out.
170
+
171
+ You can also build the tool with AI modules for log analysis. For more details refer to: [Build with AI Support](https://github.com/zhangyaoxing/x-ray/wiki/Build-with-AI-Support).
172
+
173
+ For Windows users, if `make` command is not available. You can use Python commands to build the binary:
174
+ ```powershell
175
+ python.exe -m venv .venv
176
+ .venv\Scripts\python.exe -m pip install --upgrade pip
177
+ .venv\Scripts\python.exe -m pip install -e ".[dev]"
178
+ .venv\Scripts\python.exe -m PyInstaller --onefile `
179
+ --name x-ray `
180
+ --add-data="templates;templates" `
181
+ --add-data="libs;libs" `
182
+ --icon="misc/x-ray.ico" `
183
+ --hidden-import=openai `
184
+ x-ray
185
+ ```
186
+
187
+ #### 2.3 For Developers
188
+ For developers, use `make deps` to prepare venv and dependencies
189
+ ```bash
190
+ make deps
191
+ ```
192
+ Or
193
+ ```bash
194
+ python3 -m venv .venv
195
+ python3 -m pip install --upgrade pip
196
+ python3 -m pip install -e ".[dev]"
197
+ ```
198
+
199
+ ## 3 Using the Tool
200
+ ```bash
201
+ x-ray [-h] [-q] [-c CONFIG] {healthcheck,hc,log}
202
+ ```
203
+ | Argument | Description | Default |
204
+ | ---------------- | ---------------------------------------------------------------------------------------------- | :-------------------------: |
205
+ | `-q`, `--quiet` | Quiet mode. | `false` |
206
+ | `-h`, `--help` | Show the help message and exit. | n/a |
207
+ | `-c`, `--config` | Path to configuration file. | Built-in `libs/config.json` |
208
+ | `command` | Command to run. Include:<br/>- `healthcheck` or `hc`: Health check.<br/>- `log`: Log analysis. | None |
209
+
210
+ Besides, you can use environment variables to control some behaviors:
211
+ - `ENV=development` For developing. It will change the following behaviors:
212
+ - Formatted the output JSON for for easier reading.
213
+ - The output will not create a new folder for each run but overwrite the same files.
214
+ - `LOG_LEVEL`: Can be `DEBUG`, `ERROR` or `INFO` (default).
215
+
216
+ ### 3.1 Health Check Component
217
+ #### 3.1.1 Examples
218
+ ```bash
219
+ ./x-ray healthcheck localhost:27017 # Scan the cluster with default settings.
220
+ ./x-ray hc localhost:27017 --output ./output/ # Specify output folder.
221
+ ./x-ray hc localhost:27017 --config ./config.json # Use your own configuration.
222
+ ```
223
+
224
+ #### 3.1.2 Full Arguments
225
+ ```bash
226
+ x-ray healthcheck [-h] [-s CHECKSET] [-o OUTPUT] [-f {markdown,html}] [uri]
227
+ ```
228
+ | Argument | Description | Default |
229
+ | ------------------ | ------------------------------------------- | :-------: |
230
+ | `-s`, `--checkset` | Checkset to run. | `default` |
231
+ | `-o`, `--output` | Output folder path. | `output/` |
232
+ | `-f`, `--format` | Output format. Can be `markdown` or `html`. | `html` |
233
+ | `uri` | MongoDB database URI. | None |
234
+
235
+ For security reasons you may not want to include credentials in the command. There are 2 options:
236
+ - If the URI is not provided, user will be asked to input one.
237
+ - If URI is provided but not username/password, user will also be asked to input them.
238
+
239
+ #### 3.1.3 More Info
240
+ Refer to the wiki for more details.
241
+ - [Customize the thresholds](https://github.com/zhangyaoxing/x-ray/wiki/Health-Check-Configuration)
242
+ - [Database permissions](https://github.com/zhangyaoxing/x-ray/wiki/Health-Check-Database-Permissions)
243
+ - [Output](https://github.com/zhangyaoxing/x-ray/wiki/Health-Check-Output)
244
+ - [Customize the output](https://github.com/zhangyaoxing/x-ray/wiki/Health-Check-Output-Template)
245
+
246
+ ### 3.2 Log Analysis Component
247
+ #### 3.2.1 Examples
248
+ ```bash
249
+ # Full analysis
250
+ ./x-ray log mongodb.log
251
+ # For large logs, analyze a random 10% logs
252
+ ./x-ray log -r 0.1 mongodb.log
253
+ ```
254
+
255
+ #### 3.2.2 Full Arguments
256
+ ```bash
257
+ x-ray log [-h] [-s CHECKSET] [-o OUTPUT] [-f {markdown,html}] [log_file]
258
+ ```
259
+ | Argument | Description | Default |
260
+ | ------------------ | ------------------------------------------------- | :-------: |
261
+ | `-s`, `--checkset` | Checkset to run. | `default` |
262
+ | `-o`, `--output` | Output folder path. | `output/` |
263
+ | `-f`, `--format` | Output format. Can be `markdown` or `html`. | `html` |
264
+ | `-r`, `--rate` | Sample rate. Only analyze a subset of logs. | `1` |
265
+ | `--top` | When analyzing the slow queries, only list top N. | `10` |
@@ -0,0 +1,155 @@
1
+ # x-ray
2
+ [![Makefile](https://github.com/zhangyaoxing/x-ray/actions/workflows/makefile.yml/badge.svg)](https://github.com/zhangyaoxing/x-ray/actions/workflows/makefile.yml)
3
+ [![Release](https://github.com/zhangyaoxing/x-ray/actions/workflows/release.yml/badge.svg)](https://github.com/zhangyaoxing/x-ray/actions/workflows/release.yml)
4
+ [![PyPI](https://img.shields.io/pypi/v/mongo-x-ray.svg)](https://pypi.org/project/mongo-x-ray/)
5
+
6
+
7
+ This project aims to create tools for MongoDB analysis and diagnosis. So far 3 modules are built:
8
+ - Health check module.
9
+ - Log analysis module.
10
+ - `getMongoData` visualization module (Under construction).
11
+
12
+ ## 1 Compatibility Matrix
13
+ ### Health Check
14
+ | Replica Set | Sharded Cluster | Standalone |
15
+ | :-----------: | :-------------: | :--------: |
16
+ | >=4.2 &check; | >=4.2 &check; | &cross; |
17
+
18
+ Older versions are not tested.
19
+
20
+ ### Log Analysis
21
+ Log analysis requires JSON format logs, which is supported since 4.4.
22
+ | Replica Set | Sharded Cluster | Standalone |
23
+ | :-----------: | :-------------: | :-----------: |
24
+ | >=4.4 &check; | >=4.4 &check; | >=4.4 &check; |
25
+
26
+ ## 2 How to Install
27
+ ### 2.1 PyPi
28
+ #### 2.1.1 Install with Pip
29
+ The easiest and recommended way to install x-ray is to use `pip`:
30
+ ```bash
31
+ pip install mongo-x-ray
32
+ ```
33
+
34
+ #### 2.1.2 Build from Source
35
+ ```bash
36
+ git clone https://github.com/zhangyaoxing/x-ray
37
+ cd x-ray
38
+ pip install .
39
+ ```
40
+
41
+ ### 2.2 PyInstaller
42
+ #### 2.2.1 Prebuilt Binaries
43
+ Currently the prebuilt binaries are built on 3 platforms:
44
+ - Ubuntu Latest (AMD64)
45
+ - MacOS Latest (ARM64)
46
+ - Windows Latest (AMD64)
47
+
48
+ Download them from [Releases](https://github.com/zhangyaoxing/x-ray/releases).
49
+
50
+ #### 2.2.2 Build from Source
51
+ x-ray is tested on `Python 3.9.22`. On MacOS or Linux distributions, you can use the `make` command to build the binary:
52
+ ```bash
53
+ git clone https://github.com/zhangyaoxing/x-ray
54
+ cd x-ray
55
+ make deps # if it's the first time you build the project
56
+ make # equal to `make build` and `make build-lite`
57
+ ```
58
+
59
+ There are other make targets. Use `make help` to find out.
60
+
61
+ You can also build the tool with AI modules for log analysis. For more details refer to: [Build with AI Support](https://github.com/zhangyaoxing/x-ray/wiki/Build-with-AI-Support).
62
+
63
+ For Windows users, if `make` command is not available. You can use Python commands to build the binary:
64
+ ```powershell
65
+ python.exe -m venv .venv
66
+ .venv\Scripts\python.exe -m pip install --upgrade pip
67
+ .venv\Scripts\python.exe -m pip install -e ".[dev]"
68
+ .venv\Scripts\python.exe -m PyInstaller --onefile `
69
+ --name x-ray `
70
+ --add-data="templates;templates" `
71
+ --add-data="libs;libs" `
72
+ --icon="misc/x-ray.ico" `
73
+ --hidden-import=openai `
74
+ x-ray
75
+ ```
76
+
77
+ #### 2.3 For Developers
78
+ For developers, use `make deps` to prepare venv and dependencies
79
+ ```bash
80
+ make deps
81
+ ```
82
+ Or
83
+ ```bash
84
+ python3 -m venv .venv
85
+ python3 -m pip install --upgrade pip
86
+ python3 -m pip install -e ".[dev]"
87
+ ```
88
+
89
+ ## 3 Using the Tool
90
+ ```bash
91
+ x-ray [-h] [-q] [-c CONFIG] {healthcheck,hc,log}
92
+ ```
93
+ | Argument | Description | Default |
94
+ | ---------------- | ---------------------------------------------------------------------------------------------- | :-------------------------: |
95
+ | `-q`, `--quiet` | Quiet mode. | `false` |
96
+ | `-h`, `--help` | Show the help message and exit. | n/a |
97
+ | `-c`, `--config` | Path to configuration file. | Built-in `libs/config.json` |
98
+ | `command` | Command to run. Include:<br/>- `healthcheck` or `hc`: Health check.<br/>- `log`: Log analysis. | None |
99
+
100
+ Besides, you can use environment variables to control some behaviors:
101
+ - `ENV=development` For developing. It will change the following behaviors:
102
+ - Formatted the output JSON for for easier reading.
103
+ - The output will not create a new folder for each run but overwrite the same files.
104
+ - `LOG_LEVEL`: Can be `DEBUG`, `ERROR` or `INFO` (default).
105
+
106
+ ### 3.1 Health Check Component
107
+ #### 3.1.1 Examples
108
+ ```bash
109
+ ./x-ray healthcheck localhost:27017 # Scan the cluster with default settings.
110
+ ./x-ray hc localhost:27017 --output ./output/ # Specify output folder.
111
+ ./x-ray hc localhost:27017 --config ./config.json # Use your own configuration.
112
+ ```
113
+
114
+ #### 3.1.2 Full Arguments
115
+ ```bash
116
+ x-ray healthcheck [-h] [-s CHECKSET] [-o OUTPUT] [-f {markdown,html}] [uri]
117
+ ```
118
+ | Argument | Description | Default |
119
+ | ------------------ | ------------------------------------------- | :-------: |
120
+ | `-s`, `--checkset` | Checkset to run. | `default` |
121
+ | `-o`, `--output` | Output folder path. | `output/` |
122
+ | `-f`, `--format` | Output format. Can be `markdown` or `html`. | `html` |
123
+ | `uri` | MongoDB database URI. | None |
124
+
125
+ For security reasons you may not want to include credentials in the command. There are 2 options:
126
+ - If the URI is not provided, user will be asked to input one.
127
+ - If URI is provided but not username/password, user will also be asked to input them.
128
+
129
+ #### 3.1.3 More Info
130
+ Refer to the wiki for more details.
131
+ - [Customize the thresholds](https://github.com/zhangyaoxing/x-ray/wiki/Health-Check-Configuration)
132
+ - [Database permissions](https://github.com/zhangyaoxing/x-ray/wiki/Health-Check-Database-Permissions)
133
+ - [Output](https://github.com/zhangyaoxing/x-ray/wiki/Health-Check-Output)
134
+ - [Customize the output](https://github.com/zhangyaoxing/x-ray/wiki/Health-Check-Output-Template)
135
+
136
+ ### 3.2 Log Analysis Component
137
+ #### 3.2.1 Examples
138
+ ```bash
139
+ # Full analysis
140
+ ./x-ray log mongodb.log
141
+ # For large logs, analyze a random 10% logs
142
+ ./x-ray log -r 0.1 mongodb.log
143
+ ```
144
+
145
+ #### 3.2.2 Full Arguments
146
+ ```bash
147
+ x-ray log [-h] [-s CHECKSET] [-o OUTPUT] [-f {markdown,html}] [log_file]
148
+ ```
149
+ | Argument | Description | Default |
150
+ | ------------------ | ------------------------------------------------- | :-------: |
151
+ | `-s`, `--checkset` | Checkset to run. | `default` |
152
+ | `-o`, `--output` | Output folder path. | `output/` |
153
+ | `-f`, `--format` | Output format. Can be `markdown` or `html`. | `html` |
154
+ | `-r`, `--rate` | Sample rate. Only analyze a subset of logs. | `1` |
155
+ | `--top` | When analyzing the slow queries, only list top N. | `10` |
@@ -0,0 +1 @@
1
+ """All packages"""
@@ -0,0 +1,75 @@
1
+ """AI-related functions for analyzing MongoDB log lines."""
2
+
3
+ import logging
4
+ from libs.utils import green, ai_key
5
+
6
+ MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct"
7
+ MAX_NEW_TOKENS = 256
8
+ GPT_MODEL = "gpt-5"
9
+
10
+ logger = logging.getLogger(__name__)
11
+
12
+
13
+ def detect_device():
14
+ try:
15
+ import torch
16
+
17
+ if torch.cuda.is_available():
18
+ return "cuda"
19
+ if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available():
20
+ return "mps"
21
+ return "cpu"
22
+ except ImportError:
23
+ logger.error("torch is not installed. Please install it with: pip install torch")
24
+ raise
25
+
26
+
27
+ def load_model(model_name):
28
+ try:
29
+ import torch
30
+ from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
31
+ except ImportError as e:
32
+ logger.error("Required AI libraries not installed: %s", e)
33
+ logger.error("Please install with: pip install torch transformers")
34
+ raise
35
+
36
+ device = detect_device()
37
+ logger.info("Using device: %s", green(device))
38
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
39
+ model = AutoModelForCausalLM.from_pretrained(
40
+ model_name,
41
+ device_map="auto",
42
+ dtype=torch.float16 if device != "cpu" else torch.float32,
43
+ )
44
+ model.eval()
45
+ gen_config = GenerationConfig(
46
+ max_new_tokens=MAX_NEW_TOKENS,
47
+ max_length=None,
48
+ do_sample=False,
49
+ eos_token_id=tokenizer.eos_token_id,
50
+ )
51
+ return tokenizer, model, gen_config
52
+
53
+
54
+ def analyze_log_line_local(log_line, tokenizer, model, gen_config):
55
+ prompt = f"Analyze this MongoDB log message and give me the shortest answer: {str(log_line['msg'])}".strip()
56
+ inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
57
+ outputs = model.generate(**inputs, generation_config=gen_config)
58
+ text = tokenizer.decode(outputs[0][inputs["input_ids"].shape[1] :], skip_special_tokens=True).strip()
59
+
60
+ return text
61
+
62
+
63
+ def analyze_log_line_gpt(log_line):
64
+ if ai_key == "":
65
+ logger.warning("No AI API key found. Skipping AI analysis.")
66
+ return ""
67
+ from openai import OpenAI
68
+
69
+ client = OpenAI()
70
+ prompt = f"Analyze this MongoDB log line and give me the shortest answer: {str(log_line)}"
71
+ response = client.responses.create(
72
+ model=GPT_MODEL,
73
+ input=prompt,
74
+ )
75
+ return response.output_text
@@ -0,0 +1,109 @@
1
+ {
2
+ "4.2": {
3
+ "mongoc": [1, 15],
4
+ "mongocxx": [3, 4],
5
+ "mongo-csharp-driver": [2, 0],
6
+ "mongo-go-driver": [1, 11],
7
+ "mongo-java-driver": [100, 0],
8
+ "mongo-java-driver|async": [100, 0],
9
+ "mongo-java-driver|legacy": [4, 1],
10
+ "mongo-java-driver|sync": [4, 1],
11
+ "mongo-java-driver|reactive-streams": [4, 7],
12
+ "mongo-kotlin-driver": [4, 10],
13
+ "nodejs": [3, 6],
14
+ "ext-mongodb:PHP": [1, 15],
15
+ "PyMongo": [3, 7],
16
+ "mongo-ruby-driver": [8, 0],
17
+ "mongo-rust-driver": [2, 0],
18
+ "mongo-scala-driver": [4, 7]
19
+ },
20
+ "4.4": {
21
+ "mongoc": [1, 17],
22
+ "mongocxx": [3, 6],
23
+ "mongo-csharp-driver": [2, 7],
24
+ "mongo-go-driver": [1, 11],
25
+ "mongo-java-driver": [100, 0],
26
+ "mongo-java-driver|async": [100, 0],
27
+ "mongo-java-driver|legacy": [4, 2],
28
+ "mongo-java-driver|sync": [4, 2],
29
+ "mongo-java-driver|reactive-streams": [4, 7],
30
+ "mongo-kotlin-driver": [4, 10],
31
+ "nodejs": [3, 7],
32
+ "ext-mongodb:PHP": [1, 15],
33
+ "PyMongo": [3, 7],
34
+ "mongo-ruby-driver": [8, 0],
35
+ "mongo-rust-driver": [2, 0],
36
+ "mongo-scala-driver": [4, 7]
37
+ },
38
+ "5.0": {
39
+ "mongoc": [1, 18],
40
+ "mongocxx": [3, 7],
41
+ "mongo-csharp-driver": [2, 15],
42
+ "mongo-go-driver": [1, 11],
43
+ "mongo-java-driver": [100, 0],
44
+ "mongo-java-driver|async": [100, 0],
45
+ "mongo-java-driver|legacy": [4, 3],
46
+ "mongo-java-driver|sync": [4, 3],
47
+ "mongo-java-driver|reactive-streams": [4, 7],
48
+ "mongo-kotlin-driver": [4, 10],
49
+ "nodejs": [3, 7],
50
+ "ext-mongodb:PHP": [1, 15],
51
+ "PyMongo": [3, 7],
52
+ "mongo-ruby-driver": [8, 0],
53
+ "mongo-rust-driver": [2, 0],
54
+ "mongo-scala-driver": [4, 7]
55
+ },
56
+ "6.0": {
57
+ "mongoc": [1, 22],
58
+ "mongocxx": [3, 8],
59
+ "mongo-csharp-driver": [2, 16],
60
+ "mongo-go-driver": [1, 11],
61
+ "mongo-java-driver": [100, 0],
62
+ "mongo-java-driver|async": [100, 0],
63
+ "mongo-java-driver|legacy": [4, 7],
64
+ "mongo-java-driver|sync": [4, 7],
65
+ "mongo-java-driver|reactive-streams": [4, 7],
66
+ "mongo-kotlin-driver": [4, 10],
67
+ "nodejs": [4, 8],
68
+ "PyMongo": [4, 2],
69
+ "mongo-ruby-driver": [8, 0],
70
+ "mongo-rust-driver": [2, 6],
71
+ "mongo-scala-driver": [4, 7]
72
+ },
73
+ "7.0": {
74
+ "mongoc": [1, 24],
75
+ "mongocxx": [3, 8],
76
+ "mongo-csharp-driver": [2, 20],
77
+ "mongo-go-driver": [1, 12],
78
+ "mongo-java-driver": [100, 0],
79
+ "mongo-java-driver|async": [100, 0],
80
+ "mongo-java-driver|legacy": [4, 10],
81
+ "mongo-java-driver|sync": [4, 10],
82
+ "mongo-java-driver|reactive-streams": [4, 10],
83
+ "mongo-kotlin-driver": [4, 10],
84
+ "nodejs": [5, 7],
85
+ "ext-mongodb:PHP": [1, 16],
86
+ "PyMongo": [4, 4],
87
+ "mongo-ruby-driver": [8, 0],
88
+ "mongo-rust-driver": [2, 6],
89
+ "mongo-scala-driver": [4, 10]
90
+ },
91
+ "8.0": {
92
+ "mongoc": [1, 28],
93
+ "mongocxx": [3, 11],
94
+ "mongo-csharp-driver": [2, 29],
95
+ "mongo-go-driver": [2, 1],
96
+ "mongo-java-driver": [100, 0],
97
+ "mongo-java-driver|async": [100, 0],
98
+ "mongo-java-driver|legacy": [5, 2],
99
+ "mongo-java-driver|sync": [5, 2],
100
+ "mongo-java-driver|reactive-streams": [5, 2],
101
+ "mongo-kotlin-driver": [5, 2],
102
+ "nodejs": [6, 9],
103
+ "ext-mongodb:PHP": [1, 20],
104
+ "PyMongo": [4, 9],
105
+ "mongo-ruby-driver": [9, 0],
106
+ "mongo-rust-driver": [3, 1],
107
+ "mongo-scala-driver": [5, 2]
108
+ }
109
+ }