gool 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gool-1.0.0/.gitignore +5 -0
- gool-1.0.0/.pre-commit-config.yaml +22 -0
- gool-1.0.0/.vscode/launch.json +25 -0
- gool-1.0.0/.vscode/settings.json +40 -0
- gool-1.0.0/.vscode/tasks.json +28 -0
- gool-1.0.0/CONTRIBUTING.md +126 -0
- gool-1.0.0/LICENSE +21 -0
- gool-1.0.0/Makefile +54 -0
- gool-1.0.0/PKG-INFO +54 -0
- gool-1.0.0/README.md +29 -0
- gool-1.0.0/codecov.yaml +9 -0
- gool-1.0.0/docs/configuration.md +21 -0
- gool-1.0.0/docs/index.md +89 -0
- gool-1.0.0/docs/info.md +1 -0
- gool-1.0.0/docs/license.md +1 -0
- gool-1.0.0/docs/modules.md +1 -0
- gool-1.0.0/mkdocs.yml +55 -0
- gool-1.0.0/pyproject.toml +154 -0
- gool-1.0.0/src/gool/__init__.py +0 -0
- gool-1.0.0/src/gool/_version.py +24 -0
- gool-1.0.0/src/gool/log_clustering.py +427 -0
- gool-1.0.0/tests/data/drain3.ini +14 -0
- gool-1.0.0/tests/data/log/Apache_2k.log +2000 -0
- gool-1.0.0/tests/data/log/Zookeeper_2k.log +2000 -0
- gool-1.0.0/tests/data/log/empty.log +0 -0
- gool-1.0.0/tests/data/results/6times_Zookeeper_2k_raw_output.txt +135 -0
- gool-1.0.0/tests/data/results/Apache_2k_raw_output.txt +25 -0
- gool-1.0.0/tests/data/results/Zookeeper_2k_depth10.txt +135 -0
- gool-1.0.0/tests/data/results/Zookeeper_2k_lex_order.txt +135 -0
- gool-1.0.0/tests/data/results/Zookeeper_2k_raw_output.txt +135 -0
- gool-1.0.0/tests/data/results/Zookeeper_2k_sim_thres_5.txt +40 -0
- gool-1.0.0/tests/data/results/Zookeeper_2k_size_order.txt +135 -0
- gool-1.0.0/tests/data/results/Zookeeper_2k_warn.txt +37 -0
- gool-1.0.0/tests/test_gool.py +213 -0
- gool-1.0.0/uv.lock +1557 -0
gool-1.0.0/.gitignore
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
3
|
+
rev: "v6.0.0"
|
|
4
|
+
hooks:
|
|
5
|
+
- id: check-case-conflict
|
|
6
|
+
- id: check-merge-conflict
|
|
7
|
+
- id: check-toml
|
|
8
|
+
- id: check-yaml
|
|
9
|
+
- id: check-json
|
|
10
|
+
exclude: ^.devcontainer/devcontainer.json
|
|
11
|
+
- id: pretty-format-json
|
|
12
|
+
exclude: ^.devcontainer/devcontainer.json
|
|
13
|
+
args: [--autofix, --no-sort-keys]
|
|
14
|
+
- id: end-of-file-fixer
|
|
15
|
+
- id: trailing-whitespace
|
|
16
|
+
|
|
17
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
18
|
+
rev: "v0.14.8"
|
|
19
|
+
hooks:
|
|
20
|
+
- id: ruff-check
|
|
21
|
+
args: [ --exit-non-zero-on-fix ]
|
|
22
|
+
- id: ruff-format
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "0.2.0",
|
|
3
|
+
"configurations": [
|
|
4
|
+
{
|
|
5
|
+
"name": "gool sample",
|
|
6
|
+
"type": "debugpy",
|
|
7
|
+
"request": "launch",
|
|
8
|
+
"module": "gool.log_clustering",
|
|
9
|
+
"args": [
|
|
10
|
+
"--cfg-file",
|
|
11
|
+
"${workspaceFolder}/tests/data/drain3.ini",
|
|
12
|
+
"--similarity-threshold",
|
|
13
|
+
"0.65",
|
|
14
|
+
"${workspaceFolder}/tests/data/log/Zookeeper_2k.log"
|
|
15
|
+
]
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"name": "make install",
|
|
19
|
+
"type": "node-terminal",
|
|
20
|
+
"request": "launch",
|
|
21
|
+
"command": "exit",
|
|
22
|
+
"preLaunchTask": "install"
|
|
23
|
+
}
|
|
24
|
+
]
|
|
25
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
{
|
|
2
|
+
"python.defaultInterpreterPath": "./.venv/bin/python",
|
|
3
|
+
"python.testing.pytestEnabled": true,
|
|
4
|
+
"python.testing.unittestEnabled": false,
|
|
5
|
+
"python.testing.pytestArgs": [
|
|
6
|
+
"tests"
|
|
7
|
+
],
|
|
8
|
+
"[python]": {
|
|
9
|
+
"editor.defaultFormatter": "charliermarsh.ruff",
|
|
10
|
+
"editor.formatOnSave": true
|
|
11
|
+
},
|
|
12
|
+
"cSpell.words": [
|
|
13
|
+
"addopts",
|
|
14
|
+
"deptry",
|
|
15
|
+
"docstrings",
|
|
16
|
+
"doctest",
|
|
17
|
+
"envlist",
|
|
18
|
+
"envpython",
|
|
19
|
+
"gool",
|
|
20
|
+
"isort",
|
|
21
|
+
"mccabe",
|
|
22
|
+
"mkdocs",
|
|
23
|
+
"mkdocstrings",
|
|
24
|
+
"mypy",
|
|
25
|
+
"passenv",
|
|
26
|
+
"pycodestyle",
|
|
27
|
+
"pyflakes",
|
|
28
|
+
"pygrep",
|
|
29
|
+
"pyproject",
|
|
30
|
+
"pytest",
|
|
31
|
+
"pyupgrade",
|
|
32
|
+
"skipsdist",
|
|
33
|
+
"strerror",
|
|
34
|
+
"surrogateescape",
|
|
35
|
+
"testenv",
|
|
36
|
+
"testpaths",
|
|
37
|
+
"tryceratops",
|
|
38
|
+
"urandom"
|
|
39
|
+
]
|
|
40
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "2.0.0",
|
|
3
|
+
"tasks": [
|
|
4
|
+
{
|
|
5
|
+
"label": "install",
|
|
6
|
+
"type": "shell",
|
|
7
|
+
"command": "make",
|
|
8
|
+
"isBackground": false,
|
|
9
|
+
"problemMatcher": [],
|
|
10
|
+
"args": [
|
|
11
|
+
"install"
|
|
12
|
+
],
|
|
13
|
+
"group": {
|
|
14
|
+
"kind": "build",
|
|
15
|
+
"isDefault": true
|
|
16
|
+
},
|
|
17
|
+
"presentation": {
|
|
18
|
+
"echo": true,
|
|
19
|
+
"reveal": "always",
|
|
20
|
+
"focus": true,
|
|
21
|
+
"panel": "dedicated",
|
|
22
|
+
"showReuseMessage": true,
|
|
23
|
+
"clear": false,
|
|
24
|
+
"close": true
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
]
|
|
28
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# Contributing to `gool`
|
|
2
|
+
|
|
3
|
+
Contributions are welcome, and they are greatly appreciated!
|
|
4
|
+
Every little bit helps, and credit will always be given.
|
|
5
|
+
|
|
6
|
+
You can contribute in many ways:
|
|
7
|
+
|
|
8
|
+
# Types of Contributions
|
|
9
|
+
|
|
10
|
+
## Report Bugs
|
|
11
|
+
|
|
12
|
+
Report bugs at https://github.com/leCapi/gool/issues
|
|
13
|
+
|
|
14
|
+
If you are reporting a bug, please include:
|
|
15
|
+
|
|
16
|
+
- Your operating system name and version.
|
|
17
|
+
- Any details about your local setup that might be helpful in troubleshooting.
|
|
18
|
+
- Detailed steps to reproduce the bug.
|
|
19
|
+
|
|
20
|
+
## Fix Bugs
|
|
21
|
+
|
|
22
|
+
Look through the GitHub issues for bugs.
|
|
23
|
+
Anything tagged with "bug" and "help wanted" is open to whoever wants to implement a fix for it.
|
|
24
|
+
|
|
25
|
+
## Implement Features
|
|
26
|
+
|
|
27
|
+
Look through the GitHub issues for features.
|
|
28
|
+
Anything tagged with "enhancement" and "help wanted" is open to whoever wants to implement it.
|
|
29
|
+
|
|
30
|
+
## Write Documentation
|
|
31
|
+
|
|
32
|
+
gool could always use more documentation, whether as part of the official docs, in docstrings, or even on the web in blog posts, articles, and such.
|
|
33
|
+
|
|
34
|
+
## Submit Feedback
|
|
35
|
+
|
|
36
|
+
The best way to send feedback is to file an issue at https://github.com/leCapi/gool/issues.
|
|
37
|
+
|
|
38
|
+
If you are proposing a new feature:
|
|
39
|
+
|
|
40
|
+
- Explain in detail how it would work.
|
|
41
|
+
- Keep the scope as narrow as possible, to make it easier to implement.
|
|
42
|
+
- Remember that this is a volunteer-driven project, and that contributions
|
|
43
|
+
are welcome :)
|
|
44
|
+
|
|
45
|
+
# Get Started!
|
|
46
|
+
|
|
47
|
+
Ready to contribute? Here's how to set up `gool` for local development.
|
|
48
|
+
Please note this documentation assumes you already have `uv` and `Git` installed and ready to go.
|
|
49
|
+
|
|
50
|
+
1. Fork the `gool` repo on GitHub.
|
|
51
|
+
|
|
52
|
+
2. Clone your fork locally:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
cd <directory_in_which_repo_should_be_created>
|
|
56
|
+
git clone git@github.com:YOUR_NAME/gool.git
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
3. Now we need to install the environment. Navigate into the directory
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
cd gool
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Then, install and activate the environment with:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
uv sync
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
4. Install pre-commit to run linters/formatters at commit time:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
uv run pre-commit install
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
5. Create a branch for local development:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
git checkout -b name-of-your-bugfix-or-feature
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Now you can make your changes locally.
|
|
84
|
+
|
|
85
|
+
6. Don't forget to add test cases for your added functionality to the `tests` directory.
|
|
86
|
+
|
|
87
|
+
7. When you're done making changes, check that your changes pass the formatting tests.
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
make check
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Now, validate that all unit tests are passing:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
make test
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
9. Before raising a pull request you should also run tox.
|
|
100
|
+
This will run the tests across different versions of Python:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
tox
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
This requires you to have multiple versions of python installed.
|
|
107
|
+
This step is also triggered in the CI/CD pipeline, so you could also choose to skip this step locally.
|
|
108
|
+
|
|
109
|
+
10. Commit your changes and push your branch to GitHub:
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
git add .
|
|
113
|
+
git commit -m "Your detailed description of your changes."
|
|
114
|
+
git push origin name-of-your-bugfix-or-feature
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
11. Submit a pull request through the GitHub website.
|
|
118
|
+
|
|
119
|
+
# Pull Request Guidelines
|
|
120
|
+
|
|
121
|
+
Before you submit a pull request, check that it meets these guidelines:
|
|
122
|
+
|
|
123
|
+
1. The pull request should include tests.
|
|
124
|
+
|
|
125
|
+
2. If the pull request adds functionality, the docs should be updated.
|
|
126
|
+
Put your new functionality into a function with a docstring, and add the feature to the list in `README.md`.
|
gool-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Olivier GODARD
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
gool-1.0.0/Makefile
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
.PHONY: install
|
|
2
|
+
install: ## Install the virtual environment and install the pre-commit hooks
|
|
3
|
+
@echo "🚀 Creating virtual environment using uv"
|
|
4
|
+
@uv sync
|
|
5
|
+
@uv run pre-commit install
|
|
6
|
+
|
|
7
|
+
.PHONY: check
|
|
8
|
+
check: ## Run code quality tools.
|
|
9
|
+
@echo "🚀 Checking lock file consistency with 'pyproject.toml'"
|
|
10
|
+
@uv lock --locked
|
|
11
|
+
@echo "🚀 Linting code: Running pre-commit"
|
|
12
|
+
@uv run pre-commit run -a
|
|
13
|
+
@echo "🚀 Static type checking: Running mypy"
|
|
14
|
+
@uv run mypy
|
|
15
|
+
@echo "🚀 Checking for obsolete dependencies: Running deptry"
|
|
16
|
+
@uv run deptry src
|
|
17
|
+
|
|
18
|
+
.PHONY: test
|
|
19
|
+
test: ## Test the code with pytest
|
|
20
|
+
@echo "🚀 Testing code: Running pytest"
|
|
21
|
+
@uv run python -m pytest --cov --cov-config=pyproject.toml
|
|
22
|
+
|
|
23
|
+
.PHONY: build
|
|
24
|
+
build: clean-build ## Build wheel file
|
|
25
|
+
@echo "🚀 Creating wheel file"
|
|
26
|
+
@uvx --from build pyproject-build --installer uv
|
|
27
|
+
|
|
28
|
+
.PHONY: clean-build
|
|
29
|
+
clean-build: ## Clean build artifacts
|
|
30
|
+
@echo "🚀 Removing build artifacts"
|
|
31
|
+
@uv run python -c "import shutil; import os; shutil.rmtree('dist') if os.path.exists('dist') else None"
|
|
32
|
+
|
|
33
|
+
.PHONY: publish
|
|
34
|
+
publish: ## Publish a release to PyPI.
|
|
35
|
+
@echo "🚀 Publishing."
|
|
36
|
+
@uvx twine upload --repository-url https://upload.pypi.org/legacy/ dist/*
|
|
37
|
+
|
|
38
|
+
.PHONY: build-and-publish
|
|
39
|
+
build-and-publish: build publish ## Build and publish.
|
|
40
|
+
|
|
41
|
+
.PHONY: docs-test
|
|
42
|
+
docs-test: ## Test if documentation can be built without warnings or errors
|
|
43
|
+
@uv run mkdocs build -s
|
|
44
|
+
|
|
45
|
+
.PHONY: docs
|
|
46
|
+
docs: ## Build and serve the documentation
|
|
47
|
+
@uv run mkdocs serve
|
|
48
|
+
|
|
49
|
+
.PHONY: help
|
|
50
|
+
help:
|
|
51
|
+
@uv run python -c "import re; \
|
|
52
|
+
[[print(f'\033[36m{m[0]:<20}\033[0m {m[1]}') for m in re.findall(r'^([a-zA-Z_-]+):.*?## (.*)$$', open(makefile).read(), re.M)] for makefile in ('$(MAKEFILE_LIST)').strip().split()]"
|
|
53
|
+
|
|
54
|
+
.DEFAULT_GOAL := help
|
gool-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: gool
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: cli logs clustering
|
|
5
|
+
Project-URL: Homepage, https://github.com/leCapi/gool
|
|
6
|
+
Project-URL: Repository, https://github.com/leCapi/gool
|
|
7
|
+
Project-URL: Documentation, https://github.com/leCapi/gool
|
|
8
|
+
Author: Olivier GODARD
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: python
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
20
|
+
Requires-Python: <4.0,>=3.10
|
|
21
|
+
Requires-Dist: drain3~=0.9
|
|
22
|
+
Requires-Dist: rich~=14.0
|
|
23
|
+
Requires-Dist: tyro<=1.0.8,>=0.9
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# gool, repo usage
|
|
27
|
+
|
|
28
|
+
The gool repository uses uv and git. Version is taken from git tag. The repo provide setup for VSCode.
|
|
29
|
+
|
|
30
|
+
## Setup
|
|
31
|
+
|
|
32
|
+
Below the more useful commands.
|
|
33
|
+
|
|
34
|
+
setup the uv virtual environment and install pre-commit hooks :
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
make install
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Generate and launch the documentation server :
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
make docs
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
All commands:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
make help
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
Repository initiated with [fpgmaas/cookiecutter-uv](https://github.com/fpgmaas/cookiecutter-uv).
|
gool-1.0.0/README.md
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# gool, repo usage
|
|
2
|
+
|
|
3
|
+
The gool repository uses uv and git. Version is taken from git tag. The repo provide setup for VSCode.
|
|
4
|
+
|
|
5
|
+
## Setup
|
|
6
|
+
|
|
7
|
+
Below the more useful commands.
|
|
8
|
+
|
|
9
|
+
setup the uv virtual environment and install pre-commit hooks :
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
make install
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Generate and launch the documentation server :
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
make docs
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
All commands:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
make help
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
Repository initiated with [fpgmaas/cookiecutter-uv](https://github.com/fpgmaas/cookiecutter-uv).
|
gool-1.0.0/codecov.yaml
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Example of a drain3.ini file
|
|
2
|
+
|
|
3
|
+
gool try to load .drain3 file from your home. Otherwise you can use **--cfg-file** option. Without any configuration, gool will try to mask HEX, IP and times.
|
|
4
|
+
|
|
5
|
+
``` ini
|
|
6
|
+
[MASKING]
|
|
7
|
+
masking = [
|
|
8
|
+
{"regex_pattern":"((?<=[^A-Za-z0-9])|^)(0[xX][0-9a-fA-F]+)((?=[^A-Za-z0-9])|$)", "mask_with": "HEX"},
|
|
9
|
+
{"regex_pattern":"(\\d{2}:\\d{2}:\\d{2}(.\\d+|))", "mask_with": "TIME"},
|
|
10
|
+
{"regex_pattern":"((?<=[^A-Za-z0-9])|^)(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})((?=[^A-Za-z0-9])|$)", "mask_with": "IP"}
|
|
11
|
+
]
|
|
12
|
+
mask_prefix = <:
|
|
13
|
+
mask_suffix = :>
|
|
14
|
+
|
|
15
|
+
[DRAIN]
|
|
16
|
+
sim_th = 0.9
|
|
17
|
+
depth = 7
|
|
18
|
+
max_children = 200
|
|
19
|
+
extra_delimiters = ["_"]
|
|
20
|
+
|
|
21
|
+
```
|
gool-1.0.0/docs/index.md
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# gool
|
|
2
|
+
|
|
3
|
+
<!-- [](https://img.shields.io/github/v/release/leCapi/gool)
|
|
4
|
+
[](https://github.com/leCapi/gool/actions/workflows/main.yml?query=branch%3Amain)
|
|
5
|
+
[](https://img.shields.io/github/commit-activity/m/leCapi/gool)
|
|
6
|
+
[](https://img.shields.io/github/license/leCapi/gool) -->
|
|
7
|
+
|
|
8
|
+
gool is a cli logs clustering tool using drain3 library. It takes one or more log files as input and output clusters of your logs.
|
|
9
|
+
|
|
10
|
+
The user can tune some settings to build right size clusters :
|
|
11
|
+
|
|
12
|
+
- **similarity-threshold** :
|
|
13
|
+
drain3 parameter (default 0.4) between 0 and 1. Value of 1 will lead to cluster with the same exact lines only. The higher, the more clusters you will get. This setting can be tunned in configuration file or on command line.
|
|
14
|
+
- **tree-depth** :
|
|
15
|
+
drain3 parameter (default 4) which high value lead to more clusters. This setting can be tunned in configuration file or on command line.
|
|
16
|
+
- **pattern masking** :
|
|
17
|
+
drain3 parameter to mask pattern so lines can be group easier. For example we can replace all IP, or time before processing. This setting can only be tunned in configuration file.
|
|
18
|
+
- **filter** :
|
|
19
|
+
filter to only parse some of the line of the input files. For example if the user is focuses on error we can imagine something like : '.*(\| Warning |\| Error ).*'.
|
|
20
|
+
|
|
21
|
+
For more details on drain3 parameters check the official repository :
|
|
22
|
+
https://github.com/logpai/Drain3.
|
|
23
|
+
|
|
24
|
+
Launch gool (config taken in ~/.drain3.ini):
|
|
25
|
+
```
|
|
26
|
+
gool tests/data/log/Zookeeper_2k.log
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Launch gool and filter lines before processing (config taken in ~/.drain3.ini):
|
|
30
|
+
```
|
|
31
|
+
gool tests/data/log/Zookeeper_2k.log -f ".*WARN.*"
|
|
32
|
+
```
|
|
33
|
+
which produces something like :
|
|
34
|
+
```
|
|
35
|
+
11:34:48.805482 INFO log_clustering : Loading configuration from /home/godardo/.drain3.ini log_clustering.py:111
|
|
36
|
+
11:34:48.829732 INFO log_clustering : Processed 1318 lines in 0.02 seconds (64798.84 lines/second). log_clustering.py:226
|
|
37
|
+
Zookeeper_2k.log ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100%
|
|
38
|
+
Log Clusters
|
|
39
|
+
┏━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
|
40
|
+
┃ Count ┃ Char Size (KB) ┃ Template ┃
|
|
41
|
+
┡━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
|
|
42
|
+
│ 313 │ 43 │ 2015-07-29 <TIME> - WARN [SendWorker:188978561024:QuorumCnxManager$SendWorker@679] - Interrupted while waiting for message on queue │
|
|
43
|
+
│ 289 │ 43 │ 2015-07-29 <TIME> - WARN [RecvWorker:188978561024:QuorumCnxManager$RecvWorker@762] - Connection broken for id 188978561024, my id = <*> error │
|
|
44
|
+
│ │ │ = │
|
|
45
|
+
│ 265 │ 30 │ 2015-07-29 <TIME> - WARN [RecvWorker:188978561024:QuorumCnxManager$RecvWorker@765] - Interrupting SendWorker │
|
|
46
|
+
│ 262 │ 31 │ 2015-07-29 <TIME> - WARN <*> - Send worker leaving thread │
|
|
47
|
+
│ 45 │ 7 │ <*> <TIME> - WARN [QuorumPeer[myid=1]/0:0:0:0:0:0:0:0:2181:QuorumCnxManager@368] - Cannot open channel to 2 at election address /<IP>:3888 │
|
|
48
|
+
│ 39 │ 7 │ <*> <TIME> - WARN [NIOServerCxn.Factory:<IP>/<IP>:2181:ZooKeeperServer@793] - Connection request from old client <*> will be dropped if │
|
|
49
|
+
│ │ │ server is in r-o mode │
|
|
50
|
+
│ 37 │ 4 │ <*> <TIME> - WARN [NIOServerCxn.Factory:<IP>/<IP>:2181:NIOServerCnxn@349] - caught end of stream exception │
|
|
51
|
+
│ 22 │ 3 │ 2015-08-25 <TIME> - WARN <*> - Cannot open channel to 3 at election address /<IP>:3888 │
|
|
52
|
+
│ 14 │ 2 │ 2015-08-24 <TIME> - WARN <*> - Cannot open channel to 3 at election address /<IP>:3888 │
|
|
53
|
+
│ 3 │ 0 │ 2015-07-30 <TIME> - WARN [WorkerSender[myid=1]:QuorumCnxManager@368] - Cannot open channel to <*> at election address /<IP>:3888 │
|
|
54
|
+
│ 3 │ 0 │ 2015-08-20 <TIME> - WARN [NIOServerCxn.Factory:<IP>/<IP>:2181:NIOServerCnxn@354] - Exception causing close of session <HEX> due to │
|
|
55
|
+
│ │ │ java.io.IOException: ZooKeeperServer not running │
|
|
56
|
+
│ 1 │ 0 │ 2015-07-30 <TIME> - WARN [LearnerHandler-/<IP>:35276:LearnerHandler@575] - ******* GOODBYE /<IP>:35276 ******** │
|
|
57
|
+
│ 1 │ 0 │ 2015-07-30 <TIME> - WARN [RecvWorker:3:QuorumCnxManager$RecvWorker@765] - Interrupting SendWorker │
|
|
58
|
+
│ 1 │ 0 │ 2015-08-25 <TIME> - WARN [RecvWorker:3:QuorumCnxManager$RecvWorker@762] - Connection broken for id 3, my id = 1, error = │
|
|
59
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:52264:LearnerHandler@575] - ******* GOODBYE /<IP>:52264 ******** │
|
|
60
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:52308:LearnerHandler@575] - ******* GOODBYE /<IP>:52308 ******** │
|
|
61
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:59060:LearnerHandler@575] - ******* GOODBYE /<IP>:59060 ******** │
|
|
62
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:59103:LearnerHandler@575] - ******* GOODBYE /<IP>:59103 ******** │
|
|
63
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:57247:LearnerHandler@575] - ******* GOODBYE /<IP>:57247 ******** │
|
|
64
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:57319:LearnerHandler@575] - ******* GOODBYE /<IP>:57319 ******** │
|
|
65
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:52476:LearnerHandler@575] - ******* GOODBYE /<IP>:52476 ******** │
|
|
66
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:59203:LearnerHandler@575] - ******* GOODBYE /<IP>:59203 ******** │
|
|
67
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:59211:LearnerHandler@575] - ******* GOODBYE /<IP>:59211 ******** │
|
|
68
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:52502:LearnerHandler@575] - ******* GOODBYE /<IP>:52502 ******** │
|
|
69
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:57458:LearnerHandler@575] - ******* GOODBYE /<IP>:57458 ******** │
|
|
70
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:57502:LearnerHandler@575] - ******* GOODBYE /<IP>:57502 ******** │
|
|
71
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:52703:LearnerHandler@575] - ******* GOODBYE /<IP>:52703 ******** │
|
|
72
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:59406:LearnerHandler@575] - ******* GOODBYE /<IP>:59406 ******** │
|
|
73
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:57580:LearnerHandler@575] - ******* GOODBYE /<IP>:57580 ******** │
|
|
74
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:52818:LearnerHandler@575] - ******* GOODBYE /<IP>:52818 ******** │
|
|
75
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:52844:LearnerHandler@575] - ******* GOODBYE /<IP>:52844 ******** │
|
|
76
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [LearnerHandler-/<IP>:57653:LearnerHandler@575] - ******* GOODBYE /<IP>:57653 ******** │
|
|
77
|
+
│ 1 │ 0 │ 2015-07-31 <TIME> - WARN [SendWorker:1:QuorumCnxManager$SendWorker@679] - Interrupted while waiting for message on queue │
|
|
78
|
+
│ 1 │ 0 │ 2015-08-07 <TIME> - WARN [QuorumPeer[myid=2]/0:0:0:0:0:0:0:0:2181:QuorumCnxManager@368] - Cannot open channel to 3 at election address │
|
|
79
|
+
│ │ │ /<IP>:3888 │
|
|
80
|
+
│ 1 │ 0 │ 2015-08-20 <TIME> - WARN [LearnerHandler-/<IP>:42241:Leader@576] - First is <HEX> │
|
|
81
|
+
│ 1 │ 0 │ 2015-07-29 <TIME> - WARN [WorkerSender[myid=3]:QuorumCnxManager@368] - Cannot open channel to 2 at election address /<IP>:3888 │
|
|
82
|
+
│ 1 │ 0 │ 2015-07-30 <TIME> - WARN [RecvWorker:1:QuorumCnxManager$RecvWorker@762] - Connection broken for id 1, my id = 3, error = │
|
|
83
|
+
└───────┴────────────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
More option details with :
|
|
87
|
+
```
|
|
88
|
+
gool --help
|
|
89
|
+
```
|
gool-1.0.0/docs/info.md
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
../README.md
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
../LICENSE
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
::: gool.log_clustering
|
gool-1.0.0/mkdocs.yml
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
site_name: gool
|
|
2
|
+
repo_url: https://github.com/leCapi/gool
|
|
3
|
+
site_url: https://github.com/leCapi/gool
|
|
4
|
+
site_description: cli logs clustering
|
|
5
|
+
site_author: Olivier GODARD
|
|
6
|
+
edit_uri: edit/main/docs/
|
|
7
|
+
repo_name: leCapi/gool
|
|
8
|
+
copyright: Maintained by <a href="https://github.com/leCapi/gool">leCapi</a>.
|
|
9
|
+
|
|
10
|
+
nav:
|
|
11
|
+
- Home: index.md
|
|
12
|
+
- Configuration: configuration.md
|
|
13
|
+
- Modules: modules.md
|
|
14
|
+
- Contribute: info.md
|
|
15
|
+
- License: license.md
|
|
16
|
+
plugins:
|
|
17
|
+
- search
|
|
18
|
+
- mkdocstrings:
|
|
19
|
+
handlers:
|
|
20
|
+
python:
|
|
21
|
+
paths: ["src/gool"]
|
|
22
|
+
theme:
|
|
23
|
+
name: material
|
|
24
|
+
feature:
|
|
25
|
+
tabs: true
|
|
26
|
+
palette:
|
|
27
|
+
- media: "(prefers-color-scheme: light)"
|
|
28
|
+
scheme: default
|
|
29
|
+
primary: white
|
|
30
|
+
accent: deep orange
|
|
31
|
+
toggle:
|
|
32
|
+
icon: material/brightness-7
|
|
33
|
+
name: Switch to dark mode
|
|
34
|
+
- media: "(prefers-color-scheme: dark)"
|
|
35
|
+
scheme: slate
|
|
36
|
+
primary: black
|
|
37
|
+
accent: deep orange
|
|
38
|
+
toggle:
|
|
39
|
+
icon: material/brightness-4
|
|
40
|
+
name: Switch to light mode
|
|
41
|
+
icon:
|
|
42
|
+
repo: fontawesome/brands/github
|
|
43
|
+
|
|
44
|
+
extra:
|
|
45
|
+
social:
|
|
46
|
+
- icon: fontawesome/brands/github
|
|
47
|
+
link: https://github.com/leCapi/gool
|
|
48
|
+
- icon: fontawesome/brands/python
|
|
49
|
+
link: https://pypi.org/project/gool
|
|
50
|
+
|
|
51
|
+
markdown_extensions:
|
|
52
|
+
- toc:
|
|
53
|
+
permalink: true
|
|
54
|
+
- pymdownx.arithmatex:
|
|
55
|
+
generic: true
|