gool 1.3.2__tar.gz → 1.3.3__tar.gz

{gool-1.3.2 → gool-1.3.3}/.vscode/launch.json

@@ -14,6 +14,21 @@
         "${workspaceFolder}/tests/data/log/Zookeeper_2k.log"
       ]
     },
+    {
+      "name": "gool sample crafted logs",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "gool.log_clustering",
+      "args": [
+        "--cfg-file",
+        "${workspaceFolder}/tests/data/drain3.ini",
+        "--similarity-threshold",
+        "0.65",
+        "-m",
+        "13:00:00",
+        "${workspaceFolder}/tests/data/log/Crafted_logs.log"
+      ]
+    },
     {
       "name": "gool sample apache date/time filtered",
       "type": "debugpy",

{gool-1.3.2 → gool-1.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gool
-Version: 1.3.2
+Version: 1.3.3
 Summary: cli logs clustering
 Project-URL: Homepage, https://gitlab.com/godardo/gool
 Project-URL: Repository, https://gitlab.com/godardo/gool

{gool-1.3.2 → gool-1.3.3}/src/gool/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.3.2'
-__version_tuple__ = version_tuple = (1, 3, 2)
+__version__ = version = '1.3.3'
+__version_tuple__ = version_tuple = (1, 3, 3)
 
 __commit_id__ = commit_id = None

{gool-1.3.2 → gool-1.3.3}/src/gool/arguments_parser.py

@@ -204,10 +204,16 @@ def guess_time_regexp_and_format_code(time_str: str) -> tuple[str, str] | None:
     Returns:
         A tuple containing the guessed regex pattern and format code.
     """
-    time_regex = r"(\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?)"
-    time_format = "%H:%M:%S.%f"
+    time_regex = r"(\d{2}:\d{2}:\d{2})"
+    time_format = "%H:%M:%S"
     regex = re.compile(time_regex)
     match = regex.fullmatch(time_str)
+
+    if not match:
+        time_regex = r"(\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?)"
+        time_format = "%H:%M:%S.%f"
+        regex = re.compile(time_regex)
+        match = regex.fullmatch(time_str)
     if not match:
         time_regex = r"(\d{2}:\d{2}:\d{2}(?:,\d{1,9})?)"
         time_format = "%H:%M:%S,%f"
@@ -218,9 +224,6 @@ def guess_time_regexp_and_format_code(time_str: str) -> tuple[str, str] | None:
     matched_time = match.group(1)
     with contextlib.suppress(ValueError):
         datetime.strptime(matched_time, time_format)
-        return time_regex, time_format
-    with contextlib.suppress(ValueError):
-        time_format = "%H:%M:%S"
-        datetime.strptime(matched_time, time_format)
-        return time_regex, time_format
+        return r".*" + time_regex + r".*", time_format
+
     return None

{gool-1.3.2 → gool-1.3.3}/src/gool/log_clustering.py

@@ -171,6 +171,8 @@ def display_diff_baseline(
 
     def display_data_rich(list_data: list[str], header: str) -> None:
         table = Table(title=header, highlight=True)
+        min_width_empty_tab = 100
+        table.min_width = min_width_empty_tab
         table.add_column("Template", justify="left")
         for item in list_data:
             pattern = surrogate_non_printable(item)

{gool-1.3.2 → gool-1.3.3}/src/gool/logs_miner.py

@@ -81,6 +81,8 @@ class LogsMiner:
         self.time_max = time_max
         self.unordered_time = unordered_time
         self.cluster_char_sizes: Counter = Counter()
+        # misc attr
+        self.time_warn_displayed = False
 
     def get_cluster_data(self) -> list[ClusterResult]:
         """
@@ -134,13 +136,17 @@ class LogsMiner:
     def _extract_time_from_line(self, line: str) -> datetime | None:
         if not self.time_pattern_regexp or not self.time_format:
             return None
-        match = self.time_pattern_regexp.search(line)
+        match = self.time_pattern_regexp.match(line)
         if match:
             time_str = match.group(1)
             try:
                 return datetime.strptime(time_str, self.time_format)
             except ValueError:
-                logging.warning("Failed to parse time '%s' in line: %s", time_str, line)
+                if not self.time_warn_displayed:
+                    logging.warning(
+                        "Failed to parse time '%s' in line: %s. This warning is only displayed once", time_str, line
+                    )
+                    self.time_warn_displayed = True
                 return None
         else:
             logging.warning("Time pattern not found in line: %s", line)
@@ -166,8 +172,7 @@ class LogsMiner:
             return False
         line_time = self._extract_time_from_line(line)
         if line_time is None:
-            logging.warning("Could not extract time from line, skipping time check: %s", line)
-            return False
+            return True
         if self.time_min and line_time < self.time_min:
             return True
         if self.time_max and line_time > self.time_max:

gool-1.3.3/tests/data/log/Crafted_logs.log

@@ -0,0 +1,5 @@
+12:00:12.121 | INFO  | first info 1234
+12:00:12.221 | INFO  | first info
+12:00:14.121 | INFO  | first info 8931
+12:00:14.121 | INFO  | second info
+13:00:14.121 | ERROR | second info

gool-1.3.3/tests/data/results/Crafted_logs_time_120014.txt

@@ -0,0 +1,5 @@
+Log Clusters
+1 - 0 - <TIME> | INFO | first info 8931
+1 - 0 - <TIME> | INFO | second info
+1 - 0 - <TIME> | ERROR | second info
+

{gool-1.3.2 → gool-1.3.3}/tests/test_gool.py

@@ -6,6 +6,7 @@ from pathlib import Path
 
 import pytest
 
+from gool.arguments_parser import guess_time_regexp_and_format_code
 from gool.log_clustering import (
     Arguments,
     ErrorCode,
@@ -33,6 +34,20 @@ class TestMisc:
     def test_surrogate_non_printable(self) -> None:
         assert surrogate_non_printable("foo") == "foo"
 
+    @pytest.mark.parametrize(
+        ("time_str", "expected"),
+        [
+            ("15:04:05", (r".*(\d{2}:\d{2}:\d{2}).*", "%H:%M:%S")),
+            ("15:04:05.000", (r".*(\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?).*", "%H:%M:%S.%f")),
+            ("15:04:05,000", (r".*(\d{2}:\d{2}:\d{2}(?:,\d{1,9})?).*", "%H:%M:%S,%f")),
+        ],
+    )
+    def test_guess_time_regexp_and_format_code(self, time_str: str, expected: tuple[str, str]) -> None:
+        assert guess_time_regexp_and_format_code(time_str) == expected
+
+    def test_guess_time_regexp_and_format_code_invalid(self) -> None:
+        assert guess_time_regexp_and_format_code("not-a-time") is None
+
     def test_create_config_file_if_needed(self, tmp_path: Path) -> None:
 
         home_config = tmp_path / "drain3.ini"
@@ -183,7 +198,7 @@ class TestNormalOutput:
             "--time-format",
             "%a %b %d %H:%M:%S %Y",
             "--time-pattern",
-            "(\\w{3} \\w{3} \\d{2} \\d{2}:\\d{2}:\\d{2} \\d{4})",
+            "\\[(\\w{3} \\w{3} \\d{2} \\d{2}:\\d{2}:\\d{2} \\d{4}).*",
             "--time-min",
             "Mon Dec 05 18:46:00 2005",
             "--time-max",
@@ -225,7 +240,7 @@ class TestNormalOutput:
             "--time-format",
             "%a %b %d %H:%M:%S %Y",
             "--time-pattern",
-            "(\\w{3} \\w{3} \\d{2} \\d{2}:\\d{2}:\\d{2} \\d{4})",
+            "\\[(\\w{3} \\w{3} \\d{2} \\d{2}:\\d{2}:\\d{2} \\d{4}).*",
             "--time-min",
             "Mon Dec 05 00:00:00 2005",
             "--time-max",
@@ -279,6 +294,13 @@ class TestNormalOutput:
         cmd = ["--unordered_time", "--time-max", "15:37:00,0"]
         run_gool_on_file_from_cli_and_check(cmd, input_path, expected_path)
 
+    def test_time_filter_low_precision(self) -> None:
+        input_path = input_dir_path / "Crafted_logs.log"
+        print(input_path)
+        expected_path = output_dir_path / "Crafted_logs_time_120014.txt"
+        cmd = ["--time-min", "12:00:14"]
+        run_gool_on_file_from_cli_and_check(cmd, input_path, expected_path)
+
     def test_version_cli_outputs_one_line(self) -> None:
         original_argv = sys.argv.copy()
         try:

{gool-1.3.2 → gool-1.3.3}/tests/test_logs_miner.py

@@ -11,7 +11,7 @@ from gool.logs_miner import ExceptionLineAfterMaxTime, LogsMiner
 @pytest.fixture
 def logs_miner_with_time_config() -> Any:
     """LogsMiner instance with time pattern and format configured."""
-    time_pattern = re.compile(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})")
+    time_pattern = re.compile(r".*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*")
     time_format = "%Y-%m-%d %H:%M:%S"
     with patch("gool.logs_miner.TemplateMiner"), patch("gool.logs_miner.create_drain3_cfg", return_value=Mock()):
         return LogsMiner(
@@ -40,7 +40,7 @@ class TestExtractTimeFromLine:
 
     def test_extract_time_when_no_format(self) -> None:
         """Should return None when time_format is not set."""
-        time_pattern = re.compile(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})")
+        time_pattern = re.compile(r".*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*")
         with patch("gool.logs_miner.TemplateMiner"), patch("gool.logs_miner.create_drain3_cfg", return_value=Mock()):
             logs_miner = LogsMiner(
                 drain3_cfg_file=Mock(),
@@ -90,7 +90,7 @@ class TestCheckLineOutTimeRange:
 
     def test_returns_false_when_no_time_format(self) -> None:
         """Should return False when time_format is not set."""
-        time_pattern = re.compile(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})")
+        time_pattern = re.compile(r".*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*")
         with patch("gool.logs_miner.TemplateMiner"), patch("gool.logs_miner.create_drain3_cfg", return_value=Mock()):
             logs_miner = LogsMiner(
                 drain3_cfg_file=Mock(),
@@ -102,16 +102,16 @@ class TestCheckLineOutTimeRange:
         result = logs_miner._check_line_out_time_range("2024-01-15 10:30:45 Some log")
         assert result is False
 
-    def test_returns_false_when_time_cannot_be_extracted(self, logs_miner_with_time_config: Any) -> None:
-        """Should return False and log warning when time cannot be extracted."""
+    def test_returns_true_when_time_cannot_be_extracted(self, logs_miner_with_time_config: Any) -> None:
+        """Should return True and log warning when time cannot be extracted."""
         with patch("gool.logs_miner.logging") as mock_logging:
             result = logs_miner_with_time_config._check_line_out_time_range("No time in this message")
-            assert result is False
+            assert result is True
             mock_logging.warning.assert_called()
 
     def test_returns_true_when_time_before_min(self) -> None:
         """Should return True when line time is before time_min."""
-        time_pattern = re.compile(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})")
+        time_pattern = re.compile(r".*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*")
         time_format = "%Y-%m-%d %H:%M:%S"
         time_min = datetime(2024, 1, 15, 12, 0, 0)
 
@@ -131,7 +131,7 @@ class TestCheckLineOutTimeRange:
 
     def test_returns_false_when_time_after_min(self) -> None:
         """Should return False when line time is after time_min."""
-        time_pattern = re.compile(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})")
+        time_pattern = re.compile(r".*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*")
         time_format = "%Y-%m-%d %H:%M:%S"
         time_min = datetime(2024, 1, 15, 10, 0, 0)
 
@@ -151,7 +151,7 @@ class TestCheckLineOutTimeRange:
 
     def test_raises_exception_when_time_after_max_ordered(self) -> None:
         """Should raise ExceptionLineAfterMaxTime when time > time_max and unordered_time is False."""
-        time_pattern = re.compile(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})")
+        time_pattern = re.compile(r".*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*")
         time_format = "%Y-%m-%d %H:%M:%S"
         time_max = datetime(2024, 1, 15, 10, 0, 0)
 
@@ -172,7 +172,7 @@ class TestCheckLineOutTimeRange:
 
     def test_returns_true_when_time_after_max_unordered(self) -> None:
         """Should return True when time > time_max and unordered_time is True."""
-        time_pattern = re.compile(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})")
+        time_pattern = re.compile(r".*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*")
         time_format = "%Y-%m-%d %H:%M:%S"
         time_max = datetime(2024, 1, 15, 10, 0, 0)
 
@@ -193,7 +193,7 @@ class TestCheckLineOutTimeRange:
 
     def test_returns_false_when_time_before_max(self) -> None:
         """Should return False when line time is before time_max."""
-        time_pattern = re.compile(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})")
+        time_pattern = re.compile(r".*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*")
         time_format = "%Y-%m-%d %H:%M:%S"
         time_max = datetime(2024, 1, 15, 12, 0, 0)
 
@@ -213,7 +213,7 @@ class TestCheckLineOutTimeRange:
 
     def test_returns_false_when_time_within_range(self) -> None:
         """Should return False when line time is within min and max range."""
-        time_pattern = re.compile(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})")
+        time_pattern = re.compile(r".*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*")
         time_format = "%Y-%m-%d %H:%M:%S"
         time_min = datetime(2024, 1, 15, 10, 0, 0)
         time_max = datetime(2024, 1, 15, 12, 0, 0)
@@ -235,7 +235,7 @@ class TestCheckLineOutTimeRange:
 
     def test_returns_false_when_time_equal_to_min(self) -> None:
         """Should return False when line time equals time_min."""
-        time_pattern = re.compile(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})")
+        time_pattern = re.compile(r".*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*")
         time_format = "%Y-%m-%d %H:%M:%S"
         time_min = datetime(2024, 1, 15, 10, 30, 45)
 
@@ -255,7 +255,7 @@ class TestCheckLineOutTimeRange:
 
     def test_returns_false_when_time_equal_to_max(self) -> None:
         """Should return False when line time equals time_max."""
-        time_pattern = re.compile(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})")
+        time_pattern = re.compile(r".*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*")
         time_format = "%Y-%m-%d %H:%M:%S"
         time_max = datetime(2024, 1, 15, 10, 30, 45)