logdetective 0.4.0__py3-none-any.whl → 2.11.0__py3-none-any.whl

logdetective/constants.py

@@ -1,5 +1,10 @@
+"""This file contains various constants to be used as a fallback
+in case other values are not specified. Prompt templates should be modified
+in prompts.yaml instead.
+"""
+
 # pylint: disable=line-too-long
-DEFAULT_ADVISOR = "fedora-copr/Mistral-7B-Instruct-v0.2-GGUF"
+DEFAULT_ADVISOR = "fedora-copr/Mistral-7B-Instruct-v0.3-GGUF"
 
 PROMPT_TEMPLATE = """
 Given following log snippets, and nothing else, explain what failure, if any, occured during build of this package.
@@ -11,6 +16,8 @@ Snippets are delimited with '================'.
 
 Finally, drawing on information from all snippets, provide complete explanation of the issue and recommend solution.
 
+Explanation of the issue, and recommended solution, should take handful of sentences.
+
 Snippets:
 
 {}
@@ -19,20 +26,11 @@ Analysis:
 
 """
 
-SUMMARIZE_PROMPT_TEMPLATE = """
-Does following log contain error or issue?
-
-Log:
-
-{}
-
-Answer:
-
-"""
-
 SNIPPET_PROMPT_TEMPLATE = """
 Analyse following RPM build log snippet. Describe contents accurately, without speculation or suggestions for resolution.
 
+Your analysis must be as concise as possible, while keeping relevant information intact.
+
 Snippet:
 
 {}
@@ -50,6 +48,8 @@ Snippets are delimited with '================'.
 
 Drawing on information from all snippets, provide complete explanation of the issue and recommend solution.
 
+Explanation of the issue, and recommended solution, should take handful of sentences.
+
 Snippets:
 
 {}
@@ -58,4 +58,25 @@ Analysis:
 
 """
 
+DEFAULT_SYSTEM_PROMPT = """
+You are a highly capable large language model based expert system specialized in
+packaging and delivery of software using RPM (RPM Package Manager). Your purpose is to diagnose
+RPM build failures, identifying root causes and proposing solutions if possible.
+You are truthful, concise, and helpful.
+
+You never speculate about package being built or fabricate information.
+If you do not know the answer, you acknowledge the fact and end your response.
+Your responses must be as short as possible.
+"""
+
 SNIPPET_DELIMITER = "================"
+
+DEFAULT_TEMPERATURE = 0.8
+
+# Tuning for LLM-as-a-Service
+LLM_DEFAULT_MAX_QUEUE_SIZE = 50
+LLM_DEFAULT_REQUESTS_PER_MINUTE = 60
+
+# Roles for chat API
+SYSTEM_ROLE_DEFAULT = "developer"
+USER_ROLE_DEFAULT = "user"

logdetective/extractors.py

@@ -1,99 +1,168 @@
 import os
 import logging
+import subprocess as sp
 from typing import Tuple
 
 import drain3
 from drain3.template_miner_config import TemplateMinerConfig
-from llama_cpp import Llama, LlamaGrammar
+from pydantic import ValidationError
 
-from logdetective.constants import SUMMARIZE_PROMPT_TEMPLATE
-from logdetective.utils import get_chunks
+from logdetective.utils import get_chunks, filter_snippet_patterns
+from logdetective.models import SkipSnippets, CSGrepOutput
 
 LOG = logging.getLogger("logdetective")
 
 
-class LLMExtractor:
-    """
-    A class that extracts relevant information from logs using a language model.
-    """
+class Extractor:
+    """Base extractor class."""
 
-    def __init__(self, model: Llama, n_lines: int = 2):
-        self.model = model
-        self.n_lines = n_lines
-        self.grammar = LlamaGrammar.from_string(
-            'root ::= ("Yes" | "No")', verbose=False
-        )
-
-    def __call__(
-        self, log: str, n_lines: int = 2, neighbors: bool = False
-    ) -> list[str]:
-        chunks = self.rate_chunks(log)
-        out = self.create_extract(chunks, neighbors)
-        return out
-
-    def rate_chunks(self, log: str) -> list[tuple]:
-        """Scan log by the model and store results.
-
-        :param log: log file content
-        """
-        results = []
-        log_lines = log.split("\n")
-
-        for i in range(0, len(log_lines), self.n_lines):
-            block = "\n".join(log_lines[i: i + self.n_lines])
-            prompt = SUMMARIZE_PROMPT_TEMPLATE.format(log)
-            out = self.model(prompt, max_tokens=7, grammar=self.grammar)
-            out = f"{out['choices'][0]['text']}\n"
-            results.append((block, out))
-
-        return results
-
-    def create_extract(self, chunks: list[tuple], neighbors: bool = False) -> list[str]:
-        """Extract interesting chunks from the model processing."""
-        interesting = []
-        summary = []
-        # pylint: disable=consider-using-enumerate
-        for i in range(len(chunks)):
-            if chunks[i][1].startswith("Yes"):
-                interesting.append(i)
-                if neighbors:
-                    interesting.extend([max(i - 1, 0), min(i + 1, len(chunks) - 1)])
+    def __init__(
+        self,
+        verbose: bool = False,
+        skip_snippets: SkipSnippets = SkipSnippets({}),
+        max_snippet_len: int = 2000,
+    ):
+        self.verbose = verbose
+        self.skip_snippets = skip_snippets
+        self.max_snippet_len = max_snippet_len
 
-        interesting = set(interesting)
+        if self.verbose:
+            LOG.setLevel(logging.DEBUG)
 
-        for i in interesting:
-            summary.append(chunks[i][0])
+    def __call__(self, log: str) -> list[Tuple[int, str]]:
+        raise NotImplementedError
 
-        return summary
+    def filter_snippet_patterns(
+        self, chunks: list[tuple[int, str]]
+    ) -> list[tuple[int, str]]:
+        """Keep only chunks that don't match any of the excluded patterns"""
+        chunks = [
+            (_, chunk)
+            for _, chunk in chunks
+            if not filter_snippet_patterns(chunk, self.skip_snippets)
+        ]
+        return chunks
 
 
-class DrainExtractor:
+class DrainExtractor(Extractor):
     """A class that extracts information from logs using a template miner algorithm."""
 
-    def __init__(self, verbose: bool = False, context: bool = False, max_clusters=8):
+    _clusters: list
+
+    def __init__(
+        self,
+        verbose: bool = False,
+        skip_snippets: SkipSnippets = SkipSnippets({}),
+        max_snippet_len: int = 2000,
+        max_clusters: int = 8,
+    ):
+        super().__init__(verbose, skip_snippets, max_snippet_len)
         config = TemplateMinerConfig()
         config.load(f"{os.path.dirname(__file__)}/drain3.ini")
         config.profiling_enabled = verbose
         config.drain_max_clusters = max_clusters
         self.miner = drain3.TemplateMiner(config=config)
-        self.verbose = verbose
-        self.context = context
 
     def __call__(self, log: str) -> list[Tuple[int, str]]:
-        out = []
-        # First pass create clusters
-        for _, chunk in get_chunks(log):
-            processed_chunk = self.miner.add_log_message(chunk)
-            LOG.debug(processed_chunk)
-        # Sort found clusters by size, descending order
-        sorted_clusters = sorted(
-            self.miner.drain.clusters, key=lambda it: it.size, reverse=True
-        )
+        # Create chunks
+        chunks = list(get_chunks(log, self.max_snippet_len))
+
+        chunks = self.filter_snippet_patterns(chunks)
+
+        # First pass to create clusters
+        self._create_clusters(chunks=chunks)
+
         # Second pass, only matching lines with clusters,
         # to recover original text
-        for chunk_start, chunk in get_chunks(log):
+        snippets = self._extract_messages(chunks=chunks)
+        return snippets
+
+    def _create_clusters(self, chunks: list[tuple[int, str]]):
+        """First pass to create clusters"""
+        for _, chunk in chunks:
+            processed_chunk = self.miner.add_log_message(chunk)
+            LOG.debug(processed_chunk)
+        self._clusters = list(self.miner.drain.clusters)
+
+    def _extract_messages(self, chunks: list[tuple[int, str]]) -> list[tuple[int, str]]:
+        """Second pass with drain using patterns from the first,
+        to extract matching lines and their numbers."""
+        out = []
+
+        for chunk_start, chunk in chunks:
             cluster = self.miner.match(chunk, "always")
-            if cluster in sorted_clusters:
+            if cluster in self._clusters:
                 out.append((chunk_start, chunk))
-                sorted_clusters.remove(cluster)
+                self._clusters.remove(cluster)
         return out
+
+
+class CSGrepExtractor(DrainExtractor):
+    """Extract messages using csgrep
+    This extractor is only effective at retrieving messages from GCC
+    compiler and associated utilities, it is not capable of safely
+    extracting other messages from the logs. Therefore, it must only
+    be used together with the Drain based extractor."""
+
+    def __init__(
+        self,
+        verbose: bool = False,
+        skip_snippets: SkipSnippets = SkipSnippets({}),
+        max_snippet_len: int = 2000,
+        max_clusters: int = 8,
+    ):
+        super().__init__(verbose, skip_snippets, max_snippet_len, max_clusters)
+
+    def __call__(self, log: str) -> list[Tuple[int, str]]:
+        """Extract error messages from log using csgrep"""
+        chunks = []
+        try:
+            # We are not running binary in check mode, since csgrep
+            # can produce many errors due to log file syntax
+            result = sp.run(
+                [
+                    "csgrep",
+                    "--event=error",
+                    "--remove-duplicates",
+                    "--mode=json",
+                    "--quiet",
+                ],
+                input=log,
+                shell=False,
+                check=False,
+                capture_output=True,
+                text=True,
+                timeout=1.0,
+            )
+        except sp.TimeoutExpired as ex:
+            LOG.exception("Exception encountered while parsing log with csgrep %s", ex)
+            raise ex
+        if result.returncode != 0:
+            # This can happen even if `csgrep` managed to extract useful info.
+            # Most commonly, when it encountered unexpected syntax in the log.
+            LOG.warning("csgrep call resulted in an error")
+            LOG.debug("csgrep error: `%s`", result.stderr)
+        if not result.stdout:
+            return []
+
+        # Parse JSON output from csgrep
+        try:
+            report = CSGrepOutput.model_validate_json(result.stdout)
+        except ValidationError as ex:
+            LOG.exception("Exception encountered while parsing csgrpe output %s", ex)
+            raise ex
+        for defect in report.defects:
+            # Single original error message can be split across multiple events
+            # before returning, we will turn them back into single string.
+            # We must also extract the original line number.
+            # Line number is NOT location of message in the log, but location of
+            # the issue in source, we can't really mix the two, so we'll set it to `0`.
+
+            chunks.append((0, "\n".join([event.message for event in defect.events])))
+
+        chunks = self.filter_snippet_patterns(chunks)
+        LOG.info("Total %d messages extracted with csgrep", len(chunks))
+        self._create_clusters(chunks=chunks)
+        snippets = self._extract_messages(chunks=chunks)
+
+        return snippets

logdetective/logdetective.py

@@ -1,16 +1,24 @@
 import argparse
+import asyncio
 import logging
 import sys
+import os
 
-from logdetective.constants import DEFAULT_ADVISOR
+import aiohttp
+
+from logdetective.constants import DEFAULT_ADVISOR, DEFAULT_TEMPERATURE
 from logdetective.utils import (
     process_log,
     initialize_model,
     retrieve_log_content,
     format_snippets,
     compute_certainty,
+    load_prompts,
+    load_skip_snippet_patterns,
+    check_csgrep,
+    mine_logs,
 )
-from logdetective.extractors import LLMExtractor, DrainExtractor
+from logdetective.extractors import DrainExtractor, CSGrepExtractor
 
 LOG = logging.getLogger("logdetective")
 
@@ -36,7 +44,7 @@ def setup_args():
         "--filename_suffix",
         help="Suffix of the model file name to be retrieved from Hugging Face.\
                             Makes sense only if the model is specified with Hugging Face name.",
-        default="Q4_K_S.gguf",
+        default="Q4_K.gguf",
     )
     parser.add_argument("-n", "--no-stream", action="store_true")
     parser.add_argument(
@@ -44,16 +52,16 @@ def setup_args():
         "--summarizer",
         type=str,
         default="drain",
-        help="Choose between LLM and Drain template miner as the log summarizer.\
-                                LLM must be specified as path to a model, URL or local file.",
+        help="DISABLED: LLM summarization option was removed. \
+                Argument is kept for backward compatibility only.",
     )
     parser.add_argument(
         "-N",
         "--n_lines",
         type=int,
-        default=8,
-        help="The number of lines per chunk for LLM analysis.\
-                            This only makes sense when you are summarizing with LLM.",
+        default=None,
+        help="DISABLED: LLM summarization option was removed. \
+                Argument is kept for backward compatibility only.",
     )
     parser.add_argument(
         "-C",
@@ -65,10 +73,31 @@ def setup_args():
     )
     parser.add_argument("-v", "--verbose", action="count", default=0)
     parser.add_argument("-q", "--quiet", action="store_true")
+    parser.add_argument(
+        "--prompts",
+        type=str,
+        default=f"{os.path.dirname(__file__)}/prompts.yml",
+        help="Path to prompt configuration file.",
+    )
+    parser.add_argument(
+        "--temperature",
+        type=float,
+        default=DEFAULT_TEMPERATURE,
+        help="Temperature for inference.",
+    )
+    parser.add_argument(
+        "--skip_snippets",
+        type=str,
+        default=f"{os.path.dirname(__file__)}/skip_snippets.yml",
+        help="Path to patterns for skipping snippets.",
+    )
+    parser.add_argument(
+        "--csgrep", action="store_true", help="Use csgrep to process the log."
+    )
     return parser.parse_args()
 
 
-def main():  # pylint: disable=too-many-statements
+async def run():  # pylint: disable=too-many-statements,too-many-locals,too-many-branches
     """Main execution function."""
     args = setup_args()
 
@@ -76,6 +105,10 @@ def main():  # pylint: disable=too-many-statements
         sys.stderr.write("Error: --quiet and --verbose is mutually exclusive.\n")
         sys.exit(2)
 
+    # Emit warning about use of discontinued args
+    if args.n_lines or args.summarizer != "drain":
+        LOG.warning("LLM based summarization was removed. Drain will be used instead.")
+
     # Logging facility setup
     log_level = logging.INFO
     if args.verbose >= 1:
@@ -83,6 +116,9 @@ def main():  # pylint: disable=too-many-statements
     if args.quiet:
         log_level = 0
 
+    # Get prompts configuration
+    prompts_configuration = load_prompts(args.prompts)
+
     logging.basicConfig(stream=sys.stdout)
     LOG.setLevel(log_level)
 
@@ -96,29 +132,43 @@ def main():  # pylint: disable=too-many-statements
         LOG.error("You likely do not have enough memory to load the AI model")
         sys.exit(3)
 
-    # Log file summarizer selection and initialization
-    if args.summarizer == "drain":
-        extractor = DrainExtractor(
-            args.verbose > 1, context=True, max_clusters=args.n_clusters
-        )
-    else:
-        summarizer_model = initialize_model(args.summarizer, verbose=args.verbose > 2)
-        extractor = LLMExtractor(summarizer_model, args.verbose > 1)
-
-    LOG.info("Getting summary")
-
     try:
-        log = retrieve_log_content(args.file)
-    except ValueError as e:
-        # file does not exists
+        skip_snippets = load_skip_snippet_patterns(args.skip_snippets)
+    except OSError as e:
         LOG.error(e)
-        sys.exit(4)
-    log_summary = extractor(log)
+        sys.exit(5)
+
+    # Log file summarizer initialization
+    extractors = []
+    extractors.append(
+        DrainExtractor(
+            args.verbose > 1,
+            max_clusters=args.n_clusters,
+            skip_snippets=skip_snippets,
+        )
+    )
 
-    ratio = len(log_summary) / len(log.split("\n"))
+    if args.csgrep:
+        if not check_csgrep():
+            LOG.error(
+                "You have requested use of `csgrep` when it isn't available on your system."
+            )
+            sys.exit(6)
+        extractors.append(
+            CSGrepExtractor(args.verbose > 1, skip_snippets=skip_snippets)
+        )
+
+    LOG.info("Getting summary")
 
-    LOG.info("Compression ratio: %s", ratio)
+    async with aiohttp.ClientSession() as http:
+        try:
+            log = await retrieve_log_content(http, args.file)
+        except ValueError as e:
+            # file does not exist
+            LOG.error(e)
+            sys.exit(4)
 
+    log_summary = mine_logs(log=log, extractors=extractors)
     LOG.info("Analyzing the text")
 
     log_summary = format_snippets(log_summary)
@@ -127,33 +177,52 @@ def main():  # pylint: disable=too-many-statements
     stream = True
     if args.no_stream:
         stream = False
-    response = process_log(log_summary, model, stream)
+    response = process_log(
+        log_summary,
+        model,
+        stream,
+        prompt_templates=prompts_configuration,
+        temperature=args.temperature,
+    )
     probs = []
     print("Explanation:")
     # We need to extract top token probability from the response
-    # CreateCompletionResponse structure of llama-cpp-python.
+    # CreateChatCompletionResponse structure of llama-cpp-python.
     # `compute_certainty` function expects list of dictionaries with form
     # { 'logprob': <float> } as expected from the OpenAI API.
 
     if args.no_stream:
-        print(response["choices"][0]["text"])
+        print(response["choices"][0]["message"]["content"])
         probs = [
-            {"logprob": e} for e in response["choices"][0]["logprobs"]["token_logprobs"]
+            {"logprob": e["logprob"]} for e in response["choices"][0]["logprobs"]["content"]
         ]
 
     else:
         # Stream the output
         for chunk in response:
+            # What might happen, is that first (or possibly any other) chunk may not contain
+            # fields choices[0].delta.content or choices[0].logprobs -> if so, we just skip it
+            if any([
+                'content' not in chunk["choices"][0]["delta"],
+                'logprobs' not in chunk["choices"][0]
+            ]):
+                continue
+
             if isinstance(chunk["choices"][0]["logprobs"], dict):
                 probs.append(
-                    {"logprob": chunk["choices"][0]["logprobs"]["token_logprobs"][0]}
+                    {"logprob": chunk["choices"][0]["logprobs"]["content"][0]["logprob"]}
                 )
-            delta = chunk["choices"][0]["text"]
+            delta = chunk["choices"][0]["delta"]["content"]
             print(delta, end="", flush=True)
     certainty = compute_certainty(probs)
 
     print(f"\nResponse certainty: {certainty:.2f}%\n")
 
 
+def main():
+    """Evaluate logdetective program and wait for it to finish"""
+    asyncio.run(run())
+
+
 if __name__ == "__main__":
     main()

logdetective/models.py

@@ -0,0 +1,99 @@
+import re
+from typing import Optional
+from pydantic import BaseModel, model_validator
+
+from logdetective.constants import (
+    PROMPT_TEMPLATE,
+    PROMPT_TEMPLATE_STAGED,
+    SNIPPET_PROMPT_TEMPLATE,
+    DEFAULT_SYSTEM_PROMPT,
+)
+
+
+class PromptConfig(BaseModel):
+    """Configuration for basic log detective prompts."""
+
+    prompt_template: str = PROMPT_TEMPLATE
+    snippet_prompt_template: str = SNIPPET_PROMPT_TEMPLATE
+    prompt_template_staged: str = PROMPT_TEMPLATE_STAGED
+
+    default_system_prompt: str = DEFAULT_SYSTEM_PROMPT
+    snippet_system_prompt: str = DEFAULT_SYSTEM_PROMPT
+    staged_system_prompt: str = DEFAULT_SYSTEM_PROMPT
+
+    def __init__(self, data: Optional[dict] = None):
+        super().__init__()
+        if data is None:
+            return
+        self.prompt_template = data.get("prompt_template", PROMPT_TEMPLATE)
+        self.snippet_prompt_template = data.get(
+            "snippet_prompt_template", SNIPPET_PROMPT_TEMPLATE
+        )
+        self.prompt_template_staged = data.get(
+            "prompt_template_staged", PROMPT_TEMPLATE_STAGED
+        )
+        self.default_system_prompt = data.get(
+            "default_system_prompt", DEFAULT_SYSTEM_PROMPT
+        )
+        self.snippet_system_prompt = data.get(
+            "snippet_system_prompt", DEFAULT_SYSTEM_PROMPT
+        )
+        self.staged_system_prompt = data.get(
+            "staged_system_prompt", DEFAULT_SYSTEM_PROMPT
+        )
+
+
+class SkipSnippets(BaseModel):
+    """Regular expressions defining snippets we should not analyze"""
+
+    snippet_patterns: dict[str, re.Pattern] = {}
+
+    def __init__(self, data: Optional[dict] = None):
+        super().__init__(data=data)
+        if data is None:
+            return
+        self.snippet_patterns = {
+            key: re.compile(pattern) for key, pattern in data.items()
+        }
+
+    @model_validator(mode="before")
+    @classmethod
+    def check_patterns(cls, data: dict):
+        """Check if all supplied patterns are valid regular expressions.
+        Techically replicating what is done in __init__ but with nicer error message."""
+        patterns = data["data"]
+        for key, pattern in patterns.items():
+            try:
+                re.compile(pattern=pattern)
+            except (TypeError, re.error) as ex:
+                raise ValueError(
+                    f"Invalid pattern `{pattern}` with name `{key}` supplied for skipping in logs."
+                ) from ex
+
+        return data
+
+
+class CSGrepEvent(BaseModel):
+    """`csgrep` splits error and warning messages into individual events."""
+
+    file_name: str
+    line: int
+    event: str
+    message: str
+    verbosity_level: int
+
+
+class CSGrepDefect(BaseModel):
+    """Defects detected by `csgrep`"""
+
+    checker: str
+    language: str
+    tool: str
+    key_event_idx: int
+    events: list[CSGrepEvent]
+
+
+class CSGrepOutput(BaseModel):
+    """Parsed output of `gsgrep`"""
+
+    defects: list[CSGrepDefect]

logdetective/prompts-summary-first.yml

@@ -0,0 +1,20 @@
+# This file is intended for customization of prompts
+# It is used only in server mode.
+# On command line you have to load it using --prompts
+# The defaults are stored in constants.py
+
+prompt_template: |
+  Given following log snippets, and nothing else, explain what failure, if any, occured during build of this package.
+
+  Please start with concise, one sentence long, summary describing the problem and recommend solution to fix it. And then follow with analysis.
+
+  Analysis of the snippets must be in a format of [X] : [Y], where [X] is a log snippet, and [Y] is the explanation.
+  Snippets themselves must not be altered in any way whatsoever.
+
+  Snippets are delimited with '================'.
+
+  Explanation of the issue, and recommended solution, should take handful of sentences.
+
+  Snippets:
+
+  {}

logdetective/prompts-summary-only.yml

@@ -0,0 +1,13 @@
+# This file is intended for customization of prompts
+# It is used only in server mode.
+# On command line you have to load it using --prompts
+# The defaults are stored in constants.py
+
+prompt_template: |
+  Given following log snippets, and nothing else, explain what failure, if any, occured during build of this package.
+
+  Provide concise, one paragraph long, summary describing the problem of most probable culprit and recommend solution to fix it.
+
+  Snippets:
+
+  {}