logdetective 0.5.8__tar.gz → 0.5.10__tar.gz

{logdetective-0.5.8 → logdetective-0.5.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: logdetective
-Version: 0.5.8
+Version: 0.5.10
 Summary: Log using LLM AI to search for build/test failures and provide ideas for fixing these.
 License: Apache-2.0
 Author: Jiri Podivin
@@ -29,9 +29,9 @@ Requires-Dist: matplotlib (>=3.8.4,<4.0.0) ; extra == "server" or extra == "serv
 Requires-Dist: numpy (>=1.26.0)
 Requires-Dist: psycopg2 (>=2.9.9,<3.0.0) ; extra == "server"
 Requires-Dist: psycopg2-binary (>=2.9.9,<3.0.0) ; extra == "server-testing"
-Requires-Dist: pydantic (>=2.8.2,<3.0.0) ; extra == "server" or extra == "server-testing"
+Requires-Dist: pydantic (>=2.8.2,<3.0.0)
 Requires-Dist: python-gitlab (>=4.4.0)
-Requires-Dist: pyyaml (>=6.0.1,<7.0.0) ; extra == "server" or extra == "server-testing"
+Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
 Requires-Dist: requests (>0.2.31)
 Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "server" or extra == "server-testing"
 Project-URL: homepage, https://github.com/fedora-copr/logdetective
@@ -363,6 +363,21 @@ http GET "localhost:8080/metrics/analyze/requests?days=5" > /tmp/plot_days.svg
 http GET "localhost:8080/metrics/analyze/requests?weeks=5" > /tmp/plot_weeks.svg
 ```
 
+System Prompts
+--------------
+
+Prompt templates used by Log Detective are stored in the `prompts.yml` file.
+It is possible to modify the file in place, or provide your own.
+In CLI you can override prompt templates location using `--prompts` option,
+while in the container service deployment the `LOGDETECTIVE_PROMPTS` environment variable
+is used instead.
+
+Prompts need to have a form compatible with python [format string syntax](https://docs.python.org/3/library/string.html#format-string-syntax)
+with spaces, or replacement fields marked with curly braces, `{}` left for insertion of snippets.
+
+Number of replacement fields in new prompts, must be the same as in originals.
+Although their position may be different.
+
 License
 -------
 

{logdetective-0.5.8 → logdetective-0.5.10}/README.md

@@ -323,6 +323,21 @@ http GET "localhost:8080/metrics/analyze/requests?days=5" > /tmp/plot_days.svg
 http GET "localhost:8080/metrics/analyze/requests?weeks=5" > /tmp/plot_weeks.svg
 ```
 
+System Prompts
+--------------
+
+Prompt templates used by Log Detective are stored in the `prompts.yml` file.
+It is possible to modify the file in place, or provide your own.
+In CLI you can override prompt templates location using `--prompts` option,
+while in the container service deployment the `LOGDETECTIVE_PROMPTS` environment variable
+is used instead.
+
+Prompts need to have a form compatible with python [format string syntax](https://docs.python.org/3/library/string.html#format-string-syntax)
+with spaces, or replacement fields marked with curly braces, `{}` left for insertion of snippets.
+
+Number of replacement fields in new prompts, must be the same as in originals.
+Although their position may be different.
+
 License
 -------
 

{logdetective-0.5.8 → logdetective-0.5.10}/logdetective/constants.py

@@ -1,3 +1,8 @@
+"""This file contains various constants to be used as a fallback
+in case other values are not specified. Prompt templates should be modified
+in prompts.yaml instead.
+"""
+
 # pylint: disable=line-too-long
 DEFAULT_ADVISOR = "fedora-copr/Mistral-7B-Instruct-v0.2-GGUF"
 
@@ -19,7 +24,7 @@ Analysis:
 
 """
 
-SUMMARIZE_PROMPT_TEMPLATE = """
+SUMMARIZATION_PROMPT_TEMPLATE = """
 Does following log contain error or issue?
 
 Log:

{logdetective-0.5.8 → logdetective-0.5.10}/logdetective/extractors.py

@@ -6,7 +6,7 @@ import drain3
 from drain3.template_miner_config import TemplateMinerConfig
 from llama_cpp import Llama, LlamaGrammar
 
-from logdetective.constants import SUMMARIZE_PROMPT_TEMPLATE
+from logdetective.constants import SUMMARIZATION_PROMPT_TEMPLATE
 from logdetective.utils import get_chunks
 
 LOG = logging.getLogger("logdetective")
@@ -17,12 +17,18 @@ class LLMExtractor:
     A class that extracts relevant information from logs using a language model.
     """
 
-    def __init__(self, model: Llama, n_lines: int = 2):
+    def __init__(
+        self,
+        model: Llama,
+        n_lines: int = 2,
+        prompt: str = SUMMARIZATION_PROMPT_TEMPLATE,
+    ):
         self.model = model
         self.n_lines = n_lines
         self.grammar = LlamaGrammar.from_string(
             'root ::= ("Yes" | "No")', verbose=False
         )
+        self.prompt = prompt
 
     def __call__(
         self, log: str, n_lines: int = 2, neighbors: bool = False
@@ -41,7 +47,7 @@ class LLMExtractor:
 
         for i in range(0, len(log_lines), self.n_lines):
             block = "\n".join(log_lines[i: i + self.n_lines])
-            prompt = SUMMARIZE_PROMPT_TEMPLATE.format(log)
+            prompt = self.prompt.format(log)
             out = self.model(prompt, max_tokens=7, grammar=self.grammar)
             out = f"{out['choices'][0]['text']}\n"
             results.append((block, out))

{logdetective-0.5.8 → logdetective-0.5.10}/logdetective/logdetective.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 import sys
+import os
 
 from logdetective.constants import DEFAULT_ADVISOR
 from logdetective.utils import (
@@ -9,6 +10,7 @@ from logdetective.utils import (
     retrieve_log_content,
     format_snippets,
     compute_certainty,
+    load_prompts,
 )
 from logdetective.extractors import LLMExtractor, DrainExtractor
 
@@ -65,10 +67,16 @@ def setup_args():
     )
     parser.add_argument("-v", "--verbose", action="count", default=0)
     parser.add_argument("-q", "--quiet", action="store_true")
+    parser.add_argument(
+        "--prompts",
+        type=str,
+        default=f"{os.path.dirname(__file__)}/prompts.yml",
+        help="Path to prompt configuration file."
+    )
     return parser.parse_args()
 
 
-def main():  # pylint: disable=too-many-statements
+def main():  # pylint: disable=too-many-statements,too-many-locals
     """Main execution function."""
     args = setup_args()
 
@@ -83,6 +91,9 @@ def main():  # pylint: disable=too-many-statements
     if args.quiet:
         log_level = 0
 
+    # Get prompts configuration
+    prompts_configuration = load_prompts(args.prompts)
+
     logging.basicConfig(stream=sys.stdout)
     LOG.setLevel(log_level)
 
@@ -103,7 +114,11 @@ def main():  # pylint: disable=too-many-statements
         )
     else:
         summarizer_model = initialize_model(args.summarizer, verbose=args.verbose > 2)
-        extractor = LLMExtractor(summarizer_model, args.verbose > 1)
+        extractor = LLMExtractor(
+            summarizer_model,
+            args.verbose > 1,
+            prompts_configuration.summarization_prompt_template,
+        )
 
     LOG.info("Getting summary")
 
@@ -127,7 +142,12 @@ def main():  # pylint: disable=too-many-statements
     stream = True
     if args.no_stream:
         stream = False
-    response = process_log(log_summary, model, stream)
+    response = process_log(
+        log_summary,
+        model,
+        stream,
+        prompt_template=prompts_configuration.prompt_template,
+    )
     probs = []
     print("Explanation:")
     # We need to extract top token probability from the response

logdetective-0.5.10/logdetective/models.py

@@ -0,0 +1,33 @@
+from typing import Optional
+from pydantic import BaseModel
+
+from logdetective.constants import (
+    PROMPT_TEMPLATE,
+    PROMPT_TEMPLATE_STAGED,
+    SUMMARIZATION_PROMPT_TEMPLATE,
+    SNIPPET_PROMPT_TEMPLATE,
+)
+
+
+class PromptConfig(BaseModel):
+    """Configuration for basic log detective prompts."""
+
+    prompt_template: str = PROMPT_TEMPLATE
+    summarization_prompt_template: str = SUMMARIZATION_PROMPT_TEMPLATE
+    snippet_prompt_template: str = SNIPPET_PROMPT_TEMPLATE
+    prompt_template_staged: str = PROMPT_TEMPLATE_STAGED
+
+    def __init__(self, data: Optional[dict] = None):
+        super().__init__()
+        if data is None:
+            return
+        self.prompt_template = data.get("prompt_template", PROMPT_TEMPLATE)
+        self.summarization_prompt_template = data.get(
+            "summarization_prompt_template", SUMMARIZATION_PROMPT_TEMPLATE
+        )
+        self.snippet_prompt_template = data.get(
+            "snippet_prompt_template", SNIPPET_PROMPT_TEMPLATE
+        )
+        self.prompt_template_staged = data.get(
+            "prompt_template_staged", PROMPT_TEMPLATE_STAGED
+        )

logdetective-0.5.10/logdetective/prompts.yml

@@ -0,0 +1,55 @@
+# This file is intended for customization of prompts
+# It is used only in server mode.
+# On command line you have to load it using --prompts
+# The defaults are stored in constants.py
+
+prompt_template: |
+  Given following log snippets, and nothing else, explain what failure, if any, occured during build of this package.
+
+  Analysis of the snippets must be in a format of [X] : [Y], where [X] is a log snippet, and [Y] is the explanation.
+  Snippets themselves must not be altered in any way whatsoever.
+
+  Snippets are delimited with '================'.
+
+  Finally, drawing on information from all snippets, provide complete explanation of the issue and recommend solution.
+
+  Snippets:
+
+  {}
+
+  Analysis:
+
+
+summarization_prompt_template: |
+  Does following log contain error or issue?
+
+  Log:
+
+  {}
+
+  Answer:
+
+
+snippet_prompt_template: |
+  Analyse following RPM build log snippet. Describe contents accurately, without speculation or suggestions for resolution.
+
+  Snippet:
+
+  {}
+
+  Analysis:
+
+prompt_template_staged: |
+  Given following log snippets, their explanation, and nothing else, explain what failure, if any, occured during build of this package.
+
+  Snippets are in a format of [X] : [Y], where [X] is a log snippet, and [Y] is the explanation.
+
+  Snippets are delimited with '================'.
+
+  Drawing on information from all snippets, provide complete explanation of the issue and recommend solution.
+
+  Snippets:
+
+  {}
+
+  Analysis:

{logdetective-0.5.8 → logdetective-0.5.10}/logdetective/server/server.py

@@ -21,19 +21,18 @@ import gitlab.v4.objects
 import jinja2
 import requests
 
-from logdetective.constants import (
-    PROMPT_TEMPLATE,
-    SNIPPET_PROMPT_TEMPLATE,
-    PROMPT_TEMPLATE_STAGED,
-)
 from logdetective.extractors import DrainExtractor
 from logdetective.utils import (
     validate_url,
     compute_certainty,
     format_snippets,
+    load_prompts,
+)
+from logdetective.server.utils import (
+    load_server_config,
+    get_log,
     format_analyzed_snippets,
 )
-from logdetective.server.utils import load_server_config, get_log
 from logdetective.server.metric import track_request
 from logdetective.server.models import (
     BuildLog,
@@ -51,8 +50,10 @@ LLM_CPP_SERVER_TIMEOUT = os.environ.get("LLAMA_CPP_SERVER_TIMEOUT", 600)
 LOG_SOURCE_REQUEST_TIMEOUT = os.environ.get("LOG_SOURCE_REQUEST_TIMEOUT", 60)
 API_TOKEN = os.environ.get("LOGDETECTIVE_TOKEN", None)
 SERVER_CONFIG_PATH = os.environ.get("LOGDETECTIVE_SERVER_CONF", None)
+SERVER_PROMPT_PATH = os.environ.get("LOGDETECTIVE_PROMPTS", None)
 
 SERVER_CONFIG = load_server_config(SERVER_CONFIG_PATH)
+PROMPT_CONFIG = load_prompts(SERVER_PROMPT_PATH)
 
 MR_REGEX = re.compile(r"refs/merge-requests/(\d+)/.*$")
 FAILURE_LOG_REGEX = re.compile(r"(\w*\.log)")
@@ -298,7 +299,7 @@ async def analyze_log(build_log: BuildLog):
     log_summary = mine_logs(log_text)
     log_summary = format_snippets(log_summary)
     response = await submit_text(
-        PROMPT_TEMPLATE.format(log_summary),
+        PROMPT_CONFIG.prompt_template.format(log_summary),
         api_endpoint=SERVER_CONFIG.inference.api_endpoint,
     )
     certainty = 0
@@ -338,7 +339,7 @@ async def perform_staged_analysis(log_text: str) -> StagedResponse:
     analyzed_snippets = await asyncio.gather(
         *[
             submit_text(
-                SNIPPET_PROMPT_TEMPLATE.format(s),
+                PROMPT_CONFIG.snippet_prompt_template.format(s),
                 api_endpoint=SERVER_CONFIG.inference.api_endpoint,
             )
             for s in log_summary
@@ -349,7 +350,7 @@ async def perform_staged_analysis(log_text: str) -> StagedResponse:
         AnalyzedSnippet(line_number=e[0][0], text=e[0][1], explanation=e[1])
         for e in zip(log_summary, analyzed_snippets)
     ]
-    final_prompt = PROMPT_TEMPLATE_STAGED.format(
+    final_prompt = PROMPT_CONFIG.prompt_template_staged.format(
         format_analyzed_snippets(analyzed_snippets)
     )
 
@@ -395,7 +396,7 @@ async def analyze_log_stream(build_log: BuildLog):
         headers["Authorization"] = f"Bearer {SERVER_CONFIG.inference.api_token}"
 
     stream = await submit_text_chat_completions(
-        PROMPT_TEMPLATE.format(log_summary), stream=True, headers=headers
+        PROMPT_CONFIG.prompt_template.format(log_summary), stream=True, headers=headers
     )
 
     return StreamingResponse(stream)

{logdetective-0.5.8 → logdetective-0.5.10}/logdetective/server/utils.py

@@ -1,6 +1,18 @@
 import logging
 import yaml
-from logdetective.server.models import Config
+from logdetective.constants import SNIPPET_DELIMITER
+from logdetective.server.models import Config, AnalyzedSnippet
+
+
+def format_analyzed_snippets(snippets: list[AnalyzedSnippet]) -> str:
+    """Format snippets for submission into staged prompt."""
+    summary = f"\n{SNIPPET_DELIMITER}\n".join(
+        [
+            f"[{e.text}] at line [{e.line_number}]: [{e.explanation.text}]"
+            for e in snippets
+        ]
+    )
+    return summary
 
 
 def load_server_config(path: str | None) -> Config:

{logdetective-0.5.8 → logdetective-0.5.10}/logdetective/utils.py

@@ -4,10 +4,11 @@ from typing import Iterator, List, Dict, Tuple, Generator
 from urllib.parse import urlparse
 import numpy as np
 import requests
+import yaml
 
 from llama_cpp import Llama, CreateCompletionResponse, CreateCompletionStreamResponse
-from logdetective.constants import PROMPT_TEMPLATE, SNIPPET_DELIMITER
-from logdetective.server.models import AnalyzedSnippet
+from logdetective.models import PromptConfig
+
 
 LOG = logging.getLogger("logdetective")
 
@@ -110,7 +111,7 @@ def compute_certainty(probs: List[Dict]) -> float:
 
 
 def process_log(
-    log: str, model: Llama, stream: bool
+    log: str, model: Llama, stream: bool, prompt_template: str
 ) -> CreateCompletionResponse | Iterator[CreateCompletionStreamResponse]:
     """Processes a given log using the provided language model and returns its summary.
 
@@ -122,7 +123,7 @@ def process_log(
         str: The summary of the given log generated by the language model.
     """
     response = model(
-        prompt=PROMPT_TEMPLATE.format(log), stream=stream, max_tokens=0, logprobs=1
+        prompt=prompt_template.format(log), stream=stream, max_tokens=0, logprobs=1
     )
 
     return response
@@ -175,17 +176,6 @@ def format_snippets(snippets: list[str] | list[Tuple[int, str]]) -> str:
     return summary
 
 
-def format_analyzed_snippets(snippets: list[AnalyzedSnippet]) -> str:
-    """Format snippets for submission into staged prompt."""
-    summary = f"\n{SNIPPET_DELIMITER}\n".join(
-        [
-            f"[{e.text}] at line [{e.line_number}]: [{e.explanation.text}]"
-            for e in snippets
-        ]
-    )
-    return summary
-
-
 def validate_url(url: str) -> bool:
     """Validate incoming URL to be at least somewhat sensible for log files
     Only http and https protocols permitted. No result, params or query fields allowed.
@@ -199,3 +189,15 @@ def validate_url(url: str) -> bool:
     if not (result.path or result.netloc):
         return False
     return True
+
+
+def load_prompts(path: str | None) -> PromptConfig:
+    """Load prompts from given yaml file if there is one.
+    Alternatively use defaults."""
+    if path:
+        try:
+            with open(path, "r") as file:
+                return PromptConfig(yaml.safe_load(file))
+        except FileNotFoundError:
+            print("Prompt configuration file not found, reverting to defaults.")
+    return PromptConfig()

logdetective-0.5.10/logdetective.1.asciidoc

@@ -0,0 +1,83 @@
+= logdetective(1)
+:doctype: manpage
+:man source: logdetective 1.0
+:man manual: User Commands
+
+== NAME
+
+logdetective - Analyze and summarize log files using LLM or Drain templates
+
+== SYNOPSIS
+
+*logdetective* [_OPTIONS_] *file*
+
+== DESCRIPTION
+
+*logdetective* is a tool that analyzes log files using either a large language
+model (LLM) or the Drain log template miner. It can consume logs from a local
+path or a URL, summarize them, and cluster them for easier inspection.
+
+== POSITIONAL ARGUMENTS
+
+*file*::
+  The URL or path to the log file to be analyzed.
+
+== OPTIONS
+
+*-h*, *--help*::
+  Show this help message and exit.
+
+*-M* *MODEL*, *--model* *MODEL*::
+  The path or URL of the language model for analysis. As we are using LLama.cpp we want this to be in the gguf format. You can include the download link to the model here. If the model is already on your machine it will skip the download. (optional, default: "Mistral-7B-Instruct-v0.2-GGUF")
+
+*-F* *FILENAME_SUFFIX*, *--filename_suffix* *FILENAME_SUFFIX*::
+  Define the suffix of the model file name to retrieve from Hugging Face. This option only applies when the model is specified by name (not a path).
+
+*-n*, *--no-stream*::
+  Disable streaming output of analysis results.
+
+*-S* *SUMMARIZER*, *--summarizer* *SUMMARIZER*::
+  Choose between LLM and Drain template miner as the log summarizer. You can also provide the path to an existing language model file instead of using a URL. (optional, default: "drain")
+
+*-N* *N_LINES*, *--n_lines* *N_LINES*::
+  Number of lines per chunk for LLM analysis. Only applicable when `LLM` is used as the summarizer. (optional, default: 8)
+
+*-C* *N_CLUSTERS*, *--n_clusters* *N_CLUSTERS*::
+  Number of clusters to use with the Drain summarizer. Ignored if `LLM` summarizer is selected. (optional, default 8)
+
+*-v*, *--verbose*::
+  Enable verbose output during processing.
+
+*-q*, *--quiet*::
+  Suppress non-essential output.
+
+*--prompts* *PROMPTS*::
+  Path to prompt configuration file where you can customize prompts sent to `LLM`.
+
+
+== EXAMPLES
+
+Example usage:
+
+  $ logdetective https://example.com/logs.txt
+
+Or if the log file is stored locally:
+
+  $ logdetective ./data/logs.txt
+
+Analyze a local log file using an LLM model:
+
+  $ logdetective -M /path/to/llm-model -S LLM -N 100 /var/log/syslog
+
+With specific models:
+
+  $ logdetective https://example.com/logs.txt --model https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/Meta-Llama-3-8B-Instruct.Q5_K_S.gguf?download=true
+  $ logdetective https://example.com/logs.txt --model QuantFactory/Meta-Llama-3-8B-Instruct-GGUF
+
+Cluster logs from a URL using Drain:
+
+  $ logdetective -S Drain -C 10 https://example.com/logs.txt
+
+== SEE ALSO
+
+https://logdetective.com

{logdetective-0.5.8 → logdetective-0.5.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "logdetective"
-version = "0.5.8"
+version = "0.5.10"
 description = "Log using LLM AI to search for build/test failures and provide ideas for fixing these."
 authors = ["Jiri Podivin <jpodivin@gmail.com>"]
 license = "Apache-2.0"
@@ -8,6 +8,8 @@ readme = "README.md"
 include = [
     "logdetective/drain3.ini",
     "logdetective/server/templates/gitlab_comment.md.j2",
+    "logdetective/prompts.yml",
+    "logdetective.1.asciidoc",
 ]
 packages = [
     { include = "logdetective" }
@@ -38,10 +40,10 @@ huggingface-hub = ">0.23.2"
 # we need to support both versions
 numpy = ">=1.26.0"
 python-gitlab = ">=4.4.0"
+pydantic = "^2.8.2"
+pyyaml = "^6.0.1"
 
-pydantic = {version = "^2.8.2", optional = true }
 fastapi = {version = ">=0.111.1", optional = true }
-pyyaml = {version = "^6.0.1", optional = true }
 sqlalchemy = {version = "^2.0.36", optional = true }
 psycopg2-binary = {version = "^2.9.9", optional = true }
 psycopg2 = {version = "^2.9.9", optional = true }
@@ -49,8 +51,8 @@ alembic = {version = "^1.13.3", optional = true }
 matplotlib = {version = "^3.8.4", optional = true }
 
 [tool.poetry.extras]
-server = ["pydantic", "fastapi", "pyyaml", "sqlalchemy", "psycopg2", "alembic", "matplotlib"]
-server-testing = ["pydantic", "fastapi", "pyyaml", "sqlalchemy", "psycopg2-binary", "alembic", "matplotlib"]
+server = ["fastapi", "sqlalchemy", "psycopg2", "alembic", "matplotlib"]
+server-testing = ["fastapi", "sqlalchemy", "psycopg2-binary", "alembic", "matplotlib"]
 
 [build-system]
 requires = ["poetry-core"]