logdetective 0.4.0__tar.gz → 0.5.0__tar.gz

{logdetective-0.4.0 → logdetective-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: logdetective
-Version: 0.4.0
+Version: 0.5.0
 Summary: Log using LLM AI to search for build/test failures and provide ideas for fixing these.
 License: Apache-2.0
 Author: Jiri Podivin

{logdetective-0.4.0 → logdetective-0.5.0}/logdetective/server/models.py

@@ -21,7 +21,7 @@ class JobHook(BaseModel):
 
     # The identifier of the job. We only care about 'build_rpm' and
     # 'build_centos_stream_rpm' jobs.
-    build_name: str = Field(pattern=r"^build(_.*)?_rpm$")
+    build_name: str = Field(pattern=r"^build.*rpm$")
 
     # A string representing the job status. We only care about 'failed' jobs.
     build_status: str = Field(pattern=r"^failed$")
@@ -90,6 +90,8 @@ class InferenceConfig(BaseModel):
     api_endpoint: Optional[Literal["/chat/completions", "/completions"]] = (
         "/chat/completions"
     )
+    url: str = ""
+    api_token: str = ""
 
     def __init__(self, data: Optional[dict] = None):
         super().__init__()
@@ -99,6 +101,8 @@ class InferenceConfig(BaseModel):
         self.max_tokens = data.get("max_tokens", -1)
         self.log_probs = data.get("log_probs", 1)
         self.api_endpoint = data.get("api_endpoint", "/chat/completions")
+        self.url = data.get("url", "")
+        self.api_token = data.get("api_token", "")
 
 
 class ExtractorConfig(BaseModel):

{logdetective-0.4.0 → logdetective-0.5.0}/logdetective/server/server.py

@@ -3,7 +3,7 @@ import json
 import os
 import re
 import zipfile
-from pathlib import PurePath
+from pathlib import Path, PurePath
 from tempfile import TemporaryFile
 from typing import List, Annotated, Tuple, Dict, Any
 
@@ -13,6 +13,9 @@ from fastapi import FastAPI, HTTPException, BackgroundTasks, Depends, Header
 from fastapi.responses import StreamingResponse
 from fastapi.responses import Response as BasicResponse
 import gitlab
+import gitlab.v4
+import gitlab.v4.objects
+import jinja2
 import requests
 
 from logdetective.constants import (
@@ -38,14 +41,10 @@ from logdetective.server.models import (
     AnalyzedSnippet,
 )
 
-LLM_CPP_HOST = os.environ.get("LLAMA_CPP_HOST", "localhost")
-LLM_CPP_SERVER_ADDRESS = f"http://{LLM_CPP_HOST}"
-LLM_CPP_SERVER_PORT = os.environ.get("LLAMA_CPP_SERVER_PORT", 8000)
 LLM_CPP_SERVER_TIMEOUT = os.environ.get("LLAMA_CPP_SERVER_TIMEOUT", 600)
 LOG_SOURCE_REQUEST_TIMEOUT = os.environ.get("LOG_SOURCE_REQUEST_TIMEOUT", 60)
 API_TOKEN = os.environ.get("LOGDETECTIVE_TOKEN", None)
 SERVER_CONFIG_PATH = os.environ.get("LOGDETECTIVE_SERVER_CONF", None)
-LLM_API_TOKEN = os.environ.get("LLM_API_TOKEN", None)
 
 SERVER_CONFIG = load_server_config(SERVER_CONFIG_PATH)
 
@@ -190,8 +189,8 @@ async def submit_text(  # pylint: disable=R0913,R0917
 
     headers = {"Content-Type": "application/json"}
 
-    if LLM_API_TOKEN:
-        headers["Authorization"] = f"Bearer {LLM_API_TOKEN}"
+    if SERVER_CONFIG.inference.api_token:
+        headers["Authorization"] = f"Bearer {SERVER_CONFIG.inference.api_token}"
 
     if api_endpoint == "/chat/completions":
         return await submit_text_chat_completions(
@@ -224,7 +223,7 @@ async def submit_text_completions(  # pylint: disable=R0913,R0917
     }
 
     response = await submit_to_llm_endpoint(
-        f"{LLM_CPP_SERVER_ADDRESS}:{LLM_CPP_SERVER_PORT}/v1/completions",
+        f"{SERVER_CONFIG.inference.url}/v1/completions",
         data,
         headers,
         stream,
@@ -263,7 +262,7 @@ async def submit_text_chat_completions(  # pylint: disable=R0913,R0917
     }
 
     response = await submit_to_llm_endpoint(
-        f"{LLM_CPP_SERVER_ADDRESS}:{LLM_CPP_SERVER_PORT}/v1/chat/completions",
+        f"{SERVER_CONFIG.inference.url}/v1/chat/completions",
         data,
         headers,
         stream,
@@ -305,8 +304,7 @@ async def analyze_log(build_log: BuildLog):
             LOG.error("Error encountered while computing certainty: %s", ex)
             raise HTTPException(
                 status_code=400,
-                detail=f"Couldn't compute certainty with data:\n"
-                f"{response.logprobs}",
+                detail=f"Couldn't compute certainty with data:\n{response.logprobs}",
             ) from ex
 
     return Response(explanation=response, response_certainty=certainty)
@@ -322,6 +320,12 @@ async def analyze_log_staged(build_log: BuildLog):
     while lacking  result, params or query fields.
     """
     log_text = process_url(build_log.url)
+
+    return await perform_staged_analysis(log_text=log_text)
+
+
+async def perform_staged_analysis(log_text: str) -> StagedResponse:
+    """Submit the log file snippets to the LLM and retrieve their results"""
     log_summary = mine_logs(log_text)
 
     # Process snippets asynchronously
@@ -381,8 +385,8 @@ async def analyze_log_stream(build_log: BuildLog):
     log_summary = format_snippets(log_summary)
     headers = {"Content-Type": "application/json"}
 
-    if LLM_API_TOKEN:
-        headers["Authorization"] = f"Bearer {LLM_API_TOKEN}"
+    if SERVER_CONFIG.inference.api_token:
+        headers["Authorization"] = f"Bearer {SERVER_CONFIG.inference.api_token}"
 
     stream = await submit_text_chat_completions(
         PROMPT_TEMPLATE.format(log_summary), stream=True, headers=headers
@@ -423,6 +427,11 @@ async def process_gitlab_job_event(job_hook):
     # Retrieve data about the job from the GitLab API
     job = await asyncio.to_thread(project.jobs.get, job_hook.build_id)
 
+    # For easy retrieval later, we'll add project_name and project_url to the
+    # job object
+    job.project_name = project.name
+    job.project_url = project.web_url
+
     # Retrieve the pipeline that started this job
     pipeline = await asyncio.to_thread(project.pipelines.get, job_hook.pipeline_id)
 
@@ -438,37 +447,38 @@ async def process_gitlab_job_event(job_hook):
             "Pipeline source is merge_request_event but no merge request ID was provided."
         )
         return
-    merge_request_id = int(match.group(1))
+    merge_request_iid = int(match.group(1))
 
     LOG.debug("Retrieving log artifacts")
     # Retrieve the build logs from the merge request artifacts and preprocess them
     try:
-        preprocessed_log = await retrieve_and_preprocess_koji_logs(job)
+        log_url, preprocessed_log = await retrieve_and_preprocess_koji_logs(job)
     except LogsTooLargeError:
         LOG.error("Could not retrieve logs. Too large.")
         raise
 
     # Submit log to Log Detective and await the results.
-    response = await submit_log_to_llm(preprocessed_log)
+    log_text = preprocessed_log.read().decode(encoding="utf-8")
+    staged_response = await perform_staged_analysis(log_text=log_text)
     preprocessed_log.close()
 
     # Add the Log Detective response as a comment to the merge request
-    await comment_on_mr(merge_request_id, response)
+    await comment_on_mr(project, merge_request_iid, job, log_url, staged_response)
 
 
 class LogsTooLargeError(RuntimeError):
     """The log archive exceeds the configured maximum size"""
 
 
-async def retrieve_and_preprocess_koji_logs(job):
+async def retrieve_and_preprocess_koji_logs(job: gitlab.v4.objects.ProjectJob):
     """Download logs from the merge request artifacts
 
     This function will retrieve the build logs and do some minimal
     preprocessing to determine which log is relevant for analysis.
 
-    returns: An open, file-like object containing the log contents to be sent
-    for processing by Log Detective. The calling function is responsible for
-    closing this object."""
+    returns: The URL pointing to the selected log file and an open, file-like
+    object containing the log contents to be sent for processing by Log
+    Detective. The calling function is responsible for closing this object."""
 
     # Make sure the file isn't too large to process.
     if not await check_artifacts_file_size(job):
@@ -551,11 +561,13 @@ async def retrieve_and_preprocess_koji_logs(job):
 
     LOG.debug("Failed architecture: %s", failed_arch)
 
-    log_path = failed_arches[failed_arch]
-    LOG.debug("Returning contents of %s", log_path)
+    log_path = failed_arches[failed_arch].as_posix()
+
+    log_url = f"{SERVER_CONFIG.gitlab.api_url}/projects/{job.project_id}/jobs/{job.id}/artifacts/{log_path}"  # pylint: disable=line-too-long
+    LOG.debug("Returning contents of %s", log_url)
 
     # Return the log as a file-like object with .read() function
-    return artifacts_zip.open(log_path.as_posix())
+    return log_url, artifacts_zip.open(log_path)
 
 
 async def check_artifacts_file_size(job):
@@ -582,15 +594,62 @@ async def check_artifacts_file_size(job):
     return content_length <= SERVER_CONFIG.gitlab.max_artifact_size
 
 
-async def submit_log_to_llm(log):
-    """Stream the log to the LLM for processing"""
-    # TODO: query the LLM with the log contents  # pylint: disable=fixme
-    # This function will be implemented later; right now it does nothing.
-    LOG.debug("Log contents:\n%s", log.read())
-    return ""
+async def comment_on_mr(
+    project: gitlab.v4.objects.Project,
+    merge_request_iid: int,
+    job: gitlab.v4.objects.ProjectJob,
+    log_url: str,
+    response: StagedResponse,
+):
+    """Add the Log Detective response as a comment to the merge request"""
+    LOG.debug(
+        "Primary Explanation for %s MR %d: %s",
+        project.name,
+        merge_request_iid,
+        response.explanation.text,
+    )
 
+    # Get the formatted comment.
+    comment = await generate_mr_comment(job, log_url, response)
 
-async def comment_on_mr(merge_request_id: int, response: str):  # pylint: disable=unused-argument
-    """Add the Log Detective response as a comment to the merge request"""
-    # TODO: Implement this  # pylint: disable=fixme
-    pass  # pylint: disable=unnecessary-pass
+    # Look up the merge request
+    merge_request = await asyncio.to_thread(
+        project.mergerequests.get, merge_request_iid
+    )
+
+    # Submit a new comment to the Merge Request using the Gitlab API
+    await asyncio.to_thread(merge_request.discussions.create, {"body": comment})
+
+
+async def generate_mr_comment(
+    job: gitlab.v4.objects.ProjectJob, log_url: str, response: StagedResponse
+) -> str:
+    """Use a template to generate a comment string to submit to Gitlab"""
+
+    # Locate and load the comment template
+    script_path = Path(__file__).resolve().parent
+    template_path = Path(script_path, "templates")
+    jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_path))
+    tpl = jinja_env.get_template("gitlab_comment.md.j2")
+
+    artifacts_url = f"{job.project_url}/-/jobs/{job.id}/artifacts/download"
+
+    if response.response_certainty >= 90:
+        emoji_face = ":slight_smile:"
+    elif response.response_certainty >= 70:
+        emoji_face = ":neutral_face:"
+    else:
+        emoji_face = ":frowning2:"
+
+    # Generate the comment from the template
+    content = tpl.render(
+        package=job.project_name,
+        explanation=response.explanation.text,
+        certainty=f"{response.response_certainty:.2f}",
+        emoji_face=emoji_face,
+        snippets=response.snippets,
+        log_url=log_url,
+        artifacts_url=artifacts_url,
+    )
+
+    return content

logdetective-0.5.0/logdetective/server/templates/gitlab_comment.md.j2

@@ -0,0 +1,66 @@
+The package {{ package }} failed to build, here is a possible explanation why.
+
+Please know that the explanation was provided by AI and may be incorrect.
+In this case, we are {{ certainty }}% certain of the response {{ emoji_face }}.
+
+{{ explanation }}
+
+<details>
+<ul>
+{% for snippet in snippets %}
+<li>
+<code>
+Line {{ snippet.line_number }}: {{ snippet.text }}
+</code>
+{{ snippet.explanation }}
+</li>
+{% endfor %}
+</ul>
+</details>
+
+<details>
+  <summary>Logs</summary>
+  <p>
+    Log Detective analyzed the following logs files to provide an explanation:
+  </p>
+
+  <ul>
+    <li><a href="{{ log_url }}">{{ log_url }}</a></li>
+  </ul>
+
+  <p>
+    Additional logs are available from:
+    <ul>
+    <li><a href="{{ artifacts_url }}">artifacts.zip</a></li>
+  </ul>
+  </p>
+
+  <p>
+    Please know that these log files are automatically removed after some
+    time, so you might need a backup.
+  </p>
+</details>
+
+<details>
+  <summary>Help</summary>
+  <p>Don't hesitate to reach out.</p>
+
+  <ul>
+    <li><a href="https://github.com/fedora-copr/logdetective">Upstream</a></li>
+    <li><a href="https://github.com/fedora-copr/logdetective/issues">Issue tracker</a></li>
+    <li><a href="https://redhat.enterprise.slack.com/archives/C06DWNVKKDE">Slack</a></li>
+    <li><a href="https://log-detective.com/documentation">Documentation</a></li>
+  </ul>
+</details>
+
+
+---
+This comment was created by [Log Detective][log-detective].
+
+Was the provided feedback accurate and helpful? <br>Please vote with :thumbsup:
+or :thumbsdown: to help us improve.<br>
+
+
+
+[log-detective]: https://log-detective.com/
+[contact]: https://github.com/fedora-copr

{logdetective-0.4.0 → logdetective-0.5.0}/logdetective/server/utils.py

@@ -12,7 +12,8 @@ def load_server_config(path: str | None) -> Config:
             with open(path, "r") as config_file:
                 return Config(yaml.safe_load(config_file))
         except FileNotFoundError:
-            pass
+            # This is not an error, we will fall back to default
+            print("Unable to find server config file, using default then.")
     return Config()
 
 

{logdetective-0.4.0 → logdetective-0.5.0}/pyproject.toml

@@ -1,11 +1,14 @@
 [tool.poetry]
 name = "logdetective"
-version = "0.4.0"
+version = "0.5.0"
 description = "Log using LLM AI to search for build/test failures and provide ideas for fixing these."
 authors = ["Jiri Podivin <jpodivin@gmail.com>"]
 license = "Apache-2.0"
 readme = "README.md"
-include = ["logdetective/drain3.ini"]
+include = [
+    "logdetective/drain3.ini",
+    "logdetective/server/templates/gitlab_comment.md.j2",
+]
 packages = [
     { include = "logdetective" }
 ]