logdetective 0.6.0__tar.gz → 0.9.0__tar.gz

{logdetective-0.6.0 → logdetective-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: logdetective
-Version: 0.6.0
+Version: 0.9.0
 Summary: Log using LLM AI to search for build/test failures and provide ideas for fixing these.
 License: Apache-2.0
 Author: Jiri Podivin
@@ -22,6 +22,7 @@ Provides-Extra: server
 Provides-Extra: server-testing
 Requires-Dist: aiohttp (>=3.7.4)
 Requires-Dist: alembic (>=1.13.3,<2.0.0) ; extra == "server" or extra == "server-testing"
+Requires-Dist: backoff (==2.2.1) ; extra == "server" or extra == "server-testing"
 Requires-Dist: drain3 (>=0.9.11,<0.10.0)
 Requires-Dist: fastapi (>=0.111.1) ; extra == "server" or extra == "server-testing"
 Requires-Dist: huggingface-hub (>0.23.2)
@@ -33,6 +34,7 @@ Requires-Dist: psycopg2-binary (>=2.9.9,<3.0.0) ; extra == "server-testing"
 Requires-Dist: pydantic (>=2.8.2,<3.0.0)
 Requires-Dist: python-gitlab (>=4.4.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
+Requires-Dist: sentry-sdk[fastapi] (>=2.17.0,<3.0.0)
 Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "server" or extra == "server-testing"
 Project-URL: homepage, https://github.com/fedora-copr/logdetective
 Project-URL: issues, https://github.com/fedora-copr/logdetective/issues
@@ -357,13 +359,16 @@ When no time period is specified, the query defaults to the last 2 days:
 
 You can view requests and responses statistics
  - for the `/analyze` endpoint at http://localhost:8080/metrics/analyze
- - for the `/analyze/staged` endpoint at http://localhost:8080/metrics/analyze/staged.
+ - for the `/analyze-staged` endpoint at http://localhost:8080/metrics/analyze-staged.
+ - for the requests coming from gitlab: http://localhost:8080/metrics/analyze-gitlab.
 
 You can retrieve single svg images at the following endpoints:
- - `/metrics/analyze/requests`
- - `/metrics/analyze/responses`
- - `/metrics/analyze/staged/requests`
- - `/metrics/analyze/stages/responses`
+ - http://localhost:8080/metrics/analyze/requests
+ - http://localhost:8080/metrics/analyze/responses
+ - http://localhost:8080/metrics/analyze-staged/requests
+ - http://localhost:8080/metrics/analyze-staged/responses
+ - http://localhost:8080/metrics/analyze-gitlab/requests
+ - http://localhost:8080/metrics/analyze-gitlab/responses
 
 Examples:
 

{logdetective-0.6.0 → logdetective-0.9.0}/README.md

@@ -317,13 +317,16 @@ When no time period is specified, the query defaults to the last 2 days:
 
 You can view requests and responses statistics
  - for the `/analyze` endpoint at http://localhost:8080/metrics/analyze
- - for the `/analyze/staged` endpoint at http://localhost:8080/metrics/analyze/staged.
+ - for the `/analyze-staged` endpoint at http://localhost:8080/metrics/analyze-staged.
+ - for the requests coming from gitlab: http://localhost:8080/metrics/analyze-gitlab.
 
 You can retrieve single svg images at the following endpoints:
- - `/metrics/analyze/requests`
- - `/metrics/analyze/responses`
- - `/metrics/analyze/staged/requests`
- - `/metrics/analyze/stages/responses`
+ - http://localhost:8080/metrics/analyze/requests
+ - http://localhost:8080/metrics/analyze/responses
+ - http://localhost:8080/metrics/analyze-staged/requests
+ - http://localhost:8080/metrics/analyze-staged/responses
+ - http://localhost:8080/metrics/analyze-gitlab/requests
+ - http://localhost:8080/metrics/analyze-gitlab/responses
 
 Examples:
 

{logdetective-0.6.0 → logdetective-0.9.0}/logdetective/prompts.yml

@@ -4,7 +4,7 @@
 # The defaults are stored in constants.py
 
 prompt_template: |
-  Given following log snippets, and nothing else, explain what failure, if any, occured during build of this package.
+  Given following log snippets, and nothing else, explain what failure, if any, occurred during build of this package.
 
   Analysis of the snippets must be in a format of [X] : [Y], where [X] is a log snippet, and [Y] is the explanation.
   Snippets themselves must not be altered in any way whatsoever.
@@ -44,15 +44,15 @@ snippet_prompt_template: |
   Analysis:
 
 prompt_template_staged: |
-  Given following log snippets, their explanation, and nothing else, explain what failure, if any, occured during build of this package.
+  Given following log snippets, their explanation, and nothing else, explain what failure, if any, occurred during build of this package.
 
   Snippets are in a format of [X] : [Y], where [X] is a log snippet, and [Y] is the explanation.
 
   Snippets are delimited with '================'.
 
-  Drawing on information from all snippets, provide complete explanation of the issue and recommend solution.
+  Drawing on information from all snippets, provide a concise explanation of the issue and recommend a solution.
 
-  Explanation of the issue, and recommended solution, should take handful of sentences.
+  Explanation of the issue, and recommended solution, should take a handful of sentences.
 
   Snippets:
 

logdetective-0.9.0/logdetective/server/compressors.py

@@ -0,0 +1,144 @@
+import io
+import logging
+import zipfile
+
+from typing import Union, Dict
+from logdetective.server.models import (
+    StagedResponse,
+    Response,
+    AnalyzedSnippet,
+    Explanation,
+)
+
+
+LOG = logging.getLogger("logdetective")
+
+
+class TextCompressor:
+    """
+    Encapsulates one or more texts in one or more files with the specified names
+    and provides methods to retrieve them later.
+    """
+
+    def zip(self, items: Dict[str, str]) -> bytes:
+        """
+        Compress multiple texts into different files within a zip archive.
+
+        Args:
+            items: Dictionary where keys are file names and values are text content
+                  to be compressed
+
+        Returns:
+            bytes: The compressed zip archive as bytes
+        """
+        zip_buffer = io.BytesIO()
+        with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file:
+            for key, value in items.items():
+                zip_file.writestr(key, value)
+
+        zip_buffer.seek(0)
+        return zip_buffer.getvalue()
+
+    def unzip(self, zip_data: Union[bytes, io.BytesIO]) -> str:
+        """
+        Uncompress data created by TextCompressor.zip().
+
+        Args:
+            zip_data: A zipped stream of bytes or BytesIO object
+
+        Returns:
+            {file_name: str}: The decompressed content as a dict of file names and UTF-8 strings
+        """
+        if isinstance(zip_data, bytes):
+            zip_buffer = io.BytesIO(zip_data)
+        else:
+            zip_buffer = zip_data
+
+        content = {}
+        with zipfile.ZipFile(zip_buffer, "r") as zip_file:
+            file_list = zip_file.namelist()
+            for file_name in file_list:
+                content[file_name] = zip_file.read(file_name).decode("utf-8")
+
+        return content
+
+
+class LLMResponseCompressor:
+    """
+    Handles compression and decompression of LLM responses.
+    """
+
+    EXPLANATION_FILE_NAME = "explanation.txt"
+    SNIPPET_FILE_NAME = "snippet_{number}.txt"
+    COMPRESSOR = TextCompressor()
+
+    def __init__(self, response: Union[StagedResponse, Response]):
+        """
+        Initialize with an LLM response.
+
+        Args:
+            response: Either a StagedResponse or Response object
+        """
+        self._response = response
+
+    def zip_response(self) -> bytes:
+        """
+        Compress the content of the LLM response.
+
+        Returns:
+            bytes: Compressed response as bytes
+        """
+        items = {
+            self.EXPLANATION_FILE_NAME: self._response.explanation.model_dump_json()
+        }
+
+        if isinstance(self._response, StagedResponse):
+            for i, snippet in enumerate(self._response.snippets):
+                items[self.SNIPPET_FILE_NAME.format(number=i)] = (
+                    snippet.model_dump_json()
+                )
+
+        return self.COMPRESSOR.zip(items)
+
+    @classmethod
+    def unzip(
+        cls, zip_data: Union[bytes, io.BytesIO]
+    ) -> Union[StagedResponse, Response]:
+        """
+        Uncompress the zipped content of the LLM response.
+
+        Args:
+            zip_data: Compressed data as bytes or BytesIO
+
+        Returns:
+            Union[StagedResponse, Response]: The decompressed (partial) response object,
+            missing response_certainty.
+        """
+        items = cls.COMPRESSOR.unzip(zip_data)
+        if cls.EXPLANATION_FILE_NAME not in items:
+            raise KeyError(
+                f"Required file {cls.EXPLANATION_FILE_NAME} not found in zip archive"
+            )
+        explanation = Explanation.model_validate_json(items[cls.EXPLANATION_FILE_NAME])
+
+        snippets = []
+        snippet_files = {
+            k: v
+            for k, v in items.items()
+            if cls.SNIPPET_FILE_NAME.replace("{number}.txt", "") in k
+        }
+        for i in range(len(snippet_files)):
+            snippets.append(
+                AnalyzedSnippet.model_validate_json(
+                    items[cls.SNIPPET_FILE_NAME.format(number=i)]
+                )
+            )
+
+        if snippets:
+            response = StagedResponse(
+                explanation=explanation, snippets=snippets, response_certainty=0
+            )
+        else:
+            response = Response(explanation=explanation, response_certainty=0)
+
+        return response

{logdetective-0.6.0 → logdetective-0.9.0}/logdetective/server/database/base.py

@@ -61,3 +61,6 @@ def destroy():
     """Destroy db"""
     Base.metadata.drop_all(engine)
     logger.warning("Database cleaned")
+
+
+DB_MAX_RETRIES = 3  # How many times retry a db operation

logdetective-0.9.0/logdetective/server/database/models/__init__.py

@@ -0,0 +1,21 @@
+from logdetective.server.database.base import Base
+from logdetective.server.database.models.merge_request_jobs import (
+    Forge,
+    GitlabMergeRequestJobs,
+    Comments,
+    Reactions,
+)
+from logdetective.server.database.models.metrics import (
+    AnalyzeRequestMetrics,
+    EndpointType,
+)
+
+__all__ = [
+    Base.__name__,
+    GitlabMergeRequestJobs.__name__,
+    Comments.__name__,
+    Reactions.__name__,
+    AnalyzeRequestMetrics.__name__,
+    EndpointType.__name__,
+    Forge.__name__,
+]