logdetective 0.5.11__py3-none-any.whl → 0.9.0__py3-none-any.whl

logdetective/logdetective.py

@@ -1,8 +1,11 @@
 import argparse
+import asyncio
 import logging
 import sys
 import os
 
+import aiohttp
+
 from logdetective.constants import DEFAULT_ADVISOR, DEFAULT_TEMPERATURE
 from logdetective.utils import (
     process_log,
@@ -82,7 +85,7 @@ def setup_args():
     return parser.parse_args()
 
 
-def main():  # pylint: disable=too-many-statements,too-many-locals
+async def run():  # pylint: disable=too-many-statements,too-many-locals
     """Main execution function."""
     args = setup_args()
 
@@ -128,13 +131,14 @@ def main():  # pylint: disable=too-many-statements,too-many-locals
 
     LOG.info("Getting summary")
 
-    try:
-        log = retrieve_log_content(args.file)
-    except ValueError as e:
-        # file does not exists
-        LOG.error(e)
-        sys.exit(4)
-    log_summary = extractor(log)
+    async with aiohttp.ClientSession() as http:
+        try:
+            log = await retrieve_log_content(http, args.file)
+        except ValueError as e:
+            # file does not exist
+            LOG.error(e)
+            sys.exit(4)
+        log_summary = extractor(log)
 
     ratio = len(log_summary) / len(log.split("\n"))
 
@@ -182,5 +186,10 @@ def main():  # pylint: disable=too-many-statements,too-many-locals
     print(f"\nResponse certainty: {certainty:.2f}%\n")
 
 
+def main():
+    """ Evaluate logdetective program and wait for it to finish """
+    asyncio.run(run())
+
+
 if __name__ == "__main__":
     main()

logdetective/prompts.yml

@@ -4,7 +4,7 @@
 # The defaults are stored in constants.py
 
 prompt_template: |
-  Given following log snippets, and nothing else, explain what failure, if any, occured during build of this package.
+  Given following log snippets, and nothing else, explain what failure, if any, occurred during build of this package.
 
   Analysis of the snippets must be in a format of [X] : [Y], where [X] is a log snippet, and [Y] is the explanation.
   Snippets themselves must not be altered in any way whatsoever.
@@ -44,15 +44,15 @@ snippet_prompt_template: |
   Analysis:
 
 prompt_template_staged: |
-  Given following log snippets, their explanation, and nothing else, explain what failure, if any, occured during build of this package.
+  Given following log snippets, their explanation, and nothing else, explain what failure, if any, occurred during build of this package.
 
   Snippets are in a format of [X] : [Y], where [X] is a log snippet, and [Y] is the explanation.
 
   Snippets are delimited with '================'.
 
-  Drawing on information from all snippets, provide complete explanation of the issue and recommend solution.
+  Drawing on information from all snippets, provide a concise explanation of the issue and recommend a solution.
 
-  Explanation of the issue, and recommended solution, should take handful of sentences.
+  Explanation of the issue, and recommended solution, should take a handful of sentences.
 
   Snippets:
 

logdetective/server/compressors.py

@@ -0,0 +1,144 @@
+import io
+import logging
+import zipfile
+
+from typing import Union, Dict
+from logdetective.server.models import (
+    StagedResponse,
+    Response,
+    AnalyzedSnippet,
+    Explanation,
+)
+
+
+LOG = logging.getLogger("logdetective")
+
+
+class TextCompressor:
+    """
+    Encapsulates one or more texts in one or more files with the specified names
+    and provides methods to retrieve them later.
+    """
+
+    def zip(self, items: Dict[str, str]) -> bytes:
+        """
+        Compress multiple texts into different files within a zip archive.
+
+        Args:
+            items: Dictionary where keys are file names and values are text content
+                  to be compressed
+
+        Returns:
+            bytes: The compressed zip archive as bytes
+        """
+        zip_buffer = io.BytesIO()
+        with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file:
+            for key, value in items.items():
+                zip_file.writestr(key, value)
+
+        zip_buffer.seek(0)
+        return zip_buffer.getvalue()
+
+    def unzip(self, zip_data: Union[bytes, io.BytesIO]) -> str:
+        """
+        Uncompress data created by TextCompressor.zip().
+
+        Args:
+            zip_data: A zipped stream of bytes or BytesIO object
+
+        Returns:
+            {file_name: str}: The decompressed content as a dict of file names and UTF-8 strings
+        """
+        if isinstance(zip_data, bytes):
+            zip_buffer = io.BytesIO(zip_data)
+        else:
+            zip_buffer = zip_data
+
+        content = {}
+        with zipfile.ZipFile(zip_buffer, "r") as zip_file:
+            file_list = zip_file.namelist()
+            for file_name in file_list:
+                content[file_name] = zip_file.read(file_name).decode("utf-8")
+
+        return content
+
+
+class LLMResponseCompressor:
+    """
+    Handles compression and decompression of LLM responses.
+    """
+
+    EXPLANATION_FILE_NAME = "explanation.txt"
+    SNIPPET_FILE_NAME = "snippet_{number}.txt"
+    COMPRESSOR = TextCompressor()
+
+    def __init__(self, response: Union[StagedResponse, Response]):
+        """
+        Initialize with an LLM response.
+
+        Args:
+            response: Either a StagedResponse or Response object
+        """
+        self._response = response
+
+    def zip_response(self) -> bytes:
+        """
+        Compress the content of the LLM response.
+
+        Returns:
+            bytes: Compressed response as bytes
+        """
+        items = {
+            self.EXPLANATION_FILE_NAME: self._response.explanation.model_dump_json()
+        }
+
+        if isinstance(self._response, StagedResponse):
+            for i, snippet in enumerate(self._response.snippets):
+                items[self.SNIPPET_FILE_NAME.format(number=i)] = (
+                    snippet.model_dump_json()
+                )
+
+        return self.COMPRESSOR.zip(items)
+
+    @classmethod
+    def unzip(
+        cls, zip_data: Union[bytes, io.BytesIO]
+    ) -> Union[StagedResponse, Response]:
+        """
+        Uncompress the zipped content of the LLM response.
+
+        Args:
+            zip_data: Compressed data as bytes or BytesIO
+
+        Returns:
+            Union[StagedResponse, Response]: The decompressed (partial) response object,
+            missing response_certainty.
+        """
+        items = cls.COMPRESSOR.unzip(zip_data)
+        if cls.EXPLANATION_FILE_NAME not in items:
+            raise KeyError(
+                f"Required file {cls.EXPLANATION_FILE_NAME} not found in zip archive"
+            )
+        explanation = Explanation.model_validate_json(items[cls.EXPLANATION_FILE_NAME])
+
+        snippets = []
+        snippet_files = {
+            k: v
+            for k, v in items.items()
+            if cls.SNIPPET_FILE_NAME.replace("{number}.txt", "") in k
+        }
+        for i in range(len(snippet_files)):
+            snippets.append(
+                AnalyzedSnippet.model_validate_json(
+                    items[cls.SNIPPET_FILE_NAME.format(number=i)]
+                )
+            )
+
+        if snippets:
+            response = StagedResponse(
+                explanation=explanation, snippets=snippets, response_certainty=0
+            )
+        else:
+            response = Response(explanation=explanation, response_certainty=0)
+
+        return response

logdetective/server/database/base.py

@@ -61,3 +61,6 @@ def destroy():
     """Destroy db"""
     Base.metadata.drop_all(engine)
     logger.warning("Database cleaned")
+
+
+DB_MAX_RETRIES = 3  # How many times retry a db operation

logdetective/server/database/models/__init__.py

@@ -0,0 +1,21 @@
+from logdetective.server.database.base import Base
+from logdetective.server.database.models.merge_request_jobs import (
+    Forge,
+    GitlabMergeRequestJobs,
+    Comments,
+    Reactions,
+)
+from logdetective.server.database.models.metrics import (
+    AnalyzeRequestMetrics,
+    EndpointType,
+)
+
+__all__ = [
+    Base.__name__,
+    GitlabMergeRequestJobs.__name__,
+    Comments.__name__,
+    Reactions.__name__,
+    AnalyzeRequestMetrics.__name__,
+    EndpointType.__name__,
+    Forge.__name__,
+]