logdetective 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

logdetective/server/metric.py

@@ -1,6 +1,5 @@
 import io
 import inspect
-import logging
 import datetime
 
 from typing import Union
@@ -9,12 +8,11 @@ from functools import wraps
 import aiohttp
 
 from starlette.responses import StreamingResponse
-from logdetective.server.database.models import EndpointType, AnalyzeRequestMetrics
-from logdetective.server.remote_log import RemoteLog
 from logdetective.server import models
-from logdetective.server.compressors import LLMResponseCompressor
-
-LOG = logging.getLogger("logdetective")
+from logdetective.remote_log import RemoteLog
+from logdetective.server.config import LOG
+from logdetective.server.compressors import LLMResponseCompressor, RemoteLogCompressor
+from logdetective.server.database.models import EndpointType, AnalyzeRequestMetrics
 
 
 async def add_new_metrics(
@@ -31,7 +29,9 @@ async def add_new_metrics(
     and the log (in a zip format) for which analysis is requested.
     """
     remote_log = RemoteLog(url, http_session)
-    compressed_log_content = compressed_log_content or await remote_log.zip_content()
+    compressed_log_content = (
+        compressed_log_content or await RemoteLogCompressor(remote_log).zip_content()
+    )
     return AnalyzeRequestMetrics.create(
         endpoint=EndpointType(api_name),
         compressed_log=compressed_log_content,
@@ -58,7 +58,8 @@ def update_metrics(
         compressed_response = None
         LOG.warning(
             "Given response can not be serialized "
-            "and saved in db (probably a StreamingResponse): %s.", e
+            "and saved in db (probably a StreamingResponse): %s.",
+            e,
         )
 
     response_sent_at = (

logdetective/server/models.py

@@ -9,7 +9,15 @@ from pydantic import (
     NonNegativeFloat,
     HttpUrl,
 )
-from logdetective.constants import DEFAULT_TEMPERATURE
+
+from aiolimiter import AsyncLimiter
+from gitlab import Gitlab
+
+from logdetective.constants import (
+    DEFAULT_TEMPERATURE,
+    LLM_DEFAULT_MAX_QUEUE_SIZE,
+    LLM_DEFAULT_REQUESTS_PER_MINUTE,
+)
 
 
 class BuildLog(BaseModel):
@@ -46,6 +54,33 @@ class JobHook(BaseModel):
     project_id: int
 
 
+class EmojiMergeRequest(BaseModel):
+    """Model of the 'merge_request' subsection of Emoji webhook messages.
+    This model implements only the fields that we care about. The webhook
+    sends many more fields that we will ignore."""
+
+    # The identifier of the target project
+    target_project_id: int
+
+    # The internal identifier (relative to the target project)
+    iid: int
+
+
+class EmojiHook(BaseModel):
+    """Model of Job Hook events sent from GitLab.
+    Full details of the specification are available at
+    https://docs.gitlab.com/user/project/integrations/webhook_events/#job-events
+    This model implements only the fields that we care about. The webhook
+    sends many more fields that we will ignore."""
+
+    # The kind of webhook message. We are only interested in 'emoji' messages
+    # which represents awarding or revoking emoji reactions on notes.
+    object_kind: str = Field(pattern=r"^emoji$")
+
+    # Information about the merge request this emoji applies to, if any.
+    merge_request: EmojiMergeRequest = Field(default=None)
+
+
 class Explanation(BaseModel):
     """Model of snippet or general log explanation from Log Detective"""
 
@@ -92,7 +127,7 @@ class StagedResponse(Response):
     snippets: List[AnalyzedSnippet]
 
 
-class InferenceConfig(BaseModel):
+class InferenceConfig(BaseModel):  # pylint: disable=too-many-instance-attributes
     """Model for inference configuration of logdetective server."""
 
     max_tokens: int = -1
@@ -104,6 +139,9 @@ class InferenceConfig(BaseModel):
     api_token: str = ""
     model: str = ""
     temperature: NonNegativeFloat = DEFAULT_TEMPERATURE
+    max_queue_size: int = LLM_DEFAULT_MAX_QUEUE_SIZE
+    request_period: float = 60.0 / LLM_DEFAULT_REQUESTS_PER_MINUTE
+    _limiter: AsyncLimiter = AsyncLimiter(LLM_DEFAULT_REQUESTS_PER_MINUTE)
 
     def __init__(self, data: Optional[dict] = None):
         super().__init__()
@@ -117,6 +155,16 @@ class InferenceConfig(BaseModel):
         self.api_token = data.get("api_token", "")
         self.model = data.get("model", "default-model")
         self.temperature = data.get("temperature", DEFAULT_TEMPERATURE)
+        self.max_queue_size = data.get("max_queue_size", LLM_DEFAULT_MAX_QUEUE_SIZE)
+
+        self._requests_per_minute = data.get(
+            "requests_per_minute", LLM_DEFAULT_REQUESTS_PER_MINUTE
+        )
+        self._limiter = AsyncLimiter(self._requests_per_minute)
+
+    def get_limiter(self):
+        """Return the limiter object so it can be used as a context manager"""
+        return self._limiter
 
 
 class ExtractorConfig(BaseModel):
@@ -136,26 +184,50 @@ class ExtractorConfig(BaseModel):
         self.verbose = data.get("verbose", False)
 
 
-class GitLabConfig(BaseModel):
+class GitLabInstanceConfig(BaseModel):
     """Model for GitLab configuration of logdetective server."""
 
+    name: str = None
     url: str = None
     api_url: str = None
     api_token: str = None
+    _conn: Gitlab = None
 
     # Maximum size of artifacts.zip in MiB. (default: 300 MiB)
     max_artifact_size: int = 300
 
-    def __init__(self, data: Optional[dict] = None):
+    def __init__(self, name: str, data: Optional[dict] = None):
         super().__init__()
         if data is None:
             return
 
+        self.name = name
         self.url = data.get("url", "https://gitlab.com")
         self.api_url = f"{self.url}/api/v4"
         self.api_token = data.get("api_token", None)
         self.max_artifact_size = int(data.get("max_artifact_size")) * 1024 * 1024
 
+        self._conn = Gitlab(url=self.url, private_token=self.api_token)
+
+    def get_connection(self):
+        """Get the Gitlab connection object"""
+        return self._conn
+
+
+class GitLabConfig(BaseModel):
+    """Model for GitLab configuration of logdetective server."""
+
+    instances: Dict[str, GitLabInstanceConfig] = {}
+
+    def __init__(self, data: Optional[dict] = None):
+        super().__init__()
+        if data is None:
+            return
+
+        for instance_name, instance_data in data.items():
+            instance = GitLabInstanceConfig(instance_name, instance_data)
+            self.instances[instance.url] = instance
+
 
 class LogConfig(BaseModel):
     """Logging configuration"""
@@ -232,7 +304,7 @@ class TimePeriod(BaseModel):
     @model_validator(mode="before")
     @classmethod
     def check_exclusive_fields(cls, data):
-        """ Check that only one key between weeks, days and hours is defined"""
+        """Check that only one key between weeks, days and hours is defined"""
         if isinstance(data, dict):
             how_many_fields = sum(
                 1
@@ -284,6 +356,6 @@ class TimePeriod(BaseModel):
             datetime.datetime: The start time of the period.
         """
         time = end_time or datetime.datetime.now(datetime.timezone.utc)
-        if end_time.tzinfo is None:
+        if time.tzinfo is None:
             end_time = end_time.replace(tzinfo=datetime.timezone.utc)
         return time - self.get_time_period()