logdetective 2.0.1__py3-none-any.whl → 2.11.0__py3-none-any.whl

logdetective/server/database/models/metrics.py

@@ -1,12 +1,11 @@
-import io
+from __future__ import annotations
 import enum
 import datetime
-from typing import Optional, List, Self, Tuple
+from typing import Optional, List, Self, Tuple, TYPE_CHECKING
 
 import backoff
 
 from sqlalchemy import (
-    Column,
     Integer,
     Float,
     DateTime,
@@ -17,7 +16,7 @@ from sqlalchemy import (
     ForeignKey,
     LargeBinary,
 )
-from sqlalchemy.orm import relationship, aliased
+from sqlalchemy.orm import Mapped, mapped_column, relationship, aliased
 from sqlalchemy.exc import OperationalError
 
 from logdetective.server.database.base import Base, transaction, DB_MAX_RETRIES
@@ -27,6 +26,10 @@ from logdetective.server.database.models.merge_request_jobs import (
 )
 
 
+if TYPE_CHECKING:
+    from .koji import KojiTaskAnalysis
+
+
 class EndpointType(enum.Enum):
     """Different analyze endpoints"""
 
@@ -42,43 +45,45 @@ class AnalyzeRequestMetrics(Base):
 
     __tablename__ = "analyze_request_metrics"
 
-    id = Column(Integer, primary_key=True)
-    endpoint = Column(
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    endpoint: Mapped[EndpointType] = mapped_column(
         Enum(EndpointType),
         nullable=False,
         index=True,
         comment="The service endpoint that was called",
     )
-    request_received_at = Column(
-        DateTime,
+    request_received_at: Mapped[datetime.datetime] = mapped_column(
+        DateTime(timezone=True),
         nullable=False,
         index=True,
         default=datetime.datetime.now(datetime.timezone.utc),
         comment="Timestamp when the request was received",
     )
-    compressed_log = Column(
+    compressed_log: Mapped[bytes] = mapped_column(
         LargeBinary(length=314572800),  # 300MB limit (300 * 1024 * 1024)
         nullable=False,
         index=False,
         comment="Log processed, saved in a zip format",
     )
-    compressed_response = Column(
+    compressed_response: Mapped[Optional[bytes]] = mapped_column(
         LargeBinary(length=314572800),  # 300MB limit (300 * 1024 * 1024)
         nullable=True,
         index=False,
         comment="Given response (with explanation and snippets) saved in a zip format",
     )
-    response_sent_at = Column(
-        DateTime, nullable=True, comment="Timestamp when the response was sent back"
+    response_sent_at: Mapped[Optional[datetime.datetime]] = mapped_column(
+        DateTime(timezone=True),
+        nullable=True,
+        comment="Timestamp when the response was sent back",
     )
-    response_length = Column(
+    response_length: Mapped[Optional[int]] = mapped_column(
         Integer, nullable=True, comment="Length of the response in chars"
     )
-    response_certainty = Column(
+    response_certainty: Mapped[Optional[float]] = mapped_column(
         Float, nullable=True, comment="Certainty for generated response"
     )
 
-    merge_request_job_id = Column(
+    merge_request_job_id: Mapped[Optional[int]] = mapped_column(
         Integer,
         ForeignKey("gitlab_merge_request_jobs.id"),
         nullable=True,
@@ -86,19 +91,27 @@ class AnalyzeRequestMetrics(Base):
         comment="Is this an analyze request coming from a merge request?",
     )
 
-    mr_job = relationship("GitlabMergeRequestJobs", back_populates="request_metrics")
+    mr_job: Mapped[Optional["GitlabMergeRequestJobs"]] = relationship(
+        "GitlabMergeRequestJobs",
+        back_populates="request_metrics"
+    )
+
+    koji_tasks: Mapped[List["KojiTaskAnalysis"]] = relationship(
+        "KojiTaskAnalysis",
+        back_populates="response"
+    )
 
     @classmethod
     @backoff.on_exception(backoff.expo, OperationalError, max_tries=DB_MAX_RETRIES)
-    def create(
+    async def create(
         cls,
         endpoint: EndpointType,
-        compressed_log: io.BytesIO,
+        compressed_log: bytes,
         request_received_at: Optional[datetime.datetime] = None,
     ) -> int:
         """Create AnalyzeRequestMetrics new line
         with data related to a received request"""
-        with transaction(commit=True) as session:
+        async with transaction(commit=True) as session:
             metrics = AnalyzeRequestMetrics()
             metrics.endpoint = endpoint
             metrics.compressed_log = compressed_log
@@ -106,12 +119,12 @@ class AnalyzeRequestMetrics(Base):
                 datetime.timezone.utc
             )
             session.add(metrics)
-            session.flush()
+            await session.flush()
             return metrics.id
 
     @classmethod
     @backoff.on_exception(backoff.expo, OperationalError, max_tries=DB_MAX_RETRIES)
-    def update(  # pylint: disable=too-many-arguments disable=too-many-positional-arguments
+    async def update(  # pylint: disable=too-many-arguments disable=too-many-positional-arguments
         cls,
         id_: int,
         response_sent_at: DateTime,
@@ -121,8 +134,10 @@ class AnalyzeRequestMetrics(Base):
     ) -> None:
         """Update a row
         with data related to the given response"""
-        with transaction(commit=True) as session:
-            metrics = session.query(AnalyzeRequestMetrics).filter_by(id=id_).first()
+        query = select(AnalyzeRequestMetrics).filter(AnalyzeRequestMetrics.id == id_)
+        async with transaction(commit=True) as session:
+            query_result = await session.execute(query)
+            metrics = query_result.scalars().first()
             if metrics is None:
                 raise ValueError("Returned `AnalyzeRequestMetrics` table is empty.")
             metrics.response_sent_at = response_sent_at
@@ -133,19 +148,21 @@ class AnalyzeRequestMetrics(Base):
 
     @classmethod
     @backoff.on_exception(backoff.expo, OperationalError, max_tries=DB_MAX_RETRIES)
-    def get_metric_by_id(
+    async def get_metric_by_id(
         cls,
         id_: int,
     ) -> Self:
         """Update a row
         with data related to the given response"""
-        with transaction(commit=True) as session:
-            metric = session.query(AnalyzeRequestMetrics).filter_by(id=id_).first()
+        query = select(AnalyzeRequestMetrics).filter(AnalyzeRequestMetrics.id == id_)
+        async with transaction(commit=True) as session:
+            query_result = await session.execute(query)
+            metric = query_result.scalars().first()
             if metric is None:
                 raise ValueError("Returned `AnalyzeRequestMetrics` table is empty.")
             return metric
 
-    def add_mr_job(
+    async def add_mr_job(
         self,
         forge: Forge,
         project_id: int,
@@ -161,13 +178,15 @@ class AnalyzeRequestMetrics(Base):
           mr_iid: merge request forge iid
           job_id: forge job id
         """
-        mr_job = GitlabMergeRequestJobs.get_or_create(forge, project_id, mr_iid, job_id)
+        mr_job = await GitlabMergeRequestJobs.get_or_create(
+            forge, project_id, mr_iid, job_id
+        )
         self.merge_request_job_id = mr_job.id
-        with transaction(commit=True) as session:
-            session.merge(self)
+        async with transaction(commit=True) as session:
+            await session.merge(self)
 
     @classmethod
-    def get_requests_metrics_for_mr_job(
+    async def get_requests_metrics_for_mr_job(
         cls,
         forge: Forge,
         project_id: int,
@@ -182,19 +201,20 @@ class AnalyzeRequestMetrics(Base):
           mr_iid: merge request forge iid
           job_id: forge job id
         """
-        with transaction(commit=False) as session:
-            mr_job_alias = aliased(GitlabMergeRequestJobs)
-            metrics = (
-                session.query(cls)
-                .join(mr_job_alias, cls.merge_request_job_id == mr_job_alias.id)
-                .filter(
-                    mr_job_alias.forge == forge,
-                    mr_job_alias.mr_iid == mr_iid,
-                    mr_job_alias.project_id == project_id,
-                    mr_job_alias.job_id == job_id,
-                )
-                .all()
+        mr_job_alias = aliased(GitlabMergeRequestJobs)
+        query = (
+            select(cls)
+            .join(mr_job_alias, cls.merge_request_job_id == mr_job_alias.id)
+            .filter(
+                mr_job_alias.forge == forge,
+                mr_job_alias.mr_iid == mr_iid,
+                mr_job_alias.project_id == project_id,
+                mr_job_alias.job_id == job_id,
             )
+        )
+        async with transaction(commit=False) as session:
+            query_result = await session.execute(query)
+            metrics = query_result.scalars().all()
             return metrics
 
     @classmethod
@@ -242,7 +262,7 @@ class AnalyzeRequestMetrics(Base):
         return requests_by_time_format
 
     @classmethod
-    def get_requests_in_period(
+    async def get_requests_in_period(
         cls,
         start_time: datetime.datetime,
         end_time: datetime.datetime,
@@ -261,7 +281,7 @@ class AnalyzeRequestMetrics(Base):
         Returns:
             dict[datetime, int]: A dictionary mapping datetime objects to request counts
         """
-        with transaction(commit=False) as session:
+        async with transaction(commit=False) as session:
             requests_by_time_format = cls._get_requests_by_time_for_postgres(
                 start_time, end_time, time_format, endpoint
             )
@@ -271,13 +291,13 @@ class AnalyzeRequestMetrics(Base):
                 func.count(distinct(requests_by_time_format.c.id)),  # pylint: disable=not-callable
             ).group_by("time_format")
 
-            counts = session.execute(count_requests_by_time_format)
-            results = counts.fetchall()
+            query_results = await session.execute(count_requests_by_time_format)
+            results = query_results.all()
 
             return cls.get_dictionary_with_datetime_keys(time_format, results)
 
     @classmethod
-    def _get_average_responses_times_for_postgres(
+    async def _get_average_responses_times_for_postgres(
         cls, start_time, end_time, time_format, endpoint
     ):
         """Get average responses time.
@@ -285,7 +305,7 @@ class AnalyzeRequestMetrics(Base):
         func.to_char is PostgreSQL specific.
         Let's unit tests replace this function with the SQLite version.
         """
-        with transaction(commit=False) as session:
+        async with transaction(commit=False) as session:
             pgsql_time_format = cls.get_postgres_time_format(time_format)
 
             average_responses_times = (
@@ -307,11 +327,12 @@ class AnalyzeRequestMetrics(Base):
                 .order_by("time_range")
             )
 
-            results = session.execute(average_responses_times).fetchall()
+            query_results = await session.execute(average_responses_times)
+            results = query_results.all()
             return results
 
     @classmethod
-    def get_responses_average_time_in_period(
+    async def get_responses_average_time_in_period(
         cls,
         start_time: datetime.datetime,
         end_time: datetime.datetime,
@@ -332,9 +353,11 @@ class AnalyzeRequestMetrics(Base):
             dict[datetime, int]: A dictionary mapping datetime objects
             to average responses times
         """
-        with transaction(commit=False) as _:
-            average_responses_times = cls._get_average_responses_times_for_postgres(
-                start_time, end_time, time_format, endpoint
+        async with transaction(commit=False) as _:
+            average_responses_times = (
+                await cls._get_average_responses_times_for_postgres(
+                    start_time, end_time, time_format, endpoint
+                )
             )
 
             return cls.get_dictionary_with_datetime_keys(
@@ -342,7 +365,7 @@ class AnalyzeRequestMetrics(Base):
             )
 
     @classmethod
-    def _get_average_responses_lengths_for_postgres(
+    async def _get_average_responses_lengths_for_postgres(
         cls, start_time, end_time, time_format, endpoint
     ):
         """Get average responses length.
@@ -350,7 +373,7 @@ class AnalyzeRequestMetrics(Base):
         func.to_char is PostgreSQL specific.
         Let's unit tests replace this function with the SQLite version.
         """
-        with transaction(commit=False) as session:
+        async with transaction(commit=False) as session:
             pgsql_time_format = cls.get_postgres_time_format(time_format)
 
             average_responses_lengths = (
@@ -366,11 +389,12 @@ class AnalyzeRequestMetrics(Base):
                 .order_by("time_range")
             )
 
-            results = session.execute(average_responses_lengths).fetchall()
+            query_results = await session.execute(average_responses_lengths)
+            results = query_results.all()
             return results
 
     @classmethod
-    def get_responses_average_length_in_period(
+    async def get_responses_average_length_in_period(
         cls,
         start_time: datetime.datetime,
         end_time: datetime.datetime,
@@ -391,9 +415,11 @@ class AnalyzeRequestMetrics(Base):
             dict[datetime, int]: A dictionary mapping datetime objects
             to average responses lengths
         """
-        with transaction(commit=False) as _:
-            average_responses_lengths = cls._get_average_responses_lengths_for_postgres(
-                start_time, end_time, time_format, endpoint
+        async with transaction(commit=False) as _:
+            average_responses_lengths = (
+                await cls._get_average_responses_lengths_for_postgres(
+                    start_time, end_time, time_format, endpoint
+                )
             )
 
             return cls.get_dictionary_with_datetime_keys(

logdetective/server/emoji.py

@@ -1,6 +1,6 @@
 import asyncio
 
-from typing import List, Callable
+from typing import List
 from collections import Counter
 
 import gitlab
@@ -20,7 +20,7 @@ async def collect_emojis(gitlab_conn: gitlab.Gitlab, period: TimePeriod):
     Collect emoji feedback from logdetective comments saved in database.
     Check only comments created in the last given period of time.
     """
-    comments = Comments.get_since(period.get_period_start_time()) or []
+    comments = await Comments.get_since(period.get_period_start_time()) or []
     comments_for_gitlab_connection = [
         comment for comment in comments if comment.forge == gitlab_conn.url
     ]
@@ -39,29 +39,14 @@ async def collect_emojis_for_mr(
     except ValueError as ex:
         LOG.exception("Attempt to use unrecognized Forge `%s`", gitlab_conn.url)
         raise ex
-    mr_jobs = GitlabMergeRequestJobs.get_by_mr_iid(url, project_id, mr_iid) or []
-
-    comments = [Comments.get_by_mr_job(mr_job) for mr_job in mr_jobs]
-    await collect_emojis_in_comments(comments, gitlab_conn)
+    mr_jobs = await GitlabMergeRequestJobs.get_by_mr_iid(url, project_id, mr_iid) or []
 
+    comments = [await Comments.get_by_mr_job(mr_job) for mr_job in mr_jobs]
+    # Filter all cases when no comments were found. This shouldn't happen if the database
+    # is in good order. But checking for it can't hurt.
+    comments = [comment for comment in comments if isinstance(comment, Comments)]
 
-async def _handle_gitlab_operation(func: Callable, *args):
-    """
-    It handles errors for the specified GitLab operation.
-    After executing it in a separate thread.
-    """
-    try:
-        return await asyncio.to_thread(func, *args)
-    except (gitlab.GitlabError, gitlab.GitlabGetError) as e:
-        log_msg = f"Error during GitLab operation {func}{args}: {e}"
-        if "Not Found" in str(e):
-            LOG.error(log_msg)
-        else:
-            LOG.exception(log_msg)
-    except Exception as e:  # pylint: disable=broad-exception-caught
-        LOG.exception(
-            "Unexpected error during GitLab operation %s(%s): %s", func, args, e
-        )
+    await collect_emojis_in_comments(comments, gitlab_conn)
 
 
 async def collect_emojis_in_comments(  # pylint: disable=too-many-locals
@@ -73,40 +58,57 @@ async def collect_emojis_in_comments(  # pylint: disable=too-many-locals
     projects = {}
     merge_requests = {}
     for comment in comments:
-        mr_job_db = GitlabMergeRequestJobs.get_by_id(comment.merge_request_job_id)
+        mr_job_db = await GitlabMergeRequestJobs.get_by_id(comment.merge_request_job_id)
         if not mr_job_db:
             continue
-        if mr_job_db.id not in projects:
-            project = await _handle_gitlab_operation(
-                gitlab_conn.projects.get, mr_job_db.project_id
+        try:
+            if mr_job_db.id not in projects:
+                project = await asyncio.to_thread(
+                    gitlab_conn.projects.get, mr_job_db.project_id
+                )
+
+                projects[mr_job_db.id] = project
+            else:
+                project = projects[mr_job_db.id]
+            merge_request_iid = mr_job_db.mr_iid
+            if merge_request_iid not in merge_requests:
+                merge_request = await asyncio.to_thread(
+                    project.mergerequests.get, merge_request_iid
+                )
+
+                merge_requests[merge_request_iid] = merge_request
+            else:
+                merge_request = merge_requests[merge_request_iid]
+
+            discussion = await asyncio.to_thread(
+                merge_request.discussions.get, comment.comment_id
             )
-            if not project:
-                continue
-            projects[mr_job_db.id] = project
-        else:
-            project = projects[mr_job_db.id]
-        merge_request_iid = mr_job_db.mr_iid
-        if merge_request_iid not in merge_requests:
-            merge_request = await _handle_gitlab_operation(
-                project.mergerequests.get, merge_request_iid
-            )
-            if not merge_request:
-                continue
-            merge_requests[merge_request_iid] = merge_request
-        else:
-            merge_request = merge_requests[merge_request_iid]
 
-        discussion = await _handle_gitlab_operation(
-            merge_request.discussions.get, comment.comment_id
-        )
-        if not discussion:
-            continue
+            # Get the ID of the first note
+            if "notes" not in discussion.attributes or len(discussion.attributes["notes"]) == 0:
+                LOG.warning(
+                    "No notes were found in comment %s in merge request %d",
+                    comment.comment_id,
+                    merge_request_iid,
+                )
+                continue
 
-        # Get the ID of the first note
-        note_id = discussion.attributes["notes"][0]["id"]
-        note = await _handle_gitlab_operation(merge_request.notes.get, note_id)
-        if not note:
-            continue
+            note_id = discussion.attributes["notes"][0]["id"]
+            note = await asyncio.to_thread(merge_request.notes.get, note_id)
+
+        # Log warning with full stack trace, in case we can't find the right
+        # discussion, merge request or project.
+        # All of these objects can be lost, and we shouldn't treat as an error.
+        # Other exceptions are raised.
+        except gitlab.GitlabError as e:
+            if e.response_code == 404:
+                LOG.warning(
+                    "Couldn't retrieve emoji counts for comment %s due to GitlabError",
+                    comment.comment_id, exc_info=True)
+                continue
+            LOG.error("Error encountered while processing emoji counts for GitLab comment %s",
+                      comment.comment_id, exc_info=True)
+            raise
 
         emoji_counts = Counter(emoji.name for emoji in note.awardemojis.list())
 
@@ -114,7 +116,7 @@ async def collect_emojis_in_comments(  # pylint: disable=too-many-locals
         # because we need to remove them
         old_emojis = [
             reaction.reaction_type
-            for reaction in Reactions.get_all_reactions(
+            for reaction in await Reactions.get_all_reactions(
                 comment.forge,
                 mr_job_db.project_id,
                 mr_job_db.mr_iid,
@@ -123,7 +125,7 @@ async def collect_emojis_in_comments(  # pylint: disable=too-many-locals
             )
         ]
         for key, value in emoji_counts.items():
-            Reactions.create_or_update(
+            await Reactions.create_or_update(
                 comment.forge,
                 mr_job_db.project_id,
                 mr_job_db.mr_iid,
@@ -136,7 +138,7 @@ async def collect_emojis_in_comments(  # pylint: disable=too-many-locals
                 old_emojis.remove(key)
 
         # not updated reactions has been removed, drop them
-        Reactions.delete(
+        await Reactions.delete(
             comment.forge,
             mr_job_db.project_id,
             mr_job_db.mr_iid,

logdetective/server/exceptions.py

@@ -31,3 +31,7 @@ class LogDetectiveConnectionError(LogDetectiveKojiException):
 
 class LogsTooLargeError(LogDetectiveKojiException):
     """The log archive exceeds the configured maximum size"""
+
+
+class LogDetectiveMetricsError(LogDetectiveException):
+    """Exception was encountered while recording metrics"""

logdetective/server/gitlab.py

@@ -11,9 +11,13 @@ import gitlab.v4
 import gitlab.v4.objects
 import jinja2
 import aiohttp
+import backoff
 
 from logdetective.server.config import SERVER_CONFIG, LOG
-from logdetective.server.exceptions import LogsTooLargeError
+from logdetective.server.exceptions import (
+    LogsTooLargeError,
+    LogDetectiveConnectionError,
+)
 from logdetective.server.llm import perform_staged_analysis
 from logdetective.server.metric import add_new_metrics, update_metrics
 from logdetective.server.models import (
@@ -29,6 +33,7 @@ from logdetective.server.database.models import (
     GitlabMergeRequestJobs,
 )
 from logdetective.server.compressors import RemoteLogCompressor
+from logdetective.server.utils import connection_error_giveup
 
 MR_REGEX = re.compile(r"refs/merge-requests/(\d+)/.*$")
 FAILURE_LOG_REGEX = re.compile(r"(\w*\.log)")
@@ -75,7 +80,7 @@ async def process_gitlab_job_event(
     # Check if this is a resubmission of an existing, completed job.
     # If it is, we'll exit out here and not waste time retrieving the logs,
     # running a new analysis or trying to submit a new comment.
-    mr_job_db = GitlabMergeRequestJobs.get_by_details(
+    mr_job_db = await GitlabMergeRequestJobs.get_by_details(
         forge=forge,
         project_id=project.id,
         mr_iid=merge_request_iid,
@@ -91,8 +96,8 @@ async def process_gitlab_job_event(
         log_url, preprocessed_log = await retrieve_and_preprocess_koji_logs(
             gitlab_cfg, job
         )
-    except LogsTooLargeError:
-        LOG.error("Could not retrieve logs. Too large.")
+    except (LogsTooLargeError, LogDetectiveConnectionError) as ex:
+        LOG.error("Could not retrieve logs due to %s", ex)
         raise
 
     # Submit log to Log Detective and await the results.
@@ -104,7 +109,7 @@ async def process_gitlab_job_event(
         compressed_log_content=RemoteLogCompressor.zip_text(log_text),
     )
     staged_response = await perform_staged_analysis(log_text=log_text)
-    update_metrics(metrics_id, staged_response)
+    await update_metrics(metrics_id, staged_response)
     preprocessed_log.close()
 
     # check if this project is on the opt-in list for posting comments.
@@ -151,6 +156,9 @@ def is_eligible_package(project_name: str):
     return True
 
 
+@backoff.on_exception(
+    backoff.expo, ConnectionResetError, max_time=60, on_giveup=connection_error_giveup
+)
 async def retrieve_and_preprocess_koji_logs(
     gitlab_cfg: GitLabInstanceConfig,
     job: gitlab.v4.objects.ProjectJob,
@@ -256,8 +264,7 @@ async def retrieve_and_preprocess_koji_logs(
     LOG.debug("Failed architecture: %s", failed_arch)
 
     log_path = failed_arches[failed_arch].as_posix()
-
-    log_url = f"{gitlab_cfg.api_path}/projects/{job.project_id}/jobs/{job.id}/artifacts/{log_path}"  # pylint: disable=line-too-long
+    log_url = f"{gitlab_cfg.url}/{gitlab_cfg.api_path}/projects/{job.project_id}/jobs/{job.id}/artifacts/{log_path}"  # pylint: disable=line-too-long
     LOG.debug("Returning contents of %s%s", gitlab_cfg.url, log_url)
 
     # Return the log as a file-like object with .read() function
@@ -350,13 +357,13 @@ async def comment_on_mr(  # pylint: disable=too-many-arguments disable=too-many-
     await asyncio.to_thread(note.save)
 
     # Save the new comment to the database
-    metrics = AnalyzeRequestMetrics.get_metric_by_id(metrics_id)
-    Comments.create(
+    metrics = await AnalyzeRequestMetrics.get_metric_by_id(metrics_id)
+    await Comments.create(
         forge,
         project.id,
         merge_request_iid,
         job.id,
-        discussion.id,
+        str(discussion.id),
         metrics,
     )
 
@@ -371,7 +378,7 @@ async def suppress_latest_comment(
     superseded by a new push."""
 
     # Ask the database for the last known comment for this MR
-    previous_comment = Comments.get_latest_comment(
+    previous_comment = await Comments.get_latest_comment(
         gitlab_instance, project.id, merge_request_iid
     )