logdetective 2.4.1__py3-none-any.whl → 2.5.0__py3-none-any.whl

logdetective/server/database/models/metrics.py

@@ -50,7 +50,7 @@ class AnalyzeRequestMetrics(Base):
         comment="The service endpoint that was called",
     )
     request_received_at = Column(
-        DateTime,
+        DateTime(timezone=True),
         nullable=False,
         index=True,
         default=datetime.datetime.now(datetime.timezone.utc),
@@ -69,7 +69,9 @@ class AnalyzeRequestMetrics(Base):
         comment="Given response (with explanation and snippets) saved in a zip format",
     )
     response_sent_at = Column(
-        DateTime, nullable=True, comment="Timestamp when the response was sent back"
+        DateTime(timezone=True),
+        nullable=True,
+        comment="Timestamp when the response was sent back",
     )
     response_length = Column(
         Integer, nullable=True, comment="Length of the response in chars"
@@ -90,7 +92,7 @@ class AnalyzeRequestMetrics(Base):
 
     @classmethod
     @backoff.on_exception(backoff.expo, OperationalError, max_tries=DB_MAX_RETRIES)
-    def create(
+    async def create(
         cls,
         endpoint: EndpointType,
         compressed_log: io.BytesIO,
@@ -98,7 +100,7 @@ class AnalyzeRequestMetrics(Base):
     ) -> int:
         """Create AnalyzeRequestMetrics new line
         with data related to a received request"""
-        with transaction(commit=True) as session:
+        async with transaction(commit=True) as session:
             metrics = AnalyzeRequestMetrics()
             metrics.endpoint = endpoint
             metrics.compressed_log = compressed_log
@@ -106,12 +108,12 @@ class AnalyzeRequestMetrics(Base):
                 datetime.timezone.utc
             )
             session.add(metrics)
-            session.flush()
+            await session.flush()
             return metrics.id
 
     @classmethod
     @backoff.on_exception(backoff.expo, OperationalError, max_tries=DB_MAX_RETRIES)
-    def update(  # pylint: disable=too-many-arguments disable=too-many-positional-arguments
+    async def update(  # pylint: disable=too-many-arguments disable=too-many-positional-arguments
         cls,
         id_: int,
         response_sent_at: DateTime,
@@ -121,8 +123,10 @@ class AnalyzeRequestMetrics(Base):
     ) -> None:
         """Update a row
         with data related to the given response"""
-        with transaction(commit=True) as session:
-            metrics = session.query(AnalyzeRequestMetrics).filter_by(id=id_).first()
+        query = select(AnalyzeRequestMetrics).filter(AnalyzeRequestMetrics.id == id_)
+        async with transaction(commit=True) as session:
+            query_result = await session.execute(query)
+            metrics = query_result.scalars().first()
             if metrics is None:
                 raise ValueError("Returned `AnalyzeRequestMetrics` table is empty.")
             metrics.response_sent_at = response_sent_at
@@ -133,19 +137,21 @@ class AnalyzeRequestMetrics(Base):
 
     @classmethod
     @backoff.on_exception(backoff.expo, OperationalError, max_tries=DB_MAX_RETRIES)
-    def get_metric_by_id(
+    async def get_metric_by_id(
         cls,
         id_: int,
     ) -> Self:
         """Update a row
         with data related to the given response"""
-        with transaction(commit=True) as session:
-            metric = session.query(AnalyzeRequestMetrics).filter_by(id=id_).first()
+        query = select(AnalyzeRequestMetrics).filter(AnalyzeRequestMetrics.id == id_)
+        async with transaction(commit=True) as session:
+            query_result = await session.execute(query)
+            metric = query_result.scalars().first()
             if metric is None:
                 raise ValueError("Returned `AnalyzeRequestMetrics` table is empty.")
             return metric
 
-    def add_mr_job(
+    async def add_mr_job(
         self,
         forge: Forge,
         project_id: int,
@@ -161,13 +167,15 @@ class AnalyzeRequestMetrics(Base):
           mr_iid: merge request forge iid
           job_id: forge job id
         """
-        mr_job = GitlabMergeRequestJobs.get_or_create(forge, project_id, mr_iid, job_id)
+        mr_job = await GitlabMergeRequestJobs.get_or_create(
+            forge, project_id, mr_iid, job_id
+        )
         self.merge_request_job_id = mr_job.id
-        with transaction(commit=True) as session:
-            session.merge(self)
+        async with transaction(commit=True) as session:
+            await session.merge(self)
 
     @classmethod
-    def get_requests_metrics_for_mr_job(
+    async def get_requests_metrics_for_mr_job(
         cls,
         forge: Forge,
         project_id: int,
@@ -182,19 +190,20 @@ class AnalyzeRequestMetrics(Base):
           mr_iid: merge request forge iid
           job_id: forge job id
         """
-        with transaction(commit=False) as session:
-            mr_job_alias = aliased(GitlabMergeRequestJobs)
-            metrics = (
-                session.query(cls)
-                .join(mr_job_alias, cls.merge_request_job_id == mr_job_alias.id)
-                .filter(
-                    mr_job_alias.forge == forge,
-                    mr_job_alias.mr_iid == mr_iid,
-                    mr_job_alias.project_id == project_id,
-                    mr_job_alias.job_id == job_id,
-                )
-                .all()
+        mr_job_alias = aliased(GitlabMergeRequestJobs)
+        query = (
+            select(cls)
+            .join(mr_job_alias, cls.merge_request_job_id == mr_job_alias.id)
+            .filter(
+                mr_job_alias.forge == forge,
+                mr_job_alias.mr_iid == mr_iid,
+                mr_job_alias.project_id == project_id,
+                mr_job_alias.job_id == job_id,
             )
+        )
+        async with transaction(commit=False) as session:
+            query_result = await session.execute(query)
+            metrics = query_result.scalars().all()
             return metrics
 
     @classmethod
@@ -242,7 +251,7 @@ class AnalyzeRequestMetrics(Base):
         return requests_by_time_format
 
     @classmethod
-    def get_requests_in_period(
+    async def get_requests_in_period(
         cls,
         start_time: datetime.datetime,
         end_time: datetime.datetime,
@@ -261,7 +270,7 @@ class AnalyzeRequestMetrics(Base):
         Returns:
             dict[datetime, int]: A dictionary mapping datetime objects to request counts
         """
-        with transaction(commit=False) as session:
+        async with transaction(commit=False) as session:
             requests_by_time_format = cls._get_requests_by_time_for_postgres(
                 start_time, end_time, time_format, endpoint
             )
@@ -271,13 +280,13 @@ class AnalyzeRequestMetrics(Base):
                 func.count(distinct(requests_by_time_format.c.id)),  # pylint: disable=not-callable
             ).group_by("time_format")
 
-            counts = session.execute(count_requests_by_time_format)
-            results = counts.fetchall()
+            query_results = await session.execute(count_requests_by_time_format)
+            results = query_results.all()
 
             return cls.get_dictionary_with_datetime_keys(time_format, results)
 
     @classmethod
-    def _get_average_responses_times_for_postgres(
+    async def _get_average_responses_times_for_postgres(
         cls, start_time, end_time, time_format, endpoint
     ):
         """Get average responses time.
@@ -285,7 +294,7 @@ class AnalyzeRequestMetrics(Base):
         func.to_char is PostgreSQL specific.
         Let's unit tests replace this function with the SQLite version.
         """
-        with transaction(commit=False) as session:
+        async with transaction(commit=False) as session:
             pgsql_time_format = cls.get_postgres_time_format(time_format)
 
             average_responses_times = (
@@ -307,11 +316,12 @@ class AnalyzeRequestMetrics(Base):
                 .order_by("time_range")
             )
 
-            results = session.execute(average_responses_times).fetchall()
+            query_results = await session.execute(average_responses_times)
+            results = query_results.all()
             return results
 
     @classmethod
-    def get_responses_average_time_in_period(
+    async def get_responses_average_time_in_period(
         cls,
         start_time: datetime.datetime,
         end_time: datetime.datetime,
@@ -332,9 +342,11 @@ class AnalyzeRequestMetrics(Base):
             dict[datetime, int]: A dictionary mapping datetime objects
             to average responses times
         """
-        with transaction(commit=False) as _:
-            average_responses_times = cls._get_average_responses_times_for_postgres(
-                start_time, end_time, time_format, endpoint
+        async with transaction(commit=False) as _:
+            average_responses_times = (
+                await cls._get_average_responses_times_for_postgres(
+                    start_time, end_time, time_format, endpoint
+                )
             )
 
             return cls.get_dictionary_with_datetime_keys(
@@ -342,7 +354,7 @@ class AnalyzeRequestMetrics(Base):
             )
 
     @classmethod
-    def _get_average_responses_lengths_for_postgres(
+    async def _get_average_responses_lengths_for_postgres(
         cls, start_time, end_time, time_format, endpoint
     ):
         """Get average responses length.
@@ -350,7 +362,7 @@ class AnalyzeRequestMetrics(Base):
         func.to_char is PostgreSQL specific.
         Let's unit tests replace this function with the SQLite version.
         """
-        with transaction(commit=False) as session:
+        async with transaction(commit=False) as session:
             pgsql_time_format = cls.get_postgres_time_format(time_format)
 
             average_responses_lengths = (
@@ -366,11 +378,12 @@ class AnalyzeRequestMetrics(Base):
                 .order_by("time_range")
             )
 
-            results = session.execute(average_responses_lengths).fetchall()
+            query_results = await session.execute(average_responses_lengths)
+            results = query_results.all()
             return results
 
     @classmethod
-    def get_responses_average_length_in_period(
+    async def get_responses_average_length_in_period(
         cls,
         start_time: datetime.datetime,
         end_time: datetime.datetime,
@@ -391,9 +404,11 @@ class AnalyzeRequestMetrics(Base):
             dict[datetime, int]: A dictionary mapping datetime objects
             to average responses lengths
         """
-        with transaction(commit=False) as _:
-            average_responses_lengths = cls._get_average_responses_lengths_for_postgres(
-                start_time, end_time, time_format, endpoint
+        async with transaction(commit=False) as _:
+            average_responses_lengths = (
+                await cls._get_average_responses_lengths_for_postgres(
+                    start_time, end_time, time_format, endpoint
+                )
             )
 
             return cls.get_dictionary_with_datetime_keys(

logdetective/server/emoji.py

@@ -20,7 +20,7 @@ async def collect_emojis(gitlab_conn: gitlab.Gitlab, period: TimePeriod):
     Collect emoji feedback from logdetective comments saved in database.
     Check only comments created in the last given period of time.
     """
-    comments = Comments.get_since(period.get_period_start_time()) or []
+    comments = await Comments.get_since(period.get_period_start_time()) or []
     comments_for_gitlab_connection = [
         comment for comment in comments if comment.forge == gitlab_conn.url
     ]
@@ -39,9 +39,9 @@ async def collect_emojis_for_mr(
     except ValueError as ex:
         LOG.exception("Attempt to use unrecognized Forge `%s`", gitlab_conn.url)
         raise ex
-    mr_jobs = GitlabMergeRequestJobs.get_by_mr_iid(url, project_id, mr_iid) or []
+    mr_jobs = await GitlabMergeRequestJobs.get_by_mr_iid(url, project_id, mr_iid) or []
 
-    comments = [Comments.get_by_mr_job(mr_job) for mr_job in mr_jobs]
+    comments = [await Comments.get_by_mr_job(mr_job) for mr_job in mr_jobs]
     await collect_emojis_in_comments(comments, gitlab_conn)
 
 
@@ -73,7 +73,7 @@ async def collect_emojis_in_comments(  # pylint: disable=too-many-locals
     projects = {}
     merge_requests = {}
     for comment in comments:
-        mr_job_db = GitlabMergeRequestJobs.get_by_id(comment.merge_request_job_id)
+        mr_job_db = await GitlabMergeRequestJobs.get_by_id(comment.merge_request_job_id)
         if not mr_job_db:
             continue
         if mr_job_db.id not in projects:
@@ -114,7 +114,7 @@ async def collect_emojis_in_comments(  # pylint: disable=too-many-locals
         # because we need to remove them
         old_emojis = [
             reaction.reaction_type
-            for reaction in Reactions.get_all_reactions(
+            for reaction in await Reactions.get_all_reactions(
                 comment.forge,
                 mr_job_db.project_id,
                 mr_job_db.mr_iid,
@@ -123,7 +123,7 @@ async def collect_emojis_in_comments(  # pylint: disable=too-many-locals
             )
         ]
         for key, value in emoji_counts.items():
-            Reactions.create_or_update(
+            await Reactions.create_or_update(
                 comment.forge,
                 mr_job_db.project_id,
                 mr_job_db.mr_iid,
@@ -136,7 +136,7 @@ async def collect_emojis_in_comments(  # pylint: disable=too-many-locals
                 old_emojis.remove(key)
 
         # not updated reactions has been removed, drop them
-        Reactions.delete(
+        await Reactions.delete(
             comment.forge,
             mr_job_db.project_id,
             mr_job_db.mr_iid,

logdetective/server/gitlab.py

@@ -80,7 +80,7 @@ async def process_gitlab_job_event(
     # Check if this is a resubmission of an existing, completed job.
     # If it is, we'll exit out here and not waste time retrieving the logs,
     # running a new analysis or trying to submit a new comment.
-    mr_job_db = GitlabMergeRequestJobs.get_by_details(
+    mr_job_db = await GitlabMergeRequestJobs.get_by_details(
         forge=forge,
         project_id=project.id,
         mr_iid=merge_request_iid,
@@ -109,7 +109,7 @@ async def process_gitlab_job_event(
         compressed_log_content=RemoteLogCompressor.zip_text(log_text),
     )
     staged_response = await perform_staged_analysis(log_text=log_text)
-    update_metrics(metrics_id, staged_response)
+    await update_metrics(metrics_id, staged_response)
     preprocessed_log.close()
 
     # check if this project is on the opt-in list for posting comments.
@@ -357,13 +357,13 @@ async def comment_on_mr(  # pylint: disable=too-many-arguments disable=too-many-
     await asyncio.to_thread(note.save)
 
     # Save the new comment to the database
-    metrics = AnalyzeRequestMetrics.get_metric_by_id(metrics_id)
-    Comments.create(
+    metrics = await AnalyzeRequestMetrics.get_metric_by_id(metrics_id)
+    await Comments.create(
         forge,
         project.id,
         merge_request_iid,
         job.id,
-        discussion.id,
+        str(discussion.id),
         metrics,
     )
 
@@ -378,7 +378,7 @@ async def suppress_latest_comment(
     superseded by a new push."""
 
     # Ask the database for the last known comment for this MR
-    previous_comment = Comments.get_latest_comment(
+    previous_comment = await Comments.get_latest_comment(
         gitlab_instance, project.id, merge_request_iid
     )
 

logdetective/server/metric.py

@@ -32,7 +32,7 @@ async def add_new_metrics(
         remote_log = RemoteLog(url, http_session)
         compressed_log_content = await RemoteLogCompressor(remote_log).zip_content()
 
-    return AnalyzeRequestMetrics.create(
+    return await AnalyzeRequestMetrics.create(
         endpoint=EndpointType(api_name),
         compressed_log=compressed_log_content,
         request_received_at=received_at
@@ -41,7 +41,7 @@ async def add_new_metrics(
     )
 
 
-def update_metrics(
+async def update_metrics(
     metrics_id: int,
     response: Union[models.Response, models.StagedResponse, StreamingResponse],
     sent_at: Optional[datetime.datetime] = None,
@@ -73,12 +73,12 @@ def update_metrics(
     response_certainty = (
         response.response_certainty if hasattr(response, "response_certainty") else None
     )
-    AnalyzeRequestMetrics.update(
-        metrics_id,
-        response_sent_at,
-        response_length,
-        response_certainty,
-        compressed_response,
+    await AnalyzeRequestMetrics.update(
+        id_=metrics_id,
+        response_sent_at=response_sent_at,
+        response_length=response_length,
+        response_certainty=response_certainty,
+        compressed_response=compressed_response,
     )
 
 
@@ -112,7 +112,7 @@ def track_request(name=None):
                 name if name else f.__name__, log_url, kwargs["http_session"]
             )
             response = await f(*args, **kwargs)
-            update_metrics(metrics_id, response)
+            await update_metrics(metrics_id, response)
             return response
 
         if inspect.iscoroutinefunction(f):

logdetective/server/plot.py

@@ -163,7 +163,7 @@ def _add_line_chart(  # pylint: disable=too-many-arguments disable=too-many-posi
         ax.tick_params(axis="y", labelcolor=color)
 
 
-def requests_per_time(
+async def requests_per_time(
     period_of_time: TimePeriod,
     endpoint: EndpointType = EndpointType.ANALYZE,
     end_time: Optional[datetime.datetime] = None,
@@ -191,7 +191,7 @@ def requests_per_time(
     end_time = end_time or datetime.datetime.now(datetime.timezone.utc)
     start_time = period_of_time.get_period_start_time(end_time)
     plot_def = Definition(period_of_time)
-    requests_counts = AnalyzeRequestMetrics.get_requests_in_period(
+    requests_counts = await AnalyzeRequestMetrics.get_requests_in_period(
         start_time, end_time, plot_def.time_format, endpoint
     )
     timestamps, counts = create_time_series_arrays(
@@ -218,7 +218,7 @@ def requests_per_time(
     return fig
 
 
-def average_time_per_responses(  # pylint: disable=too-many-locals
+async def average_time_per_responses(  # pylint: disable=too-many-locals
     period_of_time: TimePeriod,
     endpoint: EndpointType = EndpointType.ANALYZE,
     end_time: Optional[datetime.datetime] = None,
@@ -246,8 +246,10 @@ def average_time_per_responses(  # pylint: disable=too-many-locals
     end_time = end_time or datetime.datetime.now(datetime.timezone.utc)
     start_time = period_of_time.get_period_start_time(end_time)
     plot_def = Definition(period_of_time)
-    responses_average_time = AnalyzeRequestMetrics.get_responses_average_time_in_period(
-        start_time, end_time, plot_def.time_format, endpoint
+    responses_average_time = (
+        await AnalyzeRequestMetrics.get_responses_average_time_in_period(
+            start_time, end_time, plot_def.time_format, endpoint
+        )
     )
     timestamps, average_time = create_time_series_arrays(
         responses_average_time,
@@ -263,7 +265,7 @@ def average_time_per_responses(  # pylint: disable=too-many-locals
     )
 
     responses_average_length = (
-        AnalyzeRequestMetrics.get_responses_average_length_in_period(
+        await AnalyzeRequestMetrics.get_responses_average_length_in_period(
             start_time, end_time, plot_def.time_format, endpoint
         )
     )
@@ -292,7 +294,7 @@ def average_time_per_responses(  # pylint: disable=too-many-locals
     return fig
 
 
-def _collect_emoji_data(
+async def _collect_emoji_data(
     start_time: datetime.datetime, plot_def: Definition
 ) -> Dict[str, Dict[datetime.datetime, int]]:
     """Collect and organize emoji feedback data
@@ -300,7 +302,7 @@ def _collect_emoji_data(
     Counts all emojis given to logdetective comments created since start_time.
     Collect counts in time accordingly to the plot definition.
     """
-    reactions = Reactions.get_since(start_time)
+    reactions = await Reactions.get_since(start_time)
     reactions_values_dict: Dict[str, Dict] = {}
     for comment_created_at, reaction in reactions:
         comment_created_at_formatted = comment_created_at.strptime(
@@ -369,7 +371,7 @@ def _plot_emoji_data(  # pylint: disable=too-many-locals
     return emoji_lines, emoji_labels
 
 
-def emojis_per_time(
+async def emojis_per_time(
     period_of_time: TimePeriod,
     end_time: Optional[datetime.datetime] = None,
 ) -> figure.Figure:
@@ -395,7 +397,7 @@ def emojis_per_time(
     plot_def = Definition(period_of_time)
     end_time = end_time or datetime.datetime.now(datetime.timezone.utc)
     start_time = period_of_time.get_period_start_time(end_time)
-    reactions_values_dict = _collect_emoji_data(start_time, plot_def)
+    reactions_values_dict = await _collect_emoji_data(start_time, plot_def)
 
     fig, ax = pyplot.subplots(figsize=(12, 6))
 

logdetective/server/server.py

@@ -67,6 +67,7 @@ from logdetective.server.emoji import (
     collect_emojis_for_mr,
 )
 from logdetective.server.compressors import RemoteLogCompressor
+from logdetective.server.utils import get_version
 
 
 LOG_SOURCE_REQUEST_TIMEOUT = os.environ.get("LOG_SOURCE_REQUEST_TIMEOUT", 60)
@@ -89,7 +90,7 @@ async def lifespan(fapp: FastAPI):
     )
 
     # Ensure that the database is initialized.
-    logdetective.server.database.base.init()
+    await logdetective.server.database.base.init()
 
     # Start the background task scheduler for collecting emojis
     asyncio.create_task(schedule_collect_emojis_task())
@@ -306,11 +307,10 @@ async def analyze_koji_task(task_id: int, koji_instance_config: KojiInstanceConf
         received_at=datetime.datetime.now(datetime.timezone.utc),
         compressed_log_content=RemoteLogCompressor.zip_text(log_text),
     )
-
     # We need to associate the metric ID with the koji task analysis.
     # This will create the new row without a response, which we will use as
     # an indicator that the analysis is in progress.
-    KojiTaskAnalysis.create_or_restart(
+    await KojiTaskAnalysis.create_or_restart(
         koji_instance=koji_instance_config.xmlrpc_url,
         task_id=task_id,
         log_file_name=log_file_name,
@@ -319,8 +319,8 @@ async def analyze_koji_task(task_id: int, koji_instance_config: KojiInstanceConf
 
     # Now that we have the response, we can update the metrics and mark the
     # koji task analysis as completed.
-    update_metrics(metrics_id, response)
-    KojiTaskAnalysis.add_response(task_id, metrics_id)
+    await update_metrics(metrics_id, response)
+    await KojiTaskAnalysis.add_response(task_id, metrics_id)
 
     # Notify any callbacks that the analysis is complete.
     for callback in koji_instance_config.get_callbacks(task_id):
@@ -357,6 +357,12 @@ async def async_log(msg):
     return msg
 
 
+@app.get("/version")
+async def get_version_wrapper():
+    """Get the version of logdetective"""
+    return get_version()
+
+
 @app.post("/analyze/stream", response_class=StreamingResponse)
 @track_request()
 async def analyze_log_stream(
@@ -616,22 +622,24 @@ async def get_metrics(
     async def handler():
         """Show statistics for the specified endpoint and plot."""
         if plot == Plot.REQUESTS:
-            fig = plot_engine.requests_per_time(period_since_now, endpoint_type)
+            fig = await plot_engine.requests_per_time(period_since_now, endpoint_type)
             return _svg_figure_response(fig)
         if plot == Plot.RESPONSES:
-            fig = plot_engine.average_time_per_responses(
+            fig = await plot_engine.average_time_per_responses(
                 period_since_now, endpoint_type
             )
             return _svg_figure_response(fig)
         if plot == Plot.EMOJIS:
-            fig = plot_engine.emojis_per_time(period_since_now)
+            fig = await plot_engine.emojis_per_time(period_since_now)
             return _svg_figure_response(fig)
         # BOTH
-        fig_requests = plot_engine.requests_per_time(period_since_now, endpoint_type)
-        fig_responses = plot_engine.average_time_per_responses(
+        fig_requests = await plot_engine.requests_per_time(
+            period_since_now, endpoint_type
+        )
+        fig_responses = await plot_engine.average_time_per_responses(
             period_since_now, endpoint_type
         )
-        fig_emojis = plot_engine.emojis_per_time(period_since_now)
+        fig_emojis = await plot_engine.emojis_per_time(period_since_now)
         return _multiple_svg_figures_response([fig_requests, fig_responses, fig_emojis])
 
     descriptions = {

logdetective/server/utils.py

@@ -1,7 +1,9 @@
 from typing import List
+from importlib.metadata import version
 
 import aiohttp
 from fastapi import HTTPException
+from fastapi.responses import Response as BasicResponse
 
 from logdetective.constants import SNIPPET_DELIMITER
 from logdetective.server.config import LOG
@@ -102,3 +104,8 @@ def construct_final_prompt(formatted_snippets: str, prompt_template: str) -> str
 
     final_prompt = prompt_template.format(formatted_snippets)
     return final_prompt
+
+
+def get_version() -> BasicResponse:
+    """Obtain the version number using importlib"""
+    return BasicResponse(content=version('logdetective'))