logdetective 0.5.11__py3-none-any.whl → 0.9.0__py3-none-any.whl

logdetective/server/server.py

@@ -3,15 +3,18 @@ import json
 import os
 import re
 import zipfile
+from enum import Enum
+from contextlib import asynccontextmanager
 from pathlib import Path, PurePath
 from tempfile import TemporaryFile
-from typing import List, Annotated, Tuple, Dict, Any
+from typing import List, Annotated, Tuple, Dict, Any, Union
 from io import BytesIO
 
-
+import backoff
 import matplotlib
 import matplotlib.pyplot
-from fastapi import FastAPI, HTTPException, BackgroundTasks, Depends, Header
+from aiohttp import StreamReader
+from fastapi import FastAPI, HTTPException, BackgroundTasks, Depends, Header, Request
 
 from fastapi.responses import StreamingResponse
 from fastapi.responses import Response as BasicResponse
@@ -19,11 +22,14 @@ import gitlab
 import gitlab.v4
 import gitlab.v4.objects
 import jinja2
-import requests
+import aiohttp
+import sqlalchemy
+import sentry_sdk
+
+import logdetective.server.database.base
 
 from logdetective.extractors import DrainExtractor
 from logdetective.utils import (
-    validate_url,
     compute_certainty,
     format_snippets,
     load_prompts,
@@ -33,7 +39,7 @@ from logdetective.server.utils import (
     get_log,
     format_analyzed_snippets,
 )
-from logdetective.server.metric import track_request
+from logdetective.server.metric import track_request, add_new_metrics, update_metrics
 from logdetective.server.models import (
     BuildLog,
     JobHook,
@@ -43,8 +49,14 @@ from logdetective.server.models import (
     AnalyzedSnippet,
     TimePeriod,
 )
-from logdetective.server import plot
-from logdetective.server.database.models import EndpointType
+from logdetective.server import plot as plot_engine
+from logdetective.server.remote_log import RemoteLog
+from logdetective.server.database.models import (
+    Comments,
+    EndpointType,
+    Forge,
+)
+from logdetective.server.database.models import AnalyzeRequestMetrics
 
 LLM_CPP_SERVER_TIMEOUT = os.environ.get("LLAMA_CPP_SERVER_TIMEOUT", 600)
 LOG_SOURCE_REQUEST_TIMEOUT = os.environ.get("LOG_SOURCE_REQUEST_TIMEOUT", 60)
@@ -60,6 +72,34 @@ FAILURE_LOG_REGEX = re.compile(r"(\w*\.log)")
 
 LOG = get_log(SERVER_CONFIG)
 
+if sentry_dsn := SERVER_CONFIG.general.sentry_dsn:
+    sentry_sdk.init(dsn=str(sentry_dsn), traces_sample_rate=1.0)
+
+
+@asynccontextmanager
+async def lifespan(fapp: FastAPI):
+    """
+    Establish one HTTP session
+    """
+    fapp.http = aiohttp.ClientSession(
+        timeout=aiohttp.ClientTimeout(
+            total=int(LOG_SOURCE_REQUEST_TIMEOUT), connect=3.07
+        )
+    )
+
+    # Ensure that the database is initialized.
+    logdetective.server.database.base.init()
+
+    yield
+    await fapp.http.close()
+
+
+async def get_http_session(request: Request) -> aiohttp.ClientSession:
+    """
+    Return the single aiohttp ClientSession for this app
+    """
+    return request.app.http
+
 
 def requires_token_when_set(authentication: Annotated[str | None, Header()] = None):
     """
@@ -91,35 +131,12 @@ def requires_token_when_set(authentication: Annotated[str | None, Header()] = No
     raise HTTPException(status_code=401, detail=f"Token {token} not valid.")
 
 
-app = FastAPI(dependencies=[Depends(requires_token_when_set)])
+app = FastAPI(dependencies=[Depends(requires_token_when_set)], lifespan=lifespan)
 app.gitlab_conn = gitlab.Gitlab(
     url=SERVER_CONFIG.gitlab.url, private_token=SERVER_CONFIG.gitlab.api_token
 )
 
 
-def process_url(url: str) -> str:
-    """Validate log URL and return log text."""
-    if validate_url(url=url):
-        try:
-            log_request = requests.get(url, timeout=int(LOG_SOURCE_REQUEST_TIMEOUT))
-        except requests.RequestException as ex:
-            raise HTTPException(
-                status_code=400, detail=f"We couldn't obtain the logs: {ex}"
-            ) from ex
-
-        if not log_request.ok:
-            raise HTTPException(
-                status_code=400,
-                detail="Something went wrong while getting the logs: "
-                f"[{log_request.status_code}] {log_request.text}",
-            )
-    else:
-        LOG.error("Invalid URL received ")
-        raise HTTPException(status_code=400, detail=f"Invalid log URL: {url}")
-
-    return log_request.text
-
-
 def mine_logs(log: str) -> List[Tuple[int, str]]:
     """Extract snippets from log text"""
     extractor = DrainExtractor(
@@ -137,7 +154,11 @@ def mine_logs(log: str) -> List[Tuple[int, str]]:
 
 
 async def submit_to_llm_endpoint(
-    url: str, data: Dict[str, Any], headers: Dict[str, str], stream: bool
+    http: aiohttp.ClientSession,
+    url: str,
+    data: Dict[str, Any],
+    headers: Dict[str, str],
+    stream: bool,
 ) -> Any:
     """Send request to selected API endpoint. Verifying successful request unless
     the using the stream response.
@@ -147,40 +168,62 @@ async def submit_to_llm_endpoint(
     headers:
     stream:
     """
+    LOG.debug("async request %s headers=%s data=%s", url, headers, data)
+    response = await http.post(
+        url,
+        headers=headers,
+        # we need to use the `json=` parameter here and let aiohttp
+        # handle the json-encoding
+        json=data,
+        timeout=int(LLM_CPP_SERVER_TIMEOUT),
+        # Docs says chunked takes int, but:
+        #   DeprecationWarning: Chunk size is deprecated #1615
+        # So let's make sure we either put True or None here
+        chunked=True if stream else None,
+        raise_for_status=True,
+    )
+    if stream:
+        return response
     try:
-        # Expects llama-cpp server to run on LLM_CPP_SERVER_ADDRESS:LLM_CPP_SERVER_PORT
-        response = requests.post(
-            url,
-            headers=headers,
-            data=json.dumps(data),
-            timeout=int(LLM_CPP_SERVER_TIMEOUT),
-            stream=stream,
-        )
-    except requests.RequestException as ex:
-        LOG.error("Llama-cpp query failed: %s", ex)
+        return json.loads(await response.text())
+    except UnicodeDecodeError as ex:
+        LOG.error("Error encountered while parsing llama server response: %s", ex)
         raise HTTPException(
-            status_code=400, detail=f"Llama-cpp query failed: {ex}"
+            status_code=400,
+            detail=f"Couldn't parse the response.\nError: {ex}\nData: {response.text}",
         ) from ex
-    if not stream:
-        if not response.ok:
-            raise HTTPException(
-                status_code=400,
-                detail="Something went wrong while getting a response from the llama server: "
-                f"[{response.status_code}] {response.text}",
-            )
-        try:
-            response = json.loads(response.text)
-        except UnicodeDecodeError as ex:
-            LOG.error("Error encountered while parsing llama server response: %s", ex)
-            raise HTTPException(
-                status_code=400,
-                detail=f"Couldn't parse the response.\nError: {ex}\nData: {response.text}",
-            ) from ex
 
-    return response
+
+def should_we_giveup(exc: aiohttp.ClientResponseError) -> bool:
+    """
+    From backoff's docs:
+
+    > a function which accepts the exception and returns
+    > a truthy value if the exception should not be retried
+    """
+    LOG.info("Should we give up on retrying error %s", exc)
+    return exc.status < 500
+
+
+def we_give_up(details: backoff._typing.Details):
+    """
+    retries didn't work (or we got a different exc)
+    we give up and raise proper 500 for our API endpoint
+    """
+    LOG.error("Inference error: %s", details["args"])
+    raise HTTPException(500, "Request to the inference API failed")
 
 
+@backoff.on_exception(
+    backoff.expo,
+    aiohttp.ClientResponseError,
+    max_tries=3,
+    giveup=should_we_giveup,
+    raise_on_giveup=False,
+    on_giveup=we_give_up,
+)
 async def submit_text(  # pylint: disable=R0913,R0917
+    http: aiohttp.ClientSession,
     text: str,
     max_tokens: int = -1,
     log_probs: int = 1,
@@ -200,14 +243,15 @@ async def submit_text(  # pylint: disable=R0913,R0917
 
     if SERVER_CONFIG.inference.api_endpoint == "/chat/completions":
         return await submit_text_chat_completions(
-            text, headers, max_tokens, log_probs > 0, stream, model
+            http, text, headers, max_tokens, log_probs > 0, stream, model
         )
     return await submit_text_completions(
-        text, headers, max_tokens, log_probs, stream, model
+        http, text, headers, max_tokens, log_probs, stream, model
     )
 
 
 async def submit_text_completions(  # pylint: disable=R0913,R0917
+    http: aiohttp.ClientSession,
     text: str,
     headers: dict,
     max_tokens: int = -1,
@@ -230,6 +274,7 @@ async def submit_text_completions(  # pylint: disable=R0913,R0917
     }
 
     response = await submit_to_llm_endpoint(
+        http,
         f"{SERVER_CONFIG.inference.url}/v1/completions",
         data,
         headers,
@@ -242,13 +287,14 @@ async def submit_text_completions(  # pylint: disable=R0913,R0917
 
 
 async def submit_text_chat_completions(  # pylint: disable=R0913,R0917
+    http: aiohttp.ClientSession,
     text: str,
     headers: dict,
     max_tokens: int = -1,
     log_probs: int = 1,
     stream: bool = False,
     model: str = "default-model",
-) -> Explanation:
+) -> Union[Explanation, StreamReader]:
     """Submit prompt to OpenAI API /chat/completions endpoint.
     max_tokens: number of tokens to be produces, 0 indicates run until encountering EOS
     log_probs: number of token choices to produce log probs for
@@ -270,6 +316,7 @@ async def submit_text_chat_completions(  # pylint: disable=R0913,R0917
     }
 
     response = await submit_to_llm_endpoint(
+        http,
         f"{SERVER_CONFIG.inference.url}/v1/chat/completions",
         data,
         headers,
@@ -277,10 +324,7 @@ async def submit_text_chat_completions(  # pylint: disable=R0913,R0917
     )
 
     if stream:
-        return Explanation(
-            text=response["choices"][0]["delta"]["content"],
-            logprobs=response["choices"][0]["logprobs"]["content"],
-        )
+        return response
     return Explanation(
         text=response["choices"][0]["message"]["content"],
         logprobs=response["choices"][0]["logprobs"]["content"],
@@ -289,17 +333,21 @@ async def submit_text_chat_completions(  # pylint: disable=R0913,R0917
 
 @app.post("/analyze", response_model=Response)
 @track_request()
-async def analyze_log(build_log: BuildLog):
+async def analyze_log(
+    build_log: BuildLog, http_session: aiohttp.ClientSession = Depends(get_http_session)
+):
     """Provide endpoint for log file submission and analysis.
     Request must be in form {"url":"<YOUR_URL_HERE>"}.
     URL must be valid for the request to be passed to the LLM server.
     Meaning that it must contain appropriate scheme, path and netloc,
     while lacking  result, params or query fields.
     """
-    log_text = process_url(build_log.url)
+    remote_log = RemoteLog(build_log.url, http_session)
+    log_text = await remote_log.process_url()
     log_summary = mine_logs(log_text)
     log_summary = format_snippets(log_summary)
     response = await submit_text(
+        http_session,
         PROMPT_CONFIG.prompt_template.format(log_summary),
         model=SERVER_CONFIG.inference.model,
         max_tokens=SERVER_CONFIG.inference.max_tokens,
@@ -319,21 +367,26 @@ async def analyze_log(build_log: BuildLog):
     return Response(explanation=response, response_certainty=certainty)
 
 
-@app.post("/analyze/staged", response_model=StagedResponse)
 @track_request()
-async def analyze_log_staged(build_log: BuildLog):
+@app.post("/analyze/staged", response_model=StagedResponse)
+async def analyze_log_staged(
+    build_log: BuildLog, http_session: aiohttp.ClientSession = Depends(get_http_session)
+):
     """Provide endpoint for log file submission and analysis.
     Request must be in form {"url":"<YOUR_URL_HERE>"}.
     URL must be valid for the request to be passed to the LLM server.
     Meaning that it must contain appropriate scheme, path and netloc,
     while lacking  result, params or query fields.
     """
-    log_text = process_url(build_log.url)
+    remote_log = RemoteLog(build_log.url, http_session)
+    log_text = await remote_log.process_url()
 
-    return await perform_staged_analysis(log_text=log_text)
+    return await perform_staged_analysis(http_session, log_text=log_text)
 
 
-async def perform_staged_analysis(log_text: str) -> StagedResponse:
+async def perform_staged_analysis(
+    http: aiohttp.ClientSession, log_text: str
+) -> StagedResponse:
     """Submit the log file snippets to the LLM and retrieve their results"""
     log_summary = mine_logs(log_text)
 
@@ -341,6 +394,7 @@ async def perform_staged_analysis(log_text: str) -> StagedResponse:
     analyzed_snippets = await asyncio.gather(
         *[
             submit_text(
+                http,
                 PROMPT_CONFIG.snippet_prompt_template.format(s),
                 model=SERVER_CONFIG.inference.model,
                 max_tokens=SERVER_CONFIG.inference.max_tokens,
@@ -358,6 +412,7 @@ async def perform_staged_analysis(log_text: str) -> StagedResponse:
     )
 
     final_analysis = await submit_text(
+        http,
         final_prompt,
         model=SERVER_CONFIG.inference.model,
         max_tokens=SERVER_CONFIG.inference.max_tokens,
@@ -385,14 +440,17 @@ async def perform_staged_analysis(log_text: str) -> StagedResponse:
 
 @app.post("/analyze/stream", response_class=StreamingResponse)
 @track_request()
-async def analyze_log_stream(build_log: BuildLog):
+async def analyze_log_stream(
+    build_log: BuildLog, http_session: aiohttp.ClientSession = Depends(get_http_session)
+):
     """Stream response endpoint for Logdetective.
     Request must be in form {"url":"<YOUR_URL_HERE>"}.
     URL must be valid for the request to be passed to the LLM server.
     Meaning that it must contain appropriate scheme, path and netloc,
     while lacking  result, params or query fields.
     """
-    log_text = process_url(build_log.url)
+    remote_log = RemoteLog(build_log.url, http_session)
+    log_text = await remote_log.process_url()
     log_summary = mine_logs(log_text)
     log_summary = format_snippets(log_summary)
     headers = {"Content-Type": "application/json"}
@@ -400,42 +458,63 @@ async def analyze_log_stream(build_log: BuildLog):
     if SERVER_CONFIG.inference.api_token:
         headers["Authorization"] = f"Bearer {SERVER_CONFIG.inference.api_token}"
 
-    stream = await submit_text_chat_completions(
-        PROMPT_CONFIG.prompt_template.format(log_summary), stream=True, headers=headers,
-        model=SERVER_CONFIG.inference.model,
-        max_tokens=SERVER_CONFIG.inference.max_tokens,
-    )
+    try:
+        stream = await submit_text_chat_completions(
+            http_session,
+            PROMPT_CONFIG.prompt_template.format(log_summary),
+            stream=True,
+            headers=headers,
+            model=SERVER_CONFIG.inference.model,
+            max_tokens=SERVER_CONFIG.inference.max_tokens,
+        )
+    except aiohttp.ClientResponseError as ex:
+        raise HTTPException(
+            status_code=400,
+            detail="HTTP Error while getting response from inference server "
+            f"[{ex.status}] {ex.message}",
+        ) from ex
 
+    # we need to figure out a better response here, this is how it looks rn:
+    # b'data: {"choices":[{"finish_reason":"stop","index":0,"delta":{}}],
+    #   "created":1744818071,"id":"chatcmpl-c9geTxNcQO7M9wR...
     return StreamingResponse(stream)
 
 
 @app.post("/webhook/gitlab/job_events")
 async def receive_gitlab_job_event_webhook(
-    job_hook: JobHook, background_tasks: BackgroundTasks
+    x_gitlab_instance: Annotated[str | None, Header()],
+    job_hook: JobHook,
+    background_tasks: BackgroundTasks,
+    http: aiohttp.ClientSession = Depends(get_http_session),
 ):
     """Webhook endpoint for receiving job_events notifications from GitLab
     https://docs.gitlab.com/user/project/integrations/webhook_events/#job-events
     lists the full specification for the messages sent for job events."""
 
+    try:
+        forge = Forge(x_gitlab_instance)
+    except ValueError:
+        LOG.critical("%s is not a recognized forge. Ignoring.", x_gitlab_instance)
+        return BasicResponse(status_code=400)
+
     # Handle the message in the background so we can return 200 immediately
-    background_tasks.add_task(process_gitlab_job_event, job_hook)
+    background_tasks.add_task(process_gitlab_job_event, http, forge, job_hook)
 
     # No return value or body is required for a webhook.
     # 204: No Content
     return BasicResponse(status_code=204)
 
 
-async def process_gitlab_job_event(job_hook):
+async def process_gitlab_job_event(
+    http: aiohttp.ClientSession,
+    forge: Forge,
+    job_hook: JobHook,
+):
     """Handle a received job_event webhook from GitLab"""
-    LOG.debug("Received webhook message:\n%s", job_hook)
+    LOG.debug("Received webhook message from %s:\n%s", forge.value, job_hook)
 
     # Look up the project this job belongs to
     project = await asyncio.to_thread(app.gitlab_conn.projects.get, job_hook.project_id)
-
-    # check if this project is on the opt-in list
-    if project.name not in SERVER_CONFIG.general.packages:
-        LOG.info("Ignoring unrecognized package %s", project.name)
-        return
     LOG.info("Processing failed job for %s", project.name)
 
     # Retrieve data about the job from the GitLab API
@@ -466,25 +545,49 @@ async def process_gitlab_job_event(job_hook):
     LOG.debug("Retrieving log artifacts")
     # Retrieve the build logs from the merge request artifacts and preprocess them
     try:
-        log_url, preprocessed_log = await retrieve_and_preprocess_koji_logs(job)
+        log_url, preprocessed_log = await retrieve_and_preprocess_koji_logs(http, job)
     except LogsTooLargeError:
         LOG.error("Could not retrieve logs. Too large.")
         raise
 
     # Submit log to Log Detective and await the results.
     log_text = preprocessed_log.read().decode(encoding="utf-8")
-    staged_response = await perform_staged_analysis(log_text=log_text)
+    metrics_id = await add_new_metrics(
+        api_name=EndpointType.ANALYZE_GITLAB_JOB,
+        url=log_url,
+        http_session=http,
+        compressed_log_content=RemoteLog.zip_text(log_text),
+    )
+    staged_response = await perform_staged_analysis(http, log_text=log_text)
+    update_metrics(metrics_id, staged_response)
     preprocessed_log.close()
 
+    # check if this project is on the opt-in list for posting comments.
+    if project.name not in SERVER_CONFIG.general.packages:
+        LOG.info("Not publishing comment for unrecognized package %s", project.name)
+        return
+
     # Add the Log Detective response as a comment to the merge request
-    await comment_on_mr(project, merge_request_iid, job, log_url, staged_response)
+    await comment_on_mr(
+        forge,
+        project,
+        merge_request_iid,
+        job,
+        log_url,
+        staged_response,
+        metrics_id,
+    )
+
+    return staged_response
 
 
 class LogsTooLargeError(RuntimeError):
     """The log archive exceeds the configured maximum size"""
 
 
-async def retrieve_and_preprocess_koji_logs(job: gitlab.v4.objects.ProjectJob):
+async def retrieve_and_preprocess_koji_logs(
+    http: aiohttp.ClientSession, job: gitlab.v4.objects.ProjectJob
+):  # pylint: disable=too-many-branches
     """Download logs from the merge request artifacts
 
     This function will retrieve the build logs and do some minimal
@@ -495,7 +598,7 @@ async def retrieve_and_preprocess_koji_logs(job: gitlab.v4.objects.ProjectJob):
     Detective. The calling function is responsible for closing this object."""
 
     # Make sure the file isn't too large to process.
-    if not await check_artifacts_file_size(job):
+    if not await check_artifacts_file_size(http, job):
         raise LogsTooLargeError(
             f"Oversized logs for job {job.id} in project {job.project_id}"
         )
@@ -513,13 +616,16 @@ async def retrieve_and_preprocess_koji_logs(job: gitlab.v4.objects.ProjectJob):
         if zipinfo.filename.endswith("task_failed.log"):
             # The koji logs store this file in two places: 1) in the
             # directory with the failed architecture and 2) in the parent
-            # directory. We actually want to ignore the one in the parent
-            # directory, since the rest of the information is in the
-            # specific task directory.
+            # directory. Most of the time, we want to ignore the one in the
+            # parent directory, since the rest of the information is in the
+            # specific task directory. However, there are some situations
+            # where non-build failures (such as "Target build already exists")
+            # may be presented only at the top level.
             # The paths look like `kojilogs/noarch-XXXXXX/task_failed.log`
             # or `kojilogs/noarch-XXXXXX/x86_64-XXXXXX/task_failed.log`
             path = PurePath(zipinfo.filename)
             if len(path.parts) <= 3:
+                failed_arches["toplevel"] = path
                 continue
 
             # Extract the architecture from the immediate parent path
@@ -548,30 +654,32 @@ async def retrieve_and_preprocess_koji_logs(job: gitlab.v4.objects.ProjectJob):
             failed_arches[architecture] = PurePath(path.parent, failure_log_name)
 
     if not failed_arches:
-        # No failed task found?
+        # No failed task found in the sub-tasks.
         raise FileNotFoundError("Could not detect failed architecture.")
 
-    # First check if we only found one failed architecture
-    if len(failed_arches) == 1:
-        failed_arch = list(failed_arches.keys())[0]
-
+    # We only want to handle one arch, so we'll check them in order of
+    # "most to least likely for the maintainer to have access to hardware"
+    # This means: x86_64 > aarch64 > riscv > ppc64le > s390x
+    if "x86_64" in failed_arches:
+        failed_arch = "x86_64"
+    elif "aarch64" in failed_arches:
+        failed_arch = "aarch64"
+    elif "riscv" in failed_arches:
+        failed_arch = "riscv"
+    elif "ppc64le" in failed_arches:
+        failed_arch = "ppc64le"
+    elif "s390x" in failed_arches:
+        failed_arch = "s390x"
+    elif "noarch" in failed_arches:
+        # May have failed during BuildSRPMFromSCM phase
+        failed_arch = "noarch"
+    elif "toplevel" in failed_arches:
+        # Probably a Koji-specific error, not a build error
+        failed_arch = "toplevel"
     else:
-        # We only want to handle one arch, so we'll check them in order of
-        # "most to least likely for the maintainer to have access to hardware"
-        # This means: x86_64 > aarch64 > ppc64le > s390x
-        if "x86_64" in failed_arches:
-            failed_arch = "x86_64"
-        elif "aarch64" in failed_arches:
-            failed_arch = "aarch64"
-        elif "ppc64le" in failed_arches:
-            failed_arch = "ppc64le"
-        elif "s390x" in failed_arches:
-            failed_arch = "s390x"
-        else:
-            # It should be impossible for us to get "noarch" here, since
-            # the only way that should happen is for a single architecture
-            # build.
-            raise FileNotFoundError("No failed architecture detected.")
+        # We have one or more architectures that we don't know about? Just
+        # pick the first alphabetically.
+        failed_arch = sorted(list(failed_arches.keys()))[0]
 
     LOG.debug("Failed architecture: %s", failed_arch)
 
@@ -584,21 +692,31 @@ async def retrieve_and_preprocess_koji_logs(job: gitlab.v4.objects.ProjectJob):
     return log_url, artifacts_zip.open(log_path)
 
 
-async def check_artifacts_file_size(job):
+async def check_artifacts_file_size(
+    http: aiohttp.ClientSession,
+    job: gitlab.v4.objects.ProjectJob,
+):
     """Method to determine if the artifacts are too large to process"""
     # First, make sure that the artifacts are of a reasonable size. The
     # zipped artifact collection will be stored in memory below. The
     # python-gitlab library doesn't expose a way to check this value directly,
     # so we need to interact with directly with the headers.
     artifacts_url = f"{SERVER_CONFIG.gitlab.api_url}/projects/{job.project_id}/jobs/{job.id}/artifacts"  # pylint: disable=line-too-long
-    header_resp = await asyncio.to_thread(
-        requests.head,
-        artifacts_url,
-        allow_redirects=True,
-        headers={"Authorization": f"Bearer {SERVER_CONFIG.gitlab.api_token}"},
-        timeout=(3.07, 5),
-    )
-    content_length = int(header_resp.headers.get("content-length"))
+    LOG.debug("checking artifact URL %s", artifacts_url)
+    try:
+        head_response = await http.head(
+            artifacts_url,
+            allow_redirects=True,
+            headers={"Authorization": f"Bearer {SERVER_CONFIG.gitlab.api_token}"},
+            timeout=5,
+            raise_for_status=True,
+        )
+    except aiohttp.ClientResponseError as ex:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Unable to check artifact URL: [{ex.status}] {ex.message}",
+        ) from ex
+    content_length = int(head_response.headers.get("content-length"))
     LOG.debug(
         "URL: %s, content-length: %d, max length: %d",
         artifacts_url,
@@ -608,12 +726,14 @@ async def check_artifacts_file_size(job):
     return content_length <= SERVER_CONFIG.gitlab.max_artifact_size
 
 
-async def comment_on_mr(
+async def comment_on_mr(  # pylint: disable=too-many-arguments disable=too-many-positional-arguments
+    forge: Forge,
     project: gitlab.v4.objects.Project,
     merge_request_iid: int,
     job: gitlab.v4.objects.ProjectJob,
     log_url: str,
     response: StagedResponse,
+    metrics_id: int,
 ):
     """Add the Log Detective response as a comment to the merge request"""
     LOG.debug(
@@ -623,6 +743,10 @@ async def comment_on_mr(
         response.explanation.text,
     )
 
+    # First, we'll see if there's an existing comment on this Merge Request
+    # and wrap it in <details></details> to reduce noise.
+    await suppress_latest_comment(forge, project, merge_request_iid)
+
     # Get the formatted short comment.
     short_comment = await generate_mr_comment(job, log_url, response, full=False)
 
@@ -653,6 +777,67 @@ async def comment_on_mr(
     await asyncio.sleep(5)
     await asyncio.to_thread(note.save)
 
+    # Save the new comment to the database
+    try:
+        metrics = AnalyzeRequestMetrics.get_metric_by_id(metrics_id)
+        Comments.create(
+            forge,
+            project.id,
+            merge_request_iid,
+            job.id,
+            discussion.id,
+            metrics,
+        )
+    except sqlalchemy.exc.IntegrityError:
+        # We most likely attempted to save a new comment for the same
+        # build job. This is somewhat common during development when we're
+        # submitting requests manually. It shouldn't really happen in
+        # production.
+        if not SERVER_CONFIG.general.devmode:
+            raise
+
+
+async def suppress_latest_comment(
+    gitlab_instance: str,
+    project: gitlab.v4.objects.Project,
+    merge_request_iid: int,
+) -> None:
+    """Look up the latest comment on this Merge Request, if any, and wrap it
+    in a <details></details> block with a comment indicating that it has been
+    superseded by a new push."""
+
+    # Ask the database for the last known comment for this MR
+    previous_comment = Comments.get_latest_comment(
+        gitlab_instance, project.id, merge_request_iid
+    )
+
+    if previous_comment is None:
+        # No existing comment, so nothing to do.
+        return
+
+    # Retrieve its content from the Gitlab API
+
+    # Look up the merge request
+    merge_request = await asyncio.to_thread(
+        project.mergerequests.get, merge_request_iid
+    )
+
+    # Find the discussion matching the latest comment ID
+    discussion = await asyncio.to_thread(
+        merge_request.discussions.get, previous_comment.comment_id
+    )
+
+    # Get the ID of the first note
+    note_id = discussion.attributes["notes"][0]["id"]
+    note = discussion.notes.get(note_id)
+
+    # Wrap the note in <details>, indicating why.
+    note.body = (
+        "This comment has been superseded by a newer "
+        f"Log Detective analysis.\n<details>\n{note.body}\n</details>"
+    )
+    await asyncio.to_thread(note.save)
+
 
 async def generate_mr_comment(
     job: gitlab.v4.objects.ProjectJob,
@@ -727,61 +912,70 @@ def _multiple_svg_figures_response(figures: list[matplotlib.figure.Figure]):
     return BasicResponse(content=html_content, media_type="text/html")
 
 
-@app.get("/metrics/analyze", response_class=StreamingResponse)
-async def show_analyze_metrics(period_since_now: TimePeriod = Depends(TimePeriod)):
-    """Show statistics for requests and responses in the given period of time
-    for the /analyze API endpoint."""
-    fig_requests = plot.requests_per_time(period_since_now, EndpointType.ANALYZE)
-    fig_responses = plot.average_time_per_responses(
-        period_since_now, EndpointType.ANALYZE
-    )
-    return _multiple_svg_figures_response([fig_requests, fig_responses])
+class MetricRoute(str, Enum):
+    """Routes for metrics"""
 
+    ANALYZE = "analyze"
+    ANALYZE_STAGED = "analyze-staged"
+    ANALYZE_GITLAB_JOB = "analyze-gitlab"
 
-@app.get("/metrics/analyze/requests", response_class=StreamingResponse)
-async def show_analyze_requests(period_since_now: TimePeriod = Depends(TimePeriod)):
-    """Show statistics for the requests received in the given period of time
-    for the /analyze API endpoint."""
-    fig = plot.requests_per_time(period_since_now, EndpointType.ANALYZE)
-    return _svg_figure_response(fig)
 
+class Plot(str, Enum):
+    """Type of served plots"""
 
-@app.get("/metrics/analyze/responses", response_class=StreamingResponse)
-async def show_analyze_responses(period_since_now: TimePeriod = Depends(TimePeriod)):
-    """Show statistics for responses given in the specified period of time
-    for the /analyze API endpoint."""
-    fig = plot.average_time_per_responses(period_since_now, EndpointType.ANALYZE)
-    return _svg_figure_response(fig)
+    REQUESTS = "requests"
+    RESPONSES = "responses"
+    BOTH = ""
 
 
-@app.get("/metrics/analyze/staged", response_class=StreamingResponse)
-async def show_analyze_staged_metrics(
-    period_since_now: TimePeriod = Depends(TimePeriod),
-):
-    """Show statistics for requests and responses in the given period of time
-    for the /analyze/staged API endpoint."""
-    fig_requests = plot.requests_per_time(period_since_now, EndpointType.ANALYZE_STAGED)
-    fig_responses = plot.average_time_per_responses(
-        period_since_now, EndpointType.ANALYZE_STAGED
-    )
-    return _multiple_svg_figures_response([fig_requests, fig_responses])
+ROUTE_TO_ENDPOINT_TYPES = {
+    MetricRoute.ANALYZE: EndpointType.ANALYZE,
+    MetricRoute.ANALYZE_STAGED: EndpointType.ANALYZE_STAGED,
+    MetricRoute.ANALYZE_GITLAB_JOB: EndpointType.ANALYZE_GITLAB_JOB,
+}
 
 
-@app.get("/metrics/analyze/staged/requests", response_class=StreamingResponse)
-async def show_analyze_staged_requests(
+@app.get("/metrics/{route}/", response_class=StreamingResponse)
+@app.get("/metrics/{route}/{plot}", response_class=StreamingResponse)
+async def get_metrics(
+    route: MetricRoute,
+    plot: Plot = Plot.BOTH,
     period_since_now: TimePeriod = Depends(TimePeriod),
 ):
-    """Show statistics for the requests received in the given period of time
-    for the /analyze/staged API endpoint."""
-    fig = plot.requests_per_time(period_since_now, EndpointType.ANALYZE_STAGED)
-    return _svg_figure_response(fig)
-
+    """Get an handler for visualize statistics for the specified endpoint and plot."""
+    endpoint_type = ROUTE_TO_ENDPOINT_TYPES[route]
+
+    async def handler():
+        """Show statistics for the specified endpoint and plot."""
+        if plot == Plot.REQUESTS:
+            fig = plot_engine.requests_per_time(period_since_now, endpoint_type)
+            return _svg_figure_response(fig)
+        if plot == Plot.RESPONSES:
+            fig = plot_engine.average_time_per_responses(
+                period_since_now, endpoint_type
+            )
+            return _svg_figure_response(fig)
+        # BOTH
+        fig_requests = plot_engine.requests_per_time(period_since_now, endpoint_type)
+        fig_responses = plot_engine.average_time_per_responses(
+            period_since_now, endpoint_type
+        )
+        return _multiple_svg_figures_response([fig_requests, fig_responses])
+
+    descriptions = {
+        Plot.REQUESTS: (
+            "Show statistics for the requests received in the given period of time "
+            f"for the /{endpoint_type.value} API endpoint."
+        ),
+        Plot.RESPONSES: (
+            "Show statistics for responses given in the specified period of time "
+            f"for the /{endpoint_type.value} API endpoint."
+        ),
+        Plot.BOTH: (
+            "Show statistics for requests and responses in the given period of time "
+            f"for the /{endpoint_type.value} API endpoint."
+        ),
+    }
+    handler.__doc__ = descriptions[plot]
 
-@app.get("/metrics/analyze/staged/responses", response_class=StreamingResponse)
-async def show_analyze_staged_responses(
-    period_since_now: TimePeriod = Depends(TimePeriod),
-):
-    """Show statistics for responses given in the specified period of time
-    for the /analyze/staged API endpoint."""
-    fig = plot.average_time_per_responses(period_since_now, EndpointType.ANALYZE_STAGED)
-    return _svg_figure_response(fig)
+    return await handler()