logdetective 0.4.0__py3-none-any.whl → 2.11.0__py3-none-any.whl

logdetective/server/server.py

@@ -1,596 +1,708 @@
-import asyncio
-import json
 import os
-import re
-import zipfile
-from pathlib import PurePath
-from tempfile import TemporaryFile
-from typing import List, Annotated, Tuple, Dict, Any
-
-
-from fastapi import FastAPI, HTTPException, BackgroundTasks, Depends, Header
+import asyncio
+import datetime
+from enum import Enum
+from contextlib import asynccontextmanager
+from typing import Annotated
+from io import BytesIO
+
+import matplotlib
+import matplotlib.figure
+import matplotlib.pyplot
+from fastapi import (
+    FastAPI,
+    HTTPException,
+    BackgroundTasks,
+    Depends,
+    Header,
+    Path,
+    Request,
+)
 
 from fastapi.responses import StreamingResponse
 from fastapi.responses import Response as BasicResponse
-import gitlab
-import requests
+import aiohttp
+import sentry_sdk
+
+from logdetective.server.exceptions import KojiInvalidTaskID
 
-from logdetective.constants import (
-    PROMPT_TEMPLATE,
-    SNIPPET_PROMPT_TEMPLATE,
-    PROMPT_TEMPLATE_STAGED,
+from logdetective.server.database.models.koji import KojiTaskAnalysis
+from logdetective.server.database.models.exceptions import (
+    KojiTaskAnalysisTimeoutError,
+    KojiTaskNotAnalyzedError,
+    KojiTaskNotFoundError,
 )
-from logdetective.extractors import DrainExtractor
-from logdetective.utils import (
-    validate_url,
-    compute_certainty,
-    format_snippets,
-    format_analyzed_snippets,
+
+import logdetective.server.database.base
+
+from logdetective.server.config import SERVER_CONFIG, LOG
+from logdetective.server.koji import (
+    get_failed_log_from_task as get_failed_log_from_koji_task,
+)
+from logdetective.remote_log import RemoteLog
+from logdetective.server.llm import (
+    perform_staged_analysis,
+    perfrom_analysis,
+    perform_analyis_stream,
 )
-from logdetective.server.utils import load_server_config, get_log
-from logdetective.server.metric import track_request
+from logdetective.server.gitlab import process_gitlab_job_event
+from logdetective.server.metric import track_request, add_new_metrics, update_metrics
 from logdetective.server.models import (
     BuildLog,
+    EmojiHook,
     JobHook,
+    KojiInstanceConfig,
+    KojiStagedResponse,
     Response,
     StagedResponse,
-    Explanation,
-    AnalyzedSnippet,
+    TimePeriod,
+)
+from logdetective.server import plot as plot_engine
+from logdetective.server.database.models import (
+    EndpointType,
+    Forge,
+)
+from logdetective.server.emoji import (
+    collect_emojis,
+    collect_emojis_for_mr,
 )
+from logdetective.server.compressors import RemoteLogCompressor
+from logdetective.server.utils import get_version
+
 
-LLM_CPP_HOST = os.environ.get("LLAMA_CPP_HOST", "localhost")
-LLM_CPP_SERVER_ADDRESS = f"http://{LLM_CPP_HOST}"
-LLM_CPP_SERVER_PORT = os.environ.get("LLAMA_CPP_SERVER_PORT", 8000)
-LLM_CPP_SERVER_TIMEOUT = os.environ.get("LLAMA_CPP_SERVER_TIMEOUT", 600)
 LOG_SOURCE_REQUEST_TIMEOUT = os.environ.get("LOG_SOURCE_REQUEST_TIMEOUT", 60)
 API_TOKEN = os.environ.get("LOGDETECTIVE_TOKEN", None)
-SERVER_CONFIG_PATH = os.environ.get("LOGDETECTIVE_SERVER_CONF", None)
-LLM_API_TOKEN = os.environ.get("LLM_API_TOKEN", None)
 
-SERVER_CONFIG = load_server_config(SERVER_CONFIG_PATH)
 
-MR_REGEX = re.compile(r"refs/merge-requests/(\d+)/merge")
-FAILURE_LOG_REGEX = re.compile(r"(\w*\.log)")
+if sentry_dsn := SERVER_CONFIG.general.sentry_dsn:
+    sentry_sdk.init(dsn=str(sentry_dsn), traces_sample_rate=1.0)
+
+
+@asynccontextmanager
+async def lifespan(fapp: FastAPI):
+    """
+    Establish one HTTP session
+    """
+    fapp.http = aiohttp.ClientSession(
+        timeout=aiohttp.ClientTimeout(
+            total=int(LOG_SOURCE_REQUEST_TIMEOUT), connect=3.07
+        )
+    )
+
+    # Ensure that the database is initialized.
+    await logdetective.server.database.base.init()
+
+    # Start the background task scheduler for collecting emojis
+    asyncio.create_task(schedule_collect_emojis_task())
+
+    yield
 
-LOG = get_log(SERVER_CONFIG)
+    await fapp.http.close()
 
 
-def requires_token_when_set(authentication: Annotated[str | None, Header()] = None):
+async def get_http_session(request: Request) -> aiohttp.ClientSession:
     """
-    FastAPI Depend function that expects a header named Authentication
+    Return the single aiohttp ClientSession for this app
+    """
+    return request.app.http
+
+
+def requires_token_when_set(authorization: Annotated[str | None, Header()] = None):
+    """
+    FastAPI Depend function that expects a header named Authorization
 
     If LOGDETECTIVE_TOKEN env var is set, validate the client-supplied token
     otherwise ignore it
     """
     if not API_TOKEN:
-        LOG.info("LOGDETECTIVE_TOKEN env var not set, authentication disabled")
+        LOG.info("LOGDETECTIVE_TOKEN env var not set, authorization disabled")
         # no token required, means local dev environment
         return
-    token = None
-    if authentication:
+    if authorization:
         try:
-            token = authentication.split(" ", 1)[1]
-        except (ValueError, IndexError):
+            token = authorization.split(" ", 1)[1]
+        except (ValueError, IndexError) as ex:
             LOG.warning(
-                "Authentication header has invalid structure (%s), it should be 'Bearer TOKEN'",
-                authentication,
+                "Authorization header has invalid structure '%s', it should be 'Bearer TOKEN'",
+                authorization,
             )
             # eat the exception and raise 401 below
-            token = None
+            raise HTTPException(
+                status_code=401,
+                detail=f"Invalid authorization, HEADER '{authorization}' not valid.",
+            ) from ex
         if token == API_TOKEN:
             return
-    LOG.info(
-        "LOGDETECTIVE_TOKEN env var is set (%s), clien token = %s", API_TOKEN, token
-    )
-    raise HTTPException(status_code=401, detail=f"Token {token} not valid.")
+        LOG.info("Provided token '%s' does not match expected value.", token)
+        raise HTTPException(status_code=401, detail=f"Token '{token}' not valid.")
+    LOG.error("No authorization header provided but LOGDETECTIVE_TOKEN env var is set")
+    raise HTTPException(status_code=401, detail="No token provided.")
+
+
+app = FastAPI(
+    title="Log Detective",
+    contact={
+        "name": "Log Detective developers",
+        "url": "https://github.com/fedora-copr/logdetective",
+        "email": "copr-devel@lists.fedorahosted.org"
+    },
+    license_info={
+        "name": "Apache-2.0",
+        "url": "https://www.apache.org/licenses/LICENSE-2.0.html",
+    },
+    version=get_version(),
+    dependencies=[Depends(requires_token_when_set)], lifespan=lifespan)
 
 
-app = FastAPI(dependencies=[Depends(requires_token_when_set)])
-app.gitlab_conn = gitlab.Gitlab(
-    url=SERVER_CONFIG.gitlab.url, private_token=SERVER_CONFIG.gitlab.api_token
-)
-
+@app.post("/analyze", response_model=Response)
+@track_request()
+async def analyze_log(
+    build_log: BuildLog, http_session: aiohttp.ClientSession = Depends(get_http_session)
+):
+    """Provide endpoint for log file submission and analysis.
+    Request must be in form {"url":"<YOUR_URL_HERE>"}.
+    URL must be valid for the request to be passed to the LLM server.
+    Meaning that it must contain appropriate scheme, path and netloc,
+    while lacking  result, params or query fields.
+    """
+    remote_log = RemoteLog(build_log.url, http_session)
+    log_text = await remote_log.process_url()
 
-def process_url(url: str) -> str:
-    """Validate log URL and return log text."""
-    if validate_url(url=url):
-        try:
-            log_request = requests.get(url, timeout=int(LOG_SOURCE_REQUEST_TIMEOUT))
-        except requests.RequestException as ex:
-            raise HTTPException(
-                status_code=400, detail=f"We couldn't obtain the logs: {ex}"
-            ) from ex
+    return await perfrom_analysis(log_text)
 
-        if not log_request.ok:
-            raise HTTPException(
-                status_code=400,
-                detail="Something went wrong while getting the logs: "
-                f"[{log_request.status_code}] {log_request.text}",
-            )
-    else:
-        LOG.error("Invalid URL received ")
-        raise HTTPException(status_code=400, detail=f"Invalid log URL: {url}")
 
-    return log_request.text
+@app.post("/analyze/staged", response_model=StagedResponse)
+@track_request()
+async def analyze_log_staged(
+    build_log: BuildLog, http_session: aiohttp.ClientSession = Depends(get_http_session)
+):
+    """Provide endpoint for log file submission and analysis.
+    Request must be in form {"url":"<YOUR_URL_HERE>"}.
+    URL must be valid for the request to be passed to the LLM server.
+    Meaning that it must contain appropriate scheme, path and netloc,
+    while lacking  result, params or query fields.
+    """
+    remote_log = RemoteLog(build_log.url, http_session)
+    log_text = await remote_log.process_url()
 
+    return await perform_staged_analysis(log_text)
 
-def mine_logs(log: str) -> List[Tuple[int, str]]:
-    """Extract snippets from log text"""
-    extractor = DrainExtractor(
-        verbose=True, context=True, max_clusters=SERVER_CONFIG.extractor.max_clusters
-    )
 
-    LOG.info("Getting summary")
-    log_summary = extractor(log)
+@app.get(
+    "/analyze/rpmbuild/koji/{koji_instance}/{task_id}",
+    response_model=KojiStagedResponse,
+)
+async def get_koji_task_analysis(
+    koji_instance: Annotated[str, Path(title="The Koji instance to use")],
+    task_id: Annotated[int, Path(title="The task ID to analyze")],
+    x_koji_token: Annotated[str, Header()] = "",
+):
+    """Provide endpoint for retrieving log file analysis of a Koji task"""
 
-    ratio = len(log_summary) / len(log.split("\n"))
-    LOG.debug("Log summary: \n %s", log_summary)
-    LOG.info("Compression ratio: %s", ratio)
+    try:
+        koji_instance_config = SERVER_CONFIG.koji.instances[koji_instance]
+    except KeyError:
+        # This Koji instance is not configured, so we will return a 404.
+        return BasicResponse(status_code=404, content="Unknown Koji instance.")
+
+    # This should always be available in a production environment.
+    # In a testing environment, the tokens list may be empty, in which case
+    # it will just proceed.
+    if koji_instance_config.tokens and x_koji_token not in koji_instance_config.tokens:
+        # (Unauthorized) error.
+        return BasicResponse(x_koji_token, status_code=401)
+
+    # Check if we have a response for this task
+    try:
+        return KojiTaskAnalysis.get_response_by_task_id(task_id)
+
+    except (KojiInvalidTaskID, KojiTaskNotFoundError):
+        # This task ID is malformed, out of range, or not found, so we will
+        # return a 404.
+        return BasicResponse(status_code=404)
+
+    except KojiTaskAnalysisTimeoutError:
+        # Task analysis has timed out, so we assume that the request was lost
+        # and that we need to start another analysis.
+        # There isn't a fully-appropriate error code for this, so we'll use
+        # 503 (Service Unavailable) as our best option.
+        return BasicResponse(
+            status_code=503, content="Task analysis timed out, please retry."
+        )
 
-    return log_summary
+    except KojiTaskNotAnalyzedError:
+        # Its still running, so we need to return a 202
+        # (Accepted) code to let the client know to keep waiting.
+        return BasicResponse(
+            status_code=202, content=f"Analysis still in progress for task {task_id}"
+        )
 
 
-async def submit_to_llm_endpoint(
-    url: str, data: Dict[str, Any], headers: Dict[str, str], stream: bool
-) -> Any:
-    """Send request to selected API endpoint. Verifying successful request unless
-    the using the stream response.
+@app.post(
+    "/analyze/rpmbuild/koji/{koji_instance}/{task_id}",
+    response_model=KojiStagedResponse,
+)
+async def analyze_rpmbuild_koji(
+    koji_instance: Annotated[str, Path(title="The Koji instance to use")],
+    task_id: Annotated[int, Path(title="The task ID to analyze")],
+    x_koji_token: Annotated[str, Header()] = "",
+    x_koji_callback: Annotated[str, Header()] = "",
+    background_tasks: BackgroundTasks = BackgroundTasks(),
+):
+    """Provide endpoint for retrieving log file analysis of a Koji task"""
 
-    url:
-    data:
-    headers:
-    stream:
-    """
     try:
-        # Expects llama-cpp server to run on LLM_CPP_SERVER_ADDRESS:LLM_CPP_SERVER_PORT
-        response = requests.post(
-            url,
-            headers=headers,
-            data=json.dumps(data),
-            timeout=int(LLM_CPP_SERVER_TIMEOUT),
-            stream=stream,
-        )
-    except requests.RequestException as ex:
-        LOG.error("Llama-cpp query failed: %s", ex)
-        raise HTTPException(
-            status_code=400, detail=f"Llama-cpp query failed: {ex}"
-        ) from ex
-    if not stream:
-        if not response.ok:
-            raise HTTPException(
-                status_code=400,
-                detail="Something went wrong while getting a response from the llama server: "
-                f"[{response.status_code}] {response.text}",
-            )
-        try:
-            response = json.loads(response.text)
-        except UnicodeDecodeError as ex:
-            LOG.error("Error encountered while parsing llama server response: %s", ex)
-            raise HTTPException(
-                status_code=400,
-                detail=f"Couldn't parse the response.\nError: {ex}\nData: {response.text}",
-            ) from ex
+        koji_instance_config = SERVER_CONFIG.koji.instances[koji_instance]
+    except KeyError:
+        # This Koji instance is not configured, so we will return a 404.
+        return BasicResponse(status_code=404, content="Unknown Koji instance.")
+
+    # This should always be available in a production environment.
+    # In a testing environment, the tokens list may be empty, in which case
+    # it will just proceed.
+    if koji_instance_config.tokens and x_koji_token not in koji_instance_config.tokens:
+        # (Unauthorized) error.
+        return BasicResponse(x_koji_token, status_code=401)
+
+    # Check if we already have a response for this task
+    try:
+        response = KojiTaskAnalysis.get_response_by_task_id(task_id)
 
-    return response
+    except KojiInvalidTaskID:
+        # This task ID is malformed or out of range, so we will return a 400.
+        response = BasicResponse(status_code=404, content="Invalid or unknown task ID.")
 
+    except (KojiTaskNotFoundError, KojiTaskAnalysisTimeoutError):
+        # Task not yet analyzed or it timed out, so we need to start the
+        # analysis in the background and return a 202 (Accepted) error.
 
-async def submit_text(  # pylint: disable=R0913,R0917
-    text: str,
-    max_tokens: int = -1,
-    log_probs: int = 1,
-    stream: bool = False,
-    model: str = "default-model",
-    api_endpoint: str = "/chat/completions",
-) -> Explanation:
-    """Submit prompt to LLM using a selected endpoint.
-    max_tokens: number of tokens to be produces, 0 indicates run until encountering EOS
-    log_probs: number of token choices to produce log probs for
-    """
-    LOG.info("Analyzing the text")
+        background_tasks.add_task(
+            analyze_koji_task,
+            task_id,
+            koji_instance_config,
+        )
 
-    headers = {"Content-Type": "application/json"}
+        # If a callback URL is provided, we need to add it to the callbacks
+        # table so that we can notify it when the analysis is complete.
+        if x_koji_callback:
+            koji_instance_config.register_callback(task_id, x_koji_callback)
 
-    if LLM_API_TOKEN:
-        headers["Authorization"] = f"Bearer {LLM_API_TOKEN}"
+        response = BasicResponse(
+            status_code=202, content=f"Beginning analysis of task {task_id}"
+        )
 
-    if api_endpoint == "/chat/completions":
-        return await submit_text_chat_completions(
-            text, headers, max_tokens, log_probs > 0, stream, model
+    except KojiTaskNotAnalyzedError:
+        # Its still running, so we need to return a 202
+        # (Accepted) error.
+        response = BasicResponse(
+            status_code=202, content=f"Analysis still in progress for task {task_id}"
         )
-    return await submit_text_completions(
-        text, headers, max_tokens, log_probs, stream, model
-    )
 
+    return response
 
-async def submit_text_completions(  # pylint: disable=R0913,R0917
-    text: str,
-    headers: dict,
-    max_tokens: int = -1,
-    log_probs: int = 1,
-    stream: bool = False,
-    model: str = "default-model",
-) -> Explanation:
-    """Submit prompt to OpenAI API completions endpoint.
-    max_tokens: number of tokens to be produces, 0 indicates run until encountering EOS
-    log_probs: number of token choices to produce log probs for
-    """
-    LOG.info("Submitting to /v1/completions endpoint")
-    data = {
-        "prompt": text,
-        "max_tokens": max_tokens,
-        "logprobs": log_probs,
-        "stream": stream,
-        "model": model,
-    }
 
-    response = await submit_to_llm_endpoint(
-        f"{LLM_CPP_SERVER_ADDRESS}:{LLM_CPP_SERVER_PORT}/v1/completions",
-        data,
-        headers,
-        stream,
-    )
+async def analyze_koji_task(task_id: int, koji_instance_config: KojiInstanceConfig):
+    """Analyze a koji task and return the response"""
 
-    return Explanation(
-        text=response["choices"][0]["text"], logprobs=response["choices"][0]["logprobs"]
+    # Get the log text from the koji task
+    koji_conn = koji_instance_config.get_connection()
+    log_file_name, log_text = await get_failed_log_from_koji_task(
+        koji_conn, task_id, max_size=SERVER_CONFIG.koji.max_artifact_size
     )
 
+    # We need to handle the metric tracking manually here, because we need
+    # to retrieve the metric ID to associate it with the koji task analysis.
 
-async def submit_text_chat_completions(  # pylint: disable=R0913,R0917
-    text: str,
-    headers: dict,
-    max_tokens: int = -1,
-    log_probs: int = 1,
-    stream: bool = False,
-    model: str = "default-model",
-) -> Explanation:
-    """Submit prompt to OpenAI API /chat/completions endpoint.
-    max_tokens: number of tokens to be produces, 0 indicates run until encountering EOS
-    log_probs: number of token choices to produce log probs for
-    """
-    LOG.info("Submitting to /v1/chat/completions endpoint")
-
-    data = {
-        "messages": [
-            {
-                "role": "user",
-                "content": text,
-            }
-        ],
-        "max_tokens": max_tokens,
-        "logprobs": log_probs,
-        "stream": stream,
-        "model": model,
-    }
-
-    response = await submit_to_llm_endpoint(
-        f"{LLM_CPP_SERVER_ADDRESS}:{LLM_CPP_SERVER_PORT}/v1/chat/completions",
-        data,
-        headers,
-        stream,
+    metrics_id = await add_new_metrics(
+        EndpointType.ANALYZE_KOJI_TASK,
+        log_text,
+        received_at=datetime.datetime.now(datetime.timezone.utc),
+        compressed_log_content=RemoteLogCompressor.zip_text(log_text),
     )
-
-    if stream:
-        return Explanation(
-            text=response["choices"][0]["delta"]["content"],
-            logprobs=response["choices"][0]["logprobs"]["content"],
-        )
-    return Explanation(
-        text=response["choices"][0]["message"]["content"],
-        logprobs=response["choices"][0]["logprobs"]["content"],
+    # We need to associate the metric ID with the koji task analysis.
+    # This will create the new row without a response, which we will use as
+    # an indicator that the analysis is in progress.
+    await KojiTaskAnalysis.create_or_restart(
+        koji_instance=koji_instance_config.xmlrpc_url,
+        task_id=task_id,
+        log_file_name=log_file_name,
     )
+    response = await perform_staged_analysis(log_text)
 
+    # Now that we have the response, we can update the metrics and mark the
+    # koji task analysis as completed.
+    await update_metrics(metrics_id, response)
+    await KojiTaskAnalysis.add_response(task_id, metrics_id)
 
-@app.post("/analyze", response_model=Response)
-@track_request()
-async def analyze_log(build_log: BuildLog):
-    """Provide endpoint for log file submission and analysis.
-    Request must be in form {"url":"<YOUR_URL_HERE>"}.
-    URL must be valid for the request to be passed to the LLM server.
-    Meaning that it must contain appropriate scheme, path and netloc,
-    while lacking  result, params or query fields.
-    """
-    log_text = process_url(build_log.url)
-    log_summary = mine_logs(log_text)
-    log_summary = format_snippets(log_summary)
-    response = await submit_text(
-        PROMPT_TEMPLATE.format(log_summary),
-        api_endpoint=SERVER_CONFIG.inference.api_endpoint,
-    )
-    certainty = 0
+    # Notify any callbacks that the analysis is complete.
+    for callback in koji_instance_config.get_callbacks(task_id):
+        LOG.info("Notifying callback %s of task %d completion", callback, task_id)
+        asyncio.create_task(send_koji_callback(callback, task_id))
 
-    if response.logprobs is not None:
-        try:
-            certainty = compute_certainty(response.logprobs)
-        except ValueError as ex:
-            LOG.error("Error encountered while computing certainty: %s", ex)
-            raise HTTPException(
-                status_code=400,
-                detail=f"Couldn't compute certainty with data:\n"
-                f"{response.logprobs}",
-            ) from ex
+    # Now that it's sent, we can clear the callbacks for this task.
+    koji_instance_config.clear_callbacks(task_id)
 
-    return Response(explanation=response, response_certainty=certainty)
+    return response
 
 
-@app.post("/analyze/staged", response_model=StagedResponse)
-@track_request()
-async def analyze_log_staged(build_log: BuildLog):
-    """Provide endpoint for log file submission and analysis.
-    Request must be in form {"url":"<YOUR_URL_HERE>"}.
-    URL must be valid for the request to be passed to the LLM server.
-    Meaning that it must contain appropriate scheme, path and netloc,
-    while lacking  result, params or query fields.
-    """
-    log_text = process_url(build_log.url)
-    log_summary = mine_logs(log_text)
-
-    # Process snippets asynchronously
-    analyzed_snippets = await asyncio.gather(
-        *[
-            submit_text(
-                SNIPPET_PROMPT_TEMPLATE.format(s),
-                api_endpoint=SERVER_CONFIG.inference.api_endpoint,
-            )
-            for s in log_summary
-        ]
-    )
+async def send_koji_callback(callback: str, task_id: int):
+    """Send a callback to the specified URL with the task ID and log file name."""
+    async with aiohttp.ClientSession() as session:
+        async with session.post(callback, json={"task_id": task_id}):
+            pass
 
-    analyzed_snippets = [
-        AnalyzedSnippet(line_number=e[0][0], text=e[0][1], explanation=e[1])
-        for e in zip(log_summary, analyzed_snippets)
-    ]
-    final_prompt = PROMPT_TEMPLATE_STAGED.format(
-        format_analyzed_snippets(analyzed_snippets)
-    )
 
-    final_analysis = await submit_text(
-        final_prompt, api_endpoint=SERVER_CONFIG.inference.api_endpoint
-    )
+@app.get("/queue/print")
+async def queue_print(msg: str):
+    """Debug endpoint to test the LLM request queue"""
+    LOG.info("Will print %s", msg)
 
-    certainty = 0
+    result = await async_log(msg)
 
-    if final_analysis.logprobs:
-        try:
-            certainty = compute_certainty(final_analysis.logprobs)
-        except ValueError as ex:
-            LOG.error("Error encountered while computing certainty: %s", ex)
-            raise HTTPException(
-                status_code=400,
-                detail=f"Couldn't compute certainty with data:\n"
-                f"{final_analysis.logprobs}",
-            ) from ex
+    LOG.info("Printed %s and returned it", result)
 
-    return StagedResponse(
-        explanation=final_analysis,
-        snippets=analyzed_snippets,
-        response_certainty=certainty,
-    )
+
+async def async_log(msg):
+    """Debug function to test the LLM request queue"""
+    async with SERVER_CONFIG.inference.get_limiter():
+        LOG.critical(msg)
+    return msg
+
+
+@app.get("/version", response_class=BasicResponse)
+async def get_version_wrapper():
+    """Get the version of logdetective"""
+    return BasicResponse(content=get_version())
 
 
 @app.post("/analyze/stream", response_class=StreamingResponse)
 @track_request()
-async def analyze_log_stream(build_log: BuildLog):
+async def analyze_log_stream(
+    build_log: BuildLog, http_session: aiohttp.ClientSession = Depends(get_http_session)
+):
     """Stream response endpoint for Logdetective.
     Request must be in form {"url":"<YOUR_URL_HERE>"}.
     URL must be valid for the request to be passed to the LLM server.
     Meaning that it must contain appropriate scheme, path and netloc,
     while lacking  result, params or query fields.
     """
-    log_text = process_url(build_log.url)
-    log_summary = mine_logs(log_text)
-    log_summary = format_snippets(log_summary)
-    headers = {"Content-Type": "application/json"}
+    remote_log = RemoteLog(build_log.url, http_session)
+    log_text = await remote_log.process_url()
+    try:
+        stream = perform_analyis_stream(log_text)
+    except aiohttp.ClientResponseError as ex:
+        raise HTTPException(
+            status_code=400,
+            detail="HTTP Error while getting response from inference server "
+            f"[{ex.status}] {ex.message}",
+        ) from ex
 
-    if LLM_API_TOKEN:
-        headers["Authorization"] = f"Bearer {LLM_API_TOKEN}"
+    return StreamingResponse(stream)
 
-    stream = await submit_text_chat_completions(
-        PROMPT_TEMPLATE.format(log_summary), stream=True, headers=headers
-    )
 
-    return StreamingResponse(stream)
+def is_valid_webhook_secret(forge, x_gitlab_token):
+    """Check whether the provided x_gitlab_token matches the webhook secret
+    specified in the configuration"""
+
+    gitlab_cfg = SERVER_CONFIG.gitlab.instances[forge.value]
+
+    if not gitlab_cfg.webhook_secrets:
+        # No secrets specified, so don't bother validating.
+        # This is mostly to be used for development.
+        return True
+
+    if x_gitlab_token in gitlab_cfg.webhook_secrets:
+        return True
+
+    return False
 
 
 @app.post("/webhook/gitlab/job_events")
 async def receive_gitlab_job_event_webhook(
-    job_hook: JobHook, background_tasks: BackgroundTasks
+    job_hook: JobHook,
+    background_tasks: BackgroundTasks,
+    x_gitlab_instance: Annotated[str | None, Header()],
+    x_gitlab_token: Annotated[str | None, Header()] = None,
 ):
     """Webhook endpoint for receiving job_events notifications from GitLab
     https://docs.gitlab.com/user/project/integrations/webhook_events/#job-events
     lists the full specification for the messages sent for job events."""
 
-    # Handle the message in the background so we can return 200 immediately
-    background_tasks.add_task(process_gitlab_job_event, job_hook)
+    try:
+        forge = Forge(x_gitlab_instance)
+    except ValueError:
+        LOG.critical("%s is not a recognized forge. Ignoring.", x_gitlab_instance)
+        return BasicResponse(status_code=400)
+
+    if not is_valid_webhook_secret(forge, x_gitlab_token):
+        # This request could not be validated, so return a 401
+        # (Unauthorized) error.
+        return BasicResponse(status_code=401)
+
+    # Handle the message in the background so we can return 204 immediately
+    gitlab_cfg = SERVER_CONFIG.gitlab.instances[forge.value]
+    background_tasks.add_task(
+        process_gitlab_job_event,
+        gitlab_cfg,
+        forge,
+        job_hook,
+    )
 
     # No return value or body is required for a webhook.
     # 204: No Content
     return BasicResponse(status_code=204)
 
 
-async def process_gitlab_job_event(job_hook):
-    """Handle a received job_event webhook from GitLab"""
-    LOG.debug("Received webhook message:\n%s", job_hook)
+# A lookup table for whether we are currently processing a given merge request
+# The key is the tuple (Forge, ProjectID, MRID) and the value is a boolean
+# indicating whether we need to re-trigger the lookup immediately after
+# completion due to another request coming in during processing.
+# For example: {("https://gitlab.example.com", 23, 2): False}
+emoji_lookup = {}
 
-    # Look up the project this job belongs to
-    project = await asyncio.to_thread(app.gitlab_conn.projects.get, job_hook.project_id)
 
-    # check if this project is on the opt-in list
-    if project.name not in SERVER_CONFIG.general.packages:
-        LOG.info("Ignoring unrecognized package %s", project.name)
-        return
-    LOG.info("Processing failed job for %s", project.name)
+@app.post("/webhook/gitlab/emoji_events")
+async def receive_gitlab_emoji_event_webhook(
+    x_gitlab_instance: Annotated[str | None, Header()],
+    x_gitlab_token: Annotated[str | None, Header()],
+    emoji_hook: EmojiHook,
+    background_tasks: BackgroundTasks,
+):
+    """Webhook endpoint for receiving emoji event notifications from Gitlab
+    https://docs.gitlab.com/user/project/integrations/webhook_events/#emoji-events
+    lists the full specification for the messages sent for emoji events"""
 
-    # Retrieve data about the job from the GitLab API
-    job = await asyncio.to_thread(project.jobs.get, job_hook.build_id)
+    try:
+        forge = Forge(x_gitlab_instance)
+    except ValueError:
+        LOG.critical("%s is not a recognized forge. Ignoring.", x_gitlab_instance)
+        return BasicResponse(status_code=400)
+
+    if not is_valid_webhook_secret(forge, x_gitlab_token):
+        # This request could not be validated, so return a 401
+        # (Unauthorized) error.
+        return BasicResponse(status_code=401)
+
+    if not emoji_hook.merge_request:
+        # This is not a merge request event. It is probably an emoji applied
+        # to some other "awardable" entity. Just ignore it and return.
+        LOG.debug("Emoji event is not related to a merge request. Ignoring.")
+        return BasicResponse(status_code=204)
+
+    # We will re-process all the emojis on this merge request, to ensure that
+    # we haven't missed any messages, since webhooks do not provide delivery
+    # guarantees.
+
+    # Check whether this request is already in progress.
+    # We are single-threaded, so we can guarantee that the table won't change
+    # between here and when we schedule the lookup.
+    key = (
+        forge,
+        emoji_hook.merge_request.target_project_id,
+        emoji_hook.merge_request.iid,
+    )
+    if key in emoji_lookup:
+        # It's already in progress, so we do not want to start another pass
+        # concurrently. We'll set the value to True to indicate that we should
+        # re-enqueue this lookup after the currently-running one concludes. It
+        # is always safe to set this to True, even if it's already True. If
+        # multiple requests come in during processing, we only need to re-run
+        # it a single time, since it will pick up all the ongoing changes. The
+        # worst-case situation is the one where we receive new requests just
+        # after processing starts, which will cause the cycle to repeat again.
+        # This should be very infrequent, as emoji events are computationally
+        # rare and very quick to process.
+        emoji_lookup[key] = True
+        LOG.info("MR Emojis already being processed for %s. Rescheduling.", key)
+        return BasicResponse(status_code=204)
+
+    # Inform the lookup table that we are processing this emoji
+    emoji_lookup[key] = False
+
+    # Create a background task to process the emojis on this Merge Request.
+    background_tasks.add_task(
+        schedule_emoji_collection_for_mr,
+        forge,
+        emoji_hook.merge_request.target_project_id,
+        emoji_hook.merge_request.iid,
+        background_tasks,
+    )
 
-    # Retrieve the pipeline that started this job
-    pipeline = await asyncio.to_thread(project.pipelines.get, job_hook.pipeline_id)
+    # No return value or body is required for a webhook.
+    # 204: No Content
+    return BasicResponse(status_code=204)
 
-    # Verify this is a merge request
-    if pipeline.source != "merge_request_event":
-        LOG.info("Not a merge request pipeline. Ignoring.")
-        return
 
-    # Extract the merge-request ID from the job
-    match = MR_REGEX.search(pipeline.ref)
-    if not match:
-        LOG.error(
-            "Pipeline source is merge_request_event but no merge request ID was provided."
+async def schedule_emoji_collection_for_mr(
+    forge: Forge, project_id: int, mr_iid: int, background_tasks: BackgroundTasks
+):
+    """Background task to update the database on emoji reactions"""
+
+    key = (forge, project_id, mr_iid)
+
+    # FIXME: Look up the connection from the Forge  # pylint: disable=fixme
+    gitlab_conn = SERVER_CONFIG.gitlab.instances[forge.value].get_connection()
+
+    LOG.debug("Looking up emojis for %s, %d, %d", forge, project_id, mr_iid)
+    await collect_emojis_for_mr(project_id, mr_iid, gitlab_conn)
+
+    # Check whether we've been asked to re-schedule this lookup because
+    # another request came in while it was processing.
+    if emoji_lookup[key]:
+        # The value is Truthy, which tells us to re-schedule
+        # Reset the boolean value to indicate that we're underway again.
+        emoji_lookup[key] = False
+        background_tasks.add_task(
+            schedule_emoji_collection_for_mr,
+            forge,
+            project_id,
+            mr_iid,
+            background_tasks,
         )
         return
-    merge_request_id = int(match.group(1))
 
-    LOG.debug("Retrieving log artifacts")
-    # Retrieve the build logs from the merge request artifacts and preprocess them
-    try:
-        preprocessed_log = await retrieve_and_preprocess_koji_logs(job)
-    except LogsTooLargeError:
-        LOG.error("Could not retrieve logs. Too large.")
-        raise
+    # We're all done, so clear this entry out of the lookup
+    del emoji_lookup[key]
+
 
-    # Submit log to Log Detective and await the results.
-    response = await submit_log_to_llm(preprocessed_log)
-    preprocessed_log.close()
+def _svg_figure_response(fig: matplotlib.figure.Figure):
+    """Create a response with the given svg figure."""
+    buf = BytesIO()
+    fig.savefig(buf, format="svg", bbox_inches="tight")
+    matplotlib.pyplot.close(fig)
 
-    # Add the Log Detective response as a comment to the merge request
-    await comment_on_mr(merge_request_id, response)
+    buf.seek(0)
+    return StreamingResponse(
+        buf,
+        media_type="image/svg+xml",
+        headers={"Content-Disposition": "inline; filename=plot.svg"},
+    )
+
+
+def _multiple_svg_figures_response(figures: list[matplotlib.figure.Figure]):
+    """Create a response with multiple svg figures."""
+    svg_contents = []
+    for i, fig in enumerate(figures):
+        buf = BytesIO()
+        fig.savefig(buf, format="svg", bbox_inches="tight")
+        matplotlib.pyplot.close(fig)
+        buf.seek(0)
+        svg_contents.append(buf.read().decode("utf-8"))
+
+    html_content = "<html><body>\n"
+    for i, svg in enumerate(svg_contents):
+        html_content += f"<div id='figure-{i}'>\n{svg}\n</div>\n"
+    html_content += "</body></html>"
+
+    return BasicResponse(content=html_content, media_type="text/html")
 
 
-class LogsTooLargeError(RuntimeError):
-    """The log archive exceeds the configured maximum size"""
+class MetricRoute(str, Enum):
+    """Routes for metrics"""
 
+    ANALYZE = "analyze"
+    ANALYZE_STAGED = "analyze-staged"
+    ANALYZE_GITLAB_JOB = "analyze-gitlab"
 
-async def retrieve_and_preprocess_koji_logs(job):
-    """Download logs from the merge request artifacts
 
-    This function will retrieve the build logs and do some minimal
-    preprocessing to determine which log is relevant for analysis.
+class Plot(str, Enum):
+    """Type of served plots"""
 
-    returns: An open, file-like object containing the log contents to be sent
-    for processing by Log Detective. The calling function is responsible for
-    closing this object."""
+    REQUESTS = "requests"
+    RESPONSES = "responses"
+    EMOJIS = "emojis"
+    BOTH = ""
 
-    # Make sure the file isn't too large to process.
-    if not await check_artifacts_file_size(job):
-        raise LogsTooLargeError(
-            f"Oversized logs for job {job.id} in project {job.project_id}"
+
+ROUTE_TO_ENDPOINT_TYPES = {
+    MetricRoute.ANALYZE: EndpointType.ANALYZE,
+    MetricRoute.ANALYZE_STAGED: EndpointType.ANALYZE_STAGED,
+    MetricRoute.ANALYZE_GITLAB_JOB: EndpointType.ANALYZE_GITLAB_JOB,
+}
+
+
+@app.get("/metrics/{route}/", response_class=StreamingResponse)
+@app.get("/metrics/{route}/{plot}", response_class=StreamingResponse)
+async def get_metrics(
+    route: MetricRoute,
+    plot: Plot = Plot.BOTH,
+    period_since_now: TimePeriod = Depends(TimePeriod),
+):
+    """Get an handler for visualize statistics for the specified endpoint and plot."""
+    endpoint_type = ROUTE_TO_ENDPOINT_TYPES[route]
+
+    async def handler():
+        """Show statistics for the specified endpoint and plot."""
+        if plot == Plot.REQUESTS:
+            fig = await plot_engine.requests_per_time(period_since_now, endpoint_type)
+            return _svg_figure_response(fig)
+        if plot == Plot.RESPONSES:
+            fig = await plot_engine.average_time_per_responses(
+                period_since_now, endpoint_type
+            )
+            return _svg_figure_response(fig)
+        if plot == Plot.EMOJIS:
+            fig = await plot_engine.emojis_per_time(period_since_now)
+            return _svg_figure_response(fig)
+        # BOTH
+        fig_requests = await plot_engine.requests_per_time(
+            period_since_now, endpoint_type
         )
+        fig_responses = await plot_engine.average_time_per_responses(
+            period_since_now, endpoint_type
+        )
+        fig_emojis = await plot_engine.emojis_per_time(period_since_now)
+        return _multiple_svg_figures_response([fig_requests, fig_responses, fig_emojis])
+
+    descriptions = {
+        Plot.REQUESTS: (
+            "Show statistics for the requests received in the given period of time "
+            f"for the /{endpoint_type.value} API endpoint."
+        ),
+        Plot.RESPONSES: (
+            "Show statistics for responses given in the specified period of time "
+            f"for the /{endpoint_type.value} API endpoint."
+        ),
+        Plot.EMOJIS: (
+            "Show statistics for emoji feedback in the specified period of time "
+            f"for the /{endpoint_type.value} API endpoint."
+        ),
+        Plot.BOTH: (
+            "Show statistics for requests and responses in the given period of time "
+            f"for the /{endpoint_type.value} API endpoint."
+        ),
+    }
+    handler.__doc__ = descriptions[plot]
+
+    return await handler()
 
-    # Create a temporary file to store the downloaded log zipfile.
-    # This will be automatically deleted when the last reference into it
-    # (returned by this function) is closed.
-    tempfile = TemporaryFile(mode="w+b")
-    await asyncio.to_thread(job.artifacts, streamed=True, action=tempfile.write)
-    tempfile.seek(0)
-
-    failed_arches = {}
-    artifacts_zip = zipfile.ZipFile(tempfile, mode="r")
-    for zipinfo in artifacts_zip.infolist():
-        if zipinfo.filename.endswith("task_failed.log"):
-            # The koji logs store this file in two places: 1) in the
-            # directory with the failed architecture and 2) in the parent
-            # directory. We actually want to ignore the one in the parent
-            # directory, since the rest of the information is in the
-            # specific task directory.
-            # The paths look like `kojilogs/noarch-XXXXXX/task_failed.log`
-            # or `kojilogs/noarch-XXXXXX/x86_64-XXXXXX/task_failed.log`
-            path = PurePath(zipinfo.filename)
-            if len(path.parts) <= 3:
-                continue
-
-            # Extract the architecture from the immediate parent path
-            architecture = path.parent.parts[-1].split("-")[0]
-
-            # Open this file and read which log failed.
-            # The string in this log has the format
-            # `see <log> for more information`.
-            # Note: it may sometimes say
-            # `see build.log or root.log for more information`, but in
-            # that situation, we only want to handle build.log (for now),
-            # which means accepting only the first match for the regular
-            # expression.
-            with artifacts_zip.open(zipinfo.filename) as task_failed_log:
-                contents = task_failed_log.read().decode("utf-8")
-                match = FAILURE_LOG_REGEX.search(contents)
-                if not match:
-                    LOG.error(
-                        "task_failed.log does not indicate which log contains the failure."
-                    )
-                    raise SyntaxError(
-                        "task_failed.log does not indicate which log contains the failure."
-                    )
-                failure_log_name = match.group(1)
-
-            failed_arches[architecture] = PurePath(path.parent, failure_log_name)
-
-    if not failed_arches:
-        # No failed task found?
-        raise FileNotFoundError("Could not detect failed architecture.")
-
-    # First check if we only found one failed architecture
-    if len(failed_arches) == 1:
-        failed_arch = list(failed_arches.keys())[0]
-
-    else:
-        # We only want to handle one arch, so we'll check them in order of
-        # "most to least likely for the maintainer to have access to hardware"
-        # This means: x86_64 > aarch64 > ppc64le > s390x
-        if "x86_64" in failed_arches:
-            failed_arch = "x86_64"
-        elif "aarch64" in failed_arches:
-            failed_arch = "aarch64"
-        elif "ppc64le" in failed_arches:
-            failed_arch = "ppc64le"
-        elif "s390x" in failed_arches:
-            failed_arch = "s390x"
-        else:
-            # It should be impossible for us to get "noarch" here, since
-            # the only way that should happen is for a single architecture
-            # build.
-            raise FileNotFoundError("No failed architecture detected.")
-
-    LOG.debug("Failed architecture: %s", failed_arch)
-
-    log_path = failed_arches[failed_arch]
-    LOG.debug("Returning contents of %s", log_path)
-
-    # Return the log as a file-like object with .read() function
-    return artifacts_zip.open(log_path.as_posix())
-
-
-async def check_artifacts_file_size(job):
-    """Method to determine if the artifacts are too large to process"""
-    # First, make sure that the artifacts are of a reasonable size. The
-    # zipped artifact collection will be stored in memory below. The
-    # python-gitlab library doesn't expose a way to check this value directly,
-    # so we need to interact with directly with the headers.
-    artifacts_url = f"{SERVER_CONFIG.gitlab.api_url}/projects/{job.project_id}/jobs/{job.id}/artifacts"  # pylint: disable=line-too-long
-    header_resp = await asyncio.to_thread(
-        requests.head,
-        artifacts_url,
-        allow_redirects=True,
-        headers={"Authorization": f"Bearer {SERVER_CONFIG.gitlab.api_token}"},
-        timeout=(3.07, 5),
-    )
-    content_length = int(header_resp.headers.get("content-length"))
-    LOG.debug(
-        "URL: %s, content-length: %d, max length: %d",
-        artifacts_url,
-        content_length,
-        SERVER_CONFIG.gitlab.max_artifact_size,
-    )
-    return content_length <= SERVER_CONFIG.gitlab.max_artifact_size
 
+async def collect_emoji_task():
+    """Collect emoji feedback.
+    Query only comments created in the last year.
+    """
 
-async def submit_log_to_llm(log):
-    """Stream the log to the LLM for processing"""
-    # TODO: query the LLM with the log contents  # pylint: disable=fixme
-    # This function will be implemented later; right now it does nothing.
-    LOG.debug("Log contents:\n%s", log.read())
-    return ""
+    for instance in SERVER_CONFIG.gitlab.instances.values():
+        LOG.info(
+            "Collect emoji feedback for %s started at %s",
+            instance.url,
+            datetime.datetime.now(datetime.timezone.utc),
+        )
+        await collect_emojis(instance.get_connection(), TimePeriod(weeks=54))
+        LOG.info(
+            "Collect emoji feedback finished at %s",
+            datetime.datetime.now(datetime.timezone.utc),
+        )
 
 
-async def comment_on_mr(merge_request_id: int, response: str):  # pylint: disable=unused-argument
-    """Add the Log Detective response as a comment to the merge request"""
-    # TODO: Implement this  # pylint: disable=fixme
-    pass  # pylint: disable=unnecessary-pass
+async def schedule_collect_emojis_task():
+    """Schedule the collect_emojis_task to run on a configured interval"""
+    while True:
+        seconds_until_run = SERVER_CONFIG.general.collect_emojis_interval
+        LOG.info("Collect emojis in %d seconds", seconds_until_run)
+        await asyncio.sleep(seconds_until_run)
+
+        try:
+            await collect_emoji_task()
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            LOG.exception("Error in collect_emoji_task: %s", e)