logdetective 0.5.6__py3-none-any.whl → 0.5.8__py3-none-any.whl

logdetective/server/database/models.py

@@ -2,7 +2,17 @@ import enum
 import datetime
 
 from typing import Optional
-from sqlalchemy import Column, Integer, Float, DateTime, String, Enum
+from sqlalchemy import (
+    Column,
+    Integer,
+    Float,
+    DateTime,
+    String,
+    Enum,
+    func,
+    select,
+    distinct,
+)
 
 from logdetective.server.database.base import Base, transaction
 
@@ -86,3 +96,91 @@ class AnalyzeRequestMetrics(Base):
             metrics.response_length = response_length
             metrics.response_certainty = response_certainty
             session.add(metrics)
+
+    @classmethod
+    def _get_requests_by_time_for_postgres(
+        cls, start_time, end_time, time_format, endpoint
+    ):
+        """func.to_char is PostgreSQL specific.
+        Let's unit tests replace this function with the SQLite version.
+        """
+        if time_format == "%Y-%m-%d":
+            pgsql_time_format = "YYYY-MM-DD"
+        else:
+            pgsql_time_format = "YYYY-MM-DD HH24"
+
+        requests_by_time_format = (
+            select(
+                cls.id,
+                func.to_char(cls.request_received_at, pgsql_time_format).label(
+                    "time_format"
+                ),
+            )
+            .filter(cls.request_received_at.between(start_time, end_time))
+            .filter(cls.endpoint == endpoint)
+            .cte("requests_by_time_format")
+        )
+        return requests_by_time_format
+
+    @classmethod
+    def _get_requests_by_time_for_sqllite(
+        cls, start_time, end_time, time_format, endpoint
+    ):
+        """func.strftime is SQLite specific.
+        Use this function in unit test using flexmock:
+
+        flexmock(AnalyzeRequestMetrics).should_receive("_get_requests_by_time_for_postgres")
+        .replace_with(AnalyzeRequestMetrics._get_requests_by_time_for_sqllite)
+        """
+        requests_by_time_format = (
+            select(
+                cls.id,
+                func.strftime(time_format, cls.request_received_at).label(
+                    "time_format"
+                ),
+            )
+            .filter(cls.request_received_at.between(start_time, end_time))
+            .filter(cls.endpoint == endpoint)
+            .cte("requests_by_time_format")
+        )
+        return requests_by_time_format
+
+    @classmethod
+    def get_requests_in_period(
+        cls,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+        time_format: str,
+        endpoint: Optional[EndpointType] = EndpointType.ANALYZE,
+    ) -> dict[datetime.datetime, int]:
+        """
+        Get a dictionary with request counts grouped by time units within a specified period.
+
+        Args:
+            start_time (datetime): The start of the time period to query
+            end_time (datetime): The end of the time period to query
+            time_format (str): The strftime format string to format timestamps (e.g., '%Y-%m-%d')
+            endpoint (EndpointType): The analyze API endpoint to query
+
+        Returns:
+            dict[datetime, int]: A dictionary mapping datetime objects to request counts
+        """
+        with transaction(commit=False) as session:
+            requests_by_time_format = cls._get_requests_by_time_for_postgres(
+                start_time, end_time, time_format, endpoint
+            )
+
+            count_requests_by_time_format = select(
+                requests_by_time_format.c.time_format,
+                func.count(distinct(requests_by_time_format.c.id)),  # pylint: disable=not-callable
+            ).group_by("time_format")
+
+            counts = session.execute(count_requests_by_time_format)
+            results = counts.fetchall()
+
+            # Convert results to a dictionary with proper datetime keys
+            counts_dict = {
+                datetime.datetime.strptime(r[0], time_format): r[1] for r in results
+            }
+
+            return counts_dict

logdetective/server/models.py

@@ -1,6 +1,8 @@
+import datetime
 from logging import BASIC_FORMAT
 from typing import List, Dict, Optional, Literal
-from pydantic import BaseModel, Field
+
+from pydantic import BaseModel, Field, model_validator, field_validator
 
 
 class BuildLog(BaseModel):
@@ -54,6 +56,7 @@ class AnalyzedSnippet(BaseModel):
     text: original snippet text
     line_number: location of snippet in original log
     """
+
     explanation: Explanation
     text: str
     line_number: int
@@ -195,3 +198,75 @@ class Config(BaseModel):
         self.extractor = ExtractorConfig(data.get("extractor"))
         self.gitlab = GitLabConfig(data.get("gitlab"))
         self.general = GeneralConfig(data.get("general"))
+
+
+class TimePeriod(BaseModel):
+    """Specification for a period of time.
+
+    If no indication is given
+    it falls back to a 2 days period of time.
+
+    Can't be smaller than a hour"""
+
+    weeks: Optional[int] = None
+    days: Optional[int] = None
+    hours: Optional[int] = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def check_exclusive_fields(cls, data):
+        """ Check that only one key between weeks, days and hours is defined"""
+        if isinstance(data, dict):
+            how_many_fields = sum(
+                1
+                for field in ["weeks", "days", "hours"]
+                if field in data and data[field] is not None
+            )
+
+            if how_many_fields == 0:
+                data["days"] = 2  # by default fallback to a 2 days period
+
+            if how_many_fields > 1:
+                raise ValueError("Only one of months, weeks, days, or hours can be set")
+
+        return data
+
+    @field_validator("weeks", "days", "hours")
+    @classmethod
+    def check_positive(cls, v):
+        """Check that the given value is positive"""
+        if v is not None and v <= 0:
+            raise ValueError("Time period must be positive")
+        return v
+
+    def get_time_period(self) -> datetime.timedelta:
+        """Get the period of time represented by this input model.
+
+        Returns:
+            datetime.timedelta: The time period as a timedelta object.
+        """
+        delta = None
+        if self.weeks:
+            delta = datetime.timedelta(weeks=self.weeks)
+        elif self.days:
+            delta = datetime.timedelta(days=self.days)
+        elif self.hours:
+            delta = datetime.timedelta(hours=self.hours)
+        return delta
+
+    def get_period_start_time(
+        self, end_time: datetime.datetime = None
+    ) -> datetime.datetime:
+        """Calculate the start time of this period based on the end time.
+
+        Args:
+            end_time (datetime.datetime, optional): The end time of the period.
+                Defaults to current UTC time if not provided.
+
+        Returns:
+            datetime.datetime: The start time of the period.
+        """
+        time = end_time or datetime.datetime.now(datetime.timezone.utc)
+        if end_time.tzinfo is None:
+            end_time = end_time.replace(tzinfo=datetime.timezone.utc)
+        return time - self.get_time_period()

logdetective/server/plot.py

@@ -0,0 +1,206 @@
+import datetime
+from typing import Optional
+
+import numpy
+import matplotlib
+import matplotlib.figure
+import matplotlib.pyplot
+
+from logdetective.server import models
+from logdetective.server.database.models import AnalyzeRequestMetrics, EndpointType
+
+
+class Definition:
+    """Define plot details, given a time period."""
+
+    def __init__(self, time_period: models.TimePeriod):
+        self.time_period = time_period
+        self.days_diff = time_period.get_time_period().days
+        if self.time_period.hours:
+            self._freq = "H"
+            self._time_format = "%Y-%m-%d %H"
+            self._locator = matplotlib.dates.HourLocator(interval=2)
+            self._time_unit = "hour"
+            self._time_delta = datetime.timedelta(hours=1)
+        elif self.time_period.days:
+            self._freq = "D"
+            self._time_format = "%Y-%m-%d"
+            self._locator = matplotlib.dates.DayLocator(interval=1)
+            self._time_unit = "day"
+            self._time_delta = datetime.timedelta(days=1)
+        elif self.time_period.weeks:
+            self._freq = "W"
+            self._time_format = "%Y-%m-%d"
+            self._locator = matplotlib.dates.WeekdayLocator(interval=1)
+            self._time_unit = "week"
+            self._time_delta = datetime.timedelta(weeks=1)
+
+    @property
+    def freq(self):
+        # pylint: disable=missing-function-docstring
+        return self._freq
+
+    @property
+    def time_format(self):
+        # pylint: disable=missing-function-docstring
+        return self._time_format
+
+    @property
+    def locator(self):
+        # pylint: disable=missing-function-docstring
+        return self._locator
+
+    @property
+    def time_unit(self):
+        # pylint: disable=missing-function-docstring
+        return self._time_unit
+
+    @property
+    def time_delta(self):
+        # pylint: disable=missing-function-docstring
+        return self._time_delta
+
+
+def create_time_series_arrays(
+    counts_dict: dict[datetime.datetime, int],
+    start_time: datetime.datetime,
+    end_time: datetime.datetime,
+    time_delta: datetime.timedelta,
+    time_format: str,
+) -> tuple[numpy.ndarray, numpy.ndarray]:
+    """Create time series arrays from a dictionary of counts.
+
+    This function generates two aligned numpy arrays:
+    1. An array of timestamps from start_time to end_time
+    2. A corresponding array of counts for each timestamp
+
+    The timestamps are truncated to the precision specified by time_format.
+    If a timestamp in counts_dict matches a generated timestamp, its count is used;
+    otherwise, the count defaults to zero.
+
+    Args:
+        counts_dict: Dictionary mapping timestamps to their respective counts
+        start_time: The starting timestamp of the time series
+        end_time: The ending timestamp of the time series
+        time_delta: The time interval between consecutive timestamps
+        time_format: String format for datetime truncation (e.g., '%Y-%m-%d %H:%M')
+
+    Returns:
+        A tuple containing:
+            - numpy.ndarray: Array of timestamps
+            - numpy.ndarray: Array of corresponding counts
+    """
+    num_intervals = int((end_time - start_time) / time_delta) + 1
+
+    timestamps = numpy.array(
+        [
+            datetime.datetime.strptime(
+                (start_time + i * time_delta).strftime(format=time_format), time_format
+            )
+            for i in range(num_intervals)
+        ]
+    )
+    counts = numpy.zeros(num_intervals, dtype=int)
+
+    timestamp_to_index = {timestamp: i for i, timestamp in enumerate(timestamps)}
+
+    for timestamp, count in counts_dict.items():
+        if timestamp in timestamp_to_index:
+            counts[timestamp_to_index[timestamp]] = count
+
+    return timestamps, counts
+
+
+def _add_bar_chart_for_requests_count(
+    ax1: matplotlib.figure.Axes,
+    plot_def: Definition,
+    timestamps: numpy.array,
+    counts: numpy.array,
+) -> None:
+    """Add a bar chart for requests count (axes 1)"""
+    bar_width = (
+        0.8 * plot_def.time_delta.total_seconds() / 86400
+    )  # Convert to days for matplotlib
+    ax1.bar(
+        timestamps,
+        counts,
+        width=bar_width,
+        alpha=0.7,
+        color="skyblue",
+        label="Requests",
+    )
+    ax1.set_xlabel("Time")
+    ax1.set_ylabel("Requests", color="blue")
+    ax1.tick_params(axis="y", labelcolor="blue")
+
+    ax1.xaxis.set_major_formatter(matplotlib.dates.DateFormatter(plot_def.time_format))
+    ax1.xaxis.set_major_locator(plot_def.locator)
+
+    matplotlib.pyplot.xticks(rotation=45)
+
+    ax1.grid(True, alpha=0.3)
+
+
+def _add_cumulative_line_for_requests_count(
+    ax2: matplotlib.figure.Axes, timestamps: numpy.array, counts: numpy.array
+) -> None:
+    """Add cumulative line on secondary y-axis"""
+    cumulative = numpy.cumsum(counts)
+    ax2.plot(timestamps, cumulative, "r-", linewidth=2, label="Cumulative")
+    ax2.set_ylabel("Cumulative Requests", color="red")
+    ax2.tick_params(axis="y", labelcolor="red")
+
+
+def requests_per_time(
+    period_of_time: models.TimePeriod,
+    endpoint: EndpointType = EndpointType.ANALYZE,
+    end_time: Optional[datetime.datetime] = None,
+) -> matplotlib.figure.Figure:
+    """
+    Generate a visualization of request counts over a specified time period.
+
+    This function creates a dual-axis plot showing:
+    1. A bar chart of request counts per time interval
+    2. A line chart showing the cumulative request count
+
+    The time intervals are determined by the provided TimePeriod object, which defines
+    the granularity and formatting of the time axis.
+
+    Args:
+        period_of_time: A TimePeriod object that defines the time period and interval
+                        for the analysis (e.g., hourly, daily, weekly)
+        endpoint: One of the API endpoints
+        end_time: The end time for the analysis period. If None, defaults to the current
+                  UTC time
+
+    Returns:
+        A matplotlib Figure object containing the generated visualization
+    """
+    end_time = end_time or datetime.datetime.now(datetime.timezone.utc)
+    start_time = period_of_time.get_period_start_time(end_time)
+    plot_def = Definition(period_of_time)
+    requests_counts = AnalyzeRequestMetrics.get_requests_in_period(
+        start_time, end_time, plot_def.time_format, endpoint
+    )
+    timestamps, counts = create_time_series_arrays(
+        requests_counts, start_time, end_time, plot_def.time_delta, plot_def.time_format
+    )
+
+    fig, ax1 = matplotlib.pyplot.subplots(figsize=(12, 6))
+    _add_bar_chart_for_requests_count(ax1, plot_def, timestamps, counts)
+
+    ax2 = ax1.twinx()
+    _add_cumulative_line_for_requests_count(ax2, timestamps, counts)
+
+    matplotlib.pyplot.title(
+        f"Requests received for API {endpoint} ({start_time.strftime(plot_def.time_format)} "
+        f"to {end_time.strftime(plot_def.time_format)})"
+    )
+
+    lines1, labels1 = ax1.get_legend_handles_labels()
+    lines2, labels2 = ax2.get_legend_handles_labels()
+    ax1.legend(lines1 + lines2, labels1 + labels2, loc="center")
+
+    matplotlib.pyplot.tight_layout()
+
+    return fig

logdetective/server/server.py

@@ -6,8 +6,11 @@ import zipfile
 from pathlib import Path, PurePath
 from tempfile import TemporaryFile
 from typing import List, Annotated, Tuple, Dict, Any
+from io import BytesIO
 
 
+import matplotlib
+import matplotlib.pyplot
 from fastapi import FastAPI, HTTPException, BackgroundTasks, Depends, Header
 
 from fastapi.responses import StreamingResponse
@@ -39,7 +42,10 @@ from logdetective.server.models import (
     StagedResponse,
     Explanation,
     AnalyzedSnippet,
+    TimePeriod,
 )
+from logdetective.server import plot
+from logdetective.server.database.models import EndpointType
 
 LLM_CPP_SERVER_TIMEOUT = os.environ.get("LLAMA_CPP_SERVER_TIMEOUT", 600)
 LOG_SOURCE_REQUEST_TIMEOUT = os.environ.get("LOG_SOURCE_REQUEST_TIMEOUT", 60)
@@ -494,7 +500,7 @@ async def retrieve_and_preprocess_koji_logs(job: gitlab.v4.objects.ProjectJob):
     tempfile.seek(0)
 
     failed_arches = {}
-    artifacts_zip = zipfile.ZipFile(tempfile, mode="r")
+    artifacts_zip = zipfile.ZipFile(tempfile, mode="r")  # pylint: disable=consider-using-with
     for zipinfo in artifacts_zip.infolist():
         if zipinfo.filename.endswith("task_failed.log"):
             # The koji logs store this file in two places: 1) in the
@@ -653,3 +659,35 @@ async def generate_mr_comment(
     )
 
     return content
+
+
+def _svg_figure_response(fig: matplotlib.figure.Figure):
+    """Create a response with the given svg figure."""
+    buf = BytesIO()
+    fig.savefig(buf, format="svg", bbox_inches="tight")
+    matplotlib.pyplot.close(fig)
+
+    buf.seek(0)
+    return StreamingResponse(
+        buf,
+        media_type="image/svg+xml",
+        headers={"Content-Disposition": "inline; filename=plot.svg"},
+    )
+
+
+@app.get("/metrics/analyze/requests", response_class=StreamingResponse)
+async def show_analyze_requests(period_since_now: TimePeriod = Depends(TimePeriod)):
+    """Show statistics for the requests received in the given period of time
+    for the /analyze API endpoint."""
+    fig = plot.requests_per_time(period_since_now, EndpointType.ANALYZE)
+    return _svg_figure_response(fig)
+
+
+@app.get("/metrics/analyze/staged/requests", response_class=StreamingResponse)
+async def show_analyze_staged_requests(
+    period_since_now: TimePeriod = Depends(TimePeriod),
+):
+    """Show statistics for the requests received in the given period of time
+    for the /analyze/staged API endpoint."""
+    fig = plot.requests_per_time(period_since_now, EndpointType.ANALYZE_STAGED)
+    return _svg_figure_response(fig)

{logdetective-0.5.6.dist-info → logdetective-0.5.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: logdetective
-Version: 0.5.6
+Version: 0.5.8
 Summary: Log using LLM AI to search for build/test failures and provide ideas for fixing these.
 License: Apache-2.0
 Author: Jiri Podivin
@@ -19,18 +19,21 @@ Classifier: Topic :: Internet :: Log Analysis
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development :: Debuggers
 Provides-Extra: server
-Requires-Dist: alembic (>=1.13.3,<2.0.0) ; extra == "server"
+Provides-Extra: server-testing
+Requires-Dist: alembic (>=1.13.3,<2.0.0) ; extra == "server" or extra == "server-testing"
 Requires-Dist: drain3 (>=0.9.11,<0.10.0)
-Requires-Dist: fastapi (>=0.111.1) ; extra == "server"
+Requires-Dist: fastapi (>=0.111.1) ; extra == "server" or extra == "server-testing"
 Requires-Dist: huggingface-hub (>0.23.2)
 Requires-Dist: llama-cpp-python (>0.2.56,!=0.2.86)
+Requires-Dist: matplotlib (>=3.8.4,<4.0.0) ; extra == "server" or extra == "server-testing"
 Requires-Dist: numpy (>=1.26.0)
 Requires-Dist: psycopg2 (>=2.9.9,<3.0.0) ; extra == "server"
-Requires-Dist: pydantic (>=2.8.2,<3.0.0) ; extra == "server"
+Requires-Dist: psycopg2-binary (>=2.9.9,<3.0.0) ; extra == "server-testing"
+Requires-Dist: pydantic (>=2.8.2,<3.0.0) ; extra == "server" or extra == "server-testing"
 Requires-Dist: python-gitlab (>=4.4.0)
-Requires-Dist: pyyaml (>=6.0.1,<7.0.0) ; extra == "server"
+Requires-Dist: pyyaml (>=6.0.1,<7.0.0) ; extra == "server" or extra == "server-testing"
 Requires-Dist: requests (>0.2.31)
-Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "server"
+Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "server" or extra == "server-testing"
 Project-URL: homepage, https://github.com/fedora-copr/logdetective
 Project-URL: issues, https://github.com/fedora-copr/logdetective/issues
 Description-Content-Type: text/markdown
@@ -331,6 +334,34 @@ HTTPS certificate generated through:
 certbot certonly --standalone -d logdetective01.fedorainfracloud.org
 ```
 
+Querying statistics
+-------------------
+
+You can retrieve statistics about server requests over a specified time period
+using either the `curl` command or the `http` command (provided by the `httpie` package).
+
+When no time period is specified, the query defaults to the last 2 days:
+
+```
+http GET "localhost:8080/metrics/analyze/requests" > /tmp/plot.svg
+curl "localhost:8080/metrics/analyze/staged/requests" > /tmp/plot.svg
+```
+
+You can specify the time period in hours, days, or weeks.
+The time period:
+
+ - cannot be less than one hour
+ - cannot be negative
+ - ends at the current time (when the query is made)
+ - starts at the specified time interval before the current time.
+
+Examples:
+
+```
+http GET "localhost:8080/metrics/analyze/requests?hours=5" > /tmp/plot_hours.svg
+http GET "localhost:8080/metrics/analyze/requests?days=5" > /tmp/plot_days.svg
+http GET "localhost:8080/metrics/analyze/requests?weeks=5" > /tmp/plot_weeks.svg
+```
 
 License
 -------

{logdetective-0.5.6.dist-info → logdetective-0.5.8.dist-info}/RECORD

@@ -6,15 +6,16 @@ logdetective/logdetective.py,sha256=KN0KASW63VAnrjVeXK5AO0ob-vSexutTyeg1fd4uj70,
 logdetective/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 logdetective/server/database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 logdetective/server/database/base.py,sha256=oMJUvbWeapIUP-8Cf_DR9ptFg8CsYeaBAIjOVEzx8SM,1668
-logdetective/server/database/models.py,sha256=8jW4k03Kny_3ld35214hcjYoJqlBvQIr4LH9mfQukXw,2750
+logdetective/server/database/models.py,sha256=arIahOCT-hTmh904DXrWSkH7rlo13Ppu-OO80huX5Dc,6118
 logdetective/server/metric.py,sha256=VYMifrfIhcqgyu6YYN0c1nt8fC1iJ2_LCB7Bh2AheoE,2679
-logdetective/server/models.py,sha256=f42yMMMMfTdTN4KWpPUfaEoaiE9rhqltA0dQNKGOB2w,5660
-logdetective/server/server.py,sha256=0q7e6eMbFJTP8s41jnHQt_H1IJsoPV40ke2JZ9kjeJk,23116
+logdetective/server/models.py,sha256=cf1ngu_-19rP_i49s5cEwIzh6SfL_ZpVy4EykCpfWck,8076
+logdetective/server/plot.py,sha256=3o-CNHjel04ekpwSB4ckV7dbiF663cfPkimQ0aP9U_8,7073
+logdetective/server/server.py,sha256=ALVD9cwG4d8OQXfOPbRtt4y0nlh2C-8jP8pQeaufC3g,24533
 logdetective/server/templates/gitlab_comment.md.j2,sha256=kheTkhQ-LfuFkr8av-Mw2a-9VYEUbDTLwaa-CKI6OkI,1622
 logdetective/server/utils.py,sha256=OFvhttjv3yp8kfim5_s4mNG8ly21qyILxE0o3DcVVKg,1340
 logdetective/utils.py,sha256=eudens1_T6iTtYhyzoYCpwuWgFHUMDSt6eWnrAB-mAI,6188
-logdetective-0.5.6.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-logdetective-0.5.6.dist-info/METADATA,sha256=u57V9mWwvJxVAJR01_nUFol-8f10tSmkoLm7q94KXuo,12830
-logdetective-0.5.6.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-logdetective-0.5.6.dist-info/entry_points.txt,sha256=3K_vXja6PmcA8sNdUi63WdImeiNhVZcEGPTaoJmltfA,63
-logdetective-0.5.6.dist-info/RECORD,,
+logdetective-0.5.8.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+logdetective-0.5.8.dist-info/METADATA,sha256=dUCiCPfW8ILyshanWpb_zHdm9q3LIBYKZUWyfQWqsCA,14115
+logdetective-0.5.8.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+logdetective-0.5.8.dist-info/entry_points.txt,sha256=3K_vXja6PmcA8sNdUi63WdImeiNhVZcEGPTaoJmltfA,63
+logdetective-0.5.8.dist-info/RECORD,,