logdetective 2.13.0__py3-none-any.whl → 3.1.0__py3-none-any.whl

logdetective/constants.py

@@ -4,7 +4,7 @@ in prompts.yaml instead.
 """
 
 # pylint: disable=line-too-long
-DEFAULT_ADVISOR = "fedora-copr/Mistral-7B-Instruct-v0.3-GGUF"
+DEFAULT_ADVISOR = "fedora-copr/granite-3.2-8b-instruct-GGUF"
 
 PROMPT_TEMPLATE = """
 Given following log snippets, and nothing else, explain what failure, if any, occured during build of this package.

logdetective/logdetective.py

@@ -41,31 +41,15 @@ def setup_args():
     )
     parser.add_argument(
         "-F",
-        "--filename_suffix",
+        "--filename-suffix",
         help="Suffix of the model file name to be retrieved from Hugging Face.\
                             Makes sense only if the model is specified with Hugging Face name.",
         default="Q4_K.gguf",
     )
     parser.add_argument("-n", "--no-stream", action="store_true")
-    parser.add_argument(
-        "-S",
-        "--summarizer",
-        type=str,
-        default="drain",
-        help="DISABLED: LLM summarization option was removed. \
-                Argument is kept for backward compatibility only.",
-    )
-    parser.add_argument(
-        "-N",
-        "--n_lines",
-        type=int,
-        default=None,
-        help="DISABLED: LLM summarization option was removed. \
-                Argument is kept for backward compatibility only.",
-    )
     parser.add_argument(
         "-C",
-        "--n_clusters",
+        "--n-clusters",
         type=int,
         default=8,
         help="Number of clusters for Drain to organize log chunks into.\
@@ -75,10 +59,18 @@ def setup_args():
     parser.add_argument("-q", "--quiet", action="store_true")
     parser.add_argument(
         "--prompts",
+        "--prompts-config",
         type=str,
         default=f"{os.path.dirname(__file__)}/prompts.yml",
         help="Path to prompt configuration file.",
     )
+    parser.add_argument(
+        "--prompt-templates",
+        type=str,
+        default=f"{os.path.dirname(__file__)}/prompts",
+        help="Path to prompt template dir. Prompts must be valid Jinja templates, \
+              and system prompts must include field `system_time`.",
+    )
     parser.add_argument(
         "--temperature",
         type=float,
@@ -86,7 +78,7 @@ def setup_args():
         help="Temperature for inference.",
     )
     parser.add_argument(
-        "--skip_snippets",
+        "--skip-snippets",
         type=str,
         default=f"{os.path.dirname(__file__)}/skip_snippets.yml",
         help="Path to patterns for skipping snippets.",
@@ -105,10 +97,6 @@ async def run():  # pylint: disable=too-many-statements,too-many-locals,too-many
         sys.stderr.write("Error: --quiet and --verbose is mutually exclusive.\n")
         sys.exit(2)
 
-    # Emit warning about use of discontinued args
-    if args.n_lines or args.summarizer != "drain":
-        LOG.warning("LLM based summarization was removed. Drain will be used instead.")
-
     # Logging facility setup
     log_level = logging.INFO
     if args.verbose >= 1:
@@ -117,7 +105,7 @@ async def run():  # pylint: disable=too-many-statements,too-many-locals,too-many
         log_level = 0
 
     # Get prompts configuration
-    prompts_configuration = load_prompts(args.prompts)
+    prompts_configuration = load_prompts(args.prompts, args.prompt_templates)
 
     logging.basicConfig(stream=sys.stdout)
     LOG.setLevel(log_level)

logdetective/models.py

@@ -21,26 +21,7 @@ class PromptConfig(BaseModel):
     snippet_system_prompt: str = DEFAULT_SYSTEM_PROMPT
     staged_system_prompt: str = DEFAULT_SYSTEM_PROMPT
 
-    def __init__(self, data: Optional[dict] = None):
-        super().__init__()
-        if data is None:
-            return
-        self.prompt_template = data.get("prompt_template", PROMPT_TEMPLATE)
-        self.snippet_prompt_template = data.get(
-            "snippet_prompt_template", SNIPPET_PROMPT_TEMPLATE
-        )
-        self.prompt_template_staged = data.get(
-            "prompt_template_staged", PROMPT_TEMPLATE_STAGED
-        )
-        self.default_system_prompt = data.get(
-            "default_system_prompt", DEFAULT_SYSTEM_PROMPT
-        )
-        self.snippet_system_prompt = data.get(
-            "snippet_system_prompt", DEFAULT_SYSTEM_PROMPT
-        )
-        self.staged_system_prompt = data.get(
-            "staged_system_prompt", DEFAULT_SYSTEM_PROMPT
-        )
+    references: Optional[list[dict[str, str]]] = None
 
 
 class SkipSnippets(BaseModel):

logdetective/prompts/message_template.j2

@@ -0,0 +1,2 @@
+Snippets:
+{}

logdetective/prompts/snippet_message_template.j2

@@ -0,0 +1,2 @@
+Snippet:
+{}

logdetective/prompts/snippet_system_prompt.j2

@@ -0,0 +1,38 @@
+System time: {{ system_time }}
+
+You are a highly capable expert system specialized in packaging and delivery of software using RPM,
+within the RHEL ecosystem. Your purpose is to help package maintainers diagnose and resolve package build failures.
+You are truthful, concise, and helpful.
+
+## Input processing
+
+You will work with snippets of logs produced during package build.
+These snippets were extracted using data mining algorithm, and may not contain information
+useful for diagnosing the root cause. Snippets without useful information must be disregarded.
+
+## Analysis procedure
+
+1. Provide the snippet with a short explanation.
+2. If the snippet doesn't contain useful information, indicate the fact with a short sentence.
+
+## Examples:
+
+User: "Snippet: RPM build errors:"
+Assistant: "Errors occurred during package build.
+---
+User: "Snippet: Copr build error: Build failed"
+Assistant: "The build in Copr has failed."
+---
+User: "Snippet: /bin/tar: Removing leading `/' from member names"
+Assistant: "This snippet is irrelevant."
+---
+
+{% if references %}
+## References:
+
+When necessary, suggest resources that may be helpful to user.
+
+    {% for reference in references %}
+    * {{ reference.name }} : {{ reference.link }}
+    {% endfor %}
+{% endif %}

logdetective/prompts/staged_message_template.j2

@@ -0,0 +1,2 @@
+Snippets:
+{}

logdetective/prompts/staged_system_prompt.j2

@@ -0,0 +1,45 @@
+System time: {{ system_time }}
+
+You are a highly capable expert system specialized in packaging and delivery of software using RPM,
+within the RHEL ecosystem. Your purpose is to help package maintainers diagnose and resolve package build failures.
+You are truthful, concise, and helpful.
+
+## Input processing
+
+You will work with snippets of logs produced during package build.
+These snippets were extracted using data mining algorithm, and may not contain information
+useful for diagnosing the root cause. Snippets without useful information must be disregarded.
+
+## Analysis procedure
+
+Analyzed snippets are a format of [X] : [Y], where [X] is a log snippet, and [Y] is the explanation.
+Do not reanalyze the raw log [X].
+
+Snippets are delimited with '================'.
+
+1. Analyze individual snippets, unless they already have analysis attached.
+2. Disregard snippets that do not contain useful information.
+3. Using information from all snippets provide explanation of the issue.
+4. (Optional) Recommend a solution for the package maintainer, only if the cause is clear.
+
+## Examples:
+
+User: "
+    Snippets:
+        ================
+        Snippet No. 1 at line #452:
+            [error: command 'gcc' failed: No such file or directory]: [`gcc` compiler is not available in the build environment]
+        ================
+        Snippet No. 2 at line #452:
+            [Copr build error: Build failed]: [Package build in Copr failed]"
+Assistant: "Package build in Copr failed due to missing `gcc` compiler. Ensure that all build requirements are correctly specified in the spec file."
+
+{% if references %}
+## References:
+
+When necessary, suggest resources that may be helpful to user.
+
+    {% for reference in references %}
+    * {{ reference.name }} : {{ reference.link }}
+    {% endfor %}
+{% endif %}

logdetective/prompts/system_prompt.j2

@@ -0,0 +1,57 @@
+System time: {{ system_time }}
+
+You are a highly capable expert system specialized in packaging and delivery of software using RPM,
+within the RHEL ecosystem. Your purpose is to help package maintainers diagnose and resolve package build failures.
+You are truthful, concise, and helpful.
+
+## Input processing
+
+You will work with snippets of logs produced during a failed package build.
+These snippets were extracted using data mining algorithm, and may not contain information
+useful for diagnosing the root cause. Snippets without useful information must be disregarded.
+General error messages, such as failure of commands used during build, are expected.
+
+## Temporal Logic and Causality
+
+Log snippets are typically provided in chronological order. When analyzing multiple snippets
+the first significant error in the log is usually the root cause.
+
+An error occurring at line #500 cannot be caused by an error occurring at line #1000.
+
+Subsequent errors are often side effects of the initial failure. Focus your diagnosis on the primary trigger.
+
+## Analysis procedure
+
+Snippets are provided in order of appearance in the original log, with attached line number,
+and are delimited with '================'.
+Avoid generic or boilerplate recommendations (e.g., "check the logs," "ensure dependencies are met").
+If a specific root cause is identified, the recommendation must directly address that cause.
+
+1. Analyze individual snippets. Do not quote analyzed snippets.
+2. Disregard snippets that do not contain useful information.
+3. Using information from all snippets provide explanation of the issue. Be as specific as possible.
+4. (Optional) Recommend a solution for the package maintainer, only if the cause is clear.
+
+## Examples:
+
+User: "
+    Snippets:
+    Snippet No. 1 at line #452:
+
+    error: command 'gcc' failed: No such file or directory
+    ================
+    Snippet No. 2 at line #560:
+
+    Copr build error: Build failed
+    ================"
+Assistant: "Package build in Copr failed due to missing `gcc` compiler. Ensure that all build requirements are correctly specified in the spec file."
+
+{% if references %}
+## References:
+
+When necessary, suggest resources that may be helpful to user.
+
+    {% for reference in references %}
+    * {{ reference.name }} : {{ reference.link }}
+    {% endfor %}
+{% endif %}

logdetective/prompts.py

@@ -0,0 +1,87 @@
+from datetime import datetime, timezone
+from typing import Optional
+from jinja2 import Environment, FileSystemLoader, Template
+
+from logdetective.models import PromptConfig
+
+
+class PromptManager:  # pylint: disable=too-many-instance-attributes
+    """Manages prompts defined as jinja templates"""
+    _tmp_env: Environment
+
+    # Templates for system prompts
+    _default_system_prompt_template: Template
+    _snippet_system_prompt_template: Template
+    _staged_system_prompt_template: Template
+
+    # Templates for messages
+    default_message_template: Template
+    snippet_message_template: Template
+    staged_message_template: Template
+
+    _references: Optional[list[dict[str, str]]] = None
+
+    def __init__(
+        self, prompts_path: str, prompts_configuration: Optional[PromptConfig] = None
+    ) -> None:
+        self._tmp_env = Environment(loader=FileSystemLoader(prompts_path))
+
+        self._default_system_prompt_template = self._tmp_env.get_template(
+            "system_prompt.j2"
+        )
+        self._snippet_system_prompt_template = self._tmp_env.get_template(
+            "snippet_system_prompt.j2"
+        )
+        self._staged_system_prompt_template = self._tmp_env.get_template(
+            "staged_system_prompt.j2"
+        )
+
+        self.default_message_template = self._tmp_env.get_template(
+            "message_template.j2"
+        )
+        self.snippet_message_template = self._tmp_env.get_template(
+            "snippet_message_template.j2"
+        )
+        self.staged_message_template = self._tmp_env.get_template(
+            "staged_message_template.j2"
+        )
+
+        if prompts_configuration:
+            self._references = prompts_configuration.references
+
+    # To maintain backward compatibility with `logdetective.models.PromptConfig`
+    @property
+    def default_system_prompt(self) -> str:
+        """Render system prompt from a template"""
+        return self._default_system_prompt_template.render(
+            system_time=datetime.now(timezone.utc), references=self._references
+        )
+
+    @property
+    def snippet_system_prompt(self) -> str:
+        """Render system prompt from a template"""
+        return self._snippet_system_prompt_template.render(
+            system_time=datetime.now(timezone.utc), references=self._references
+        )
+
+    @property
+    def staged_system_prompt(self) -> str:
+        """Render system prompt from a template"""
+        return self._staged_system_prompt_template.render(
+            system_time=datetime.now(timezone.utc), references=self._references
+        )
+
+    @property
+    def prompt_template(self) -> str:
+        """Render message prompt from the template"""
+        return self.default_message_template.render()
+
+    @property
+    def snippet_prompt_template(self) -> str:
+        """Render message prompt from the template"""
+        return self.snippet_message_template.render()
+
+    @property
+    def prompt_template_staged(self) -> str:
+        """Render message prompt from the template"""
+        return self.staged_message_template.render()

logdetective/prompts.yml

@@ -88,3 +88,10 @@ staged_system_prompt: |
   You never speculate about package being built or fabricate information.
   If you do not know the answer, you acknowledge the fact and end your response.
   Your responses must be as short as possible.
+
+# Optional references, to be used when constructing prompt from Jinja template
+# references:
+#   - name: Fedora Packaging Guidelines
+#     link: https://docs.fedoraproject.org/en-US/packaging-guidelines/
+#   - name: Mock user documentation
+#     link: https://rpm-software-management.github.io/mock/

logdetective/server/config.py

@@ -61,7 +61,8 @@ def get_openai_api_client(inference_config: InferenceConfig):
 
 
 SERVER_CONFIG_PATH = os.environ.get("LOGDETECTIVE_SERVER_CONF", None)
-SERVER_PROMPT_PATH = os.environ.get("LOGDETECTIVE_PROMPTS", None)
+SERVER_PROMPT_CONF_PATH = os.environ.get("LOGDETECTIVE_PROMPTS", None)
+SERVER_PROMPT_PATH = os.environ.get("LOGDETECTIVE_PROMPT_TEMPLATES", None)
 # The default location for skip patterns is in the same directory
 # as logdetective __init__.py file.
 SERVER_SKIP_PATTERNS_PATH = os.environ.get(
@@ -70,7 +71,7 @@ SERVER_SKIP_PATTERNS_PATH = os.environ.get(
 )
 
 SERVER_CONFIG = load_server_config(SERVER_CONFIG_PATH)
-PROMPT_CONFIG = load_prompts(SERVER_PROMPT_PATH)
+PROMPT_CONFIG = load_prompts(SERVER_PROMPT_CONF_PATH, SERVER_PROMPT_PATH)
 SKIP_SNIPPETS_CONFIG = load_skip_snippet_patterns(SERVER_SKIP_PATTERNS_PATH)
 
 LOG = get_log(SERVER_CONFIG)

logdetective/server/database/models/metrics.py

@@ -314,10 +314,13 @@ class AnalyzeRequestMetrics(Base):
                         "time_range"
                     ),
                     (
-                        func.avg(
-                            func.extract(  # pylint: disable=not-callable
-                                "epoch", cls.response_sent_at - cls.request_received_at
-                            )
+                        func.coalesce(
+                            func.avg(
+                                func.extract(  # pylint: disable=not-callable
+                                    "epoch", cls.response_sent_at - cls.request_received_at
+                                )
+                            ),
+                            0
                         )
                     ).label("average_response_seconds"),
                 )

logdetective/server/metric.py

@@ -1,17 +1,24 @@
 import inspect
+from collections import defaultdict
 import datetime
-
-from typing import Optional, Union
+from typing import Optional, Union, Dict
 from functools import wraps
 
 import aiohttp
-
+import numpy
 from starlette.responses import StreamingResponse
-from logdetective.server import models
+
 from logdetective.remote_log import RemoteLog
 from logdetective.server.config import LOG
 from logdetective.server.compressors import LLMResponseCompressor, RemoteLogCompressor
-from logdetective.server.database.models import EndpointType, AnalyzeRequestMetrics
+from logdetective.server.models import (
+    TimePeriod,
+    MetricTimeSeries,
+    StagedResponse,
+    Response,
+    Explanation,
+)
+from logdetective.server.database.models import EndpointType, AnalyzeRequestMetrics, Reactions
 from logdetective.server.exceptions import LogDetectiveMetricsError
 
 
@@ -47,7 +54,7 @@ async def add_new_metrics(
 
 async def update_metrics(
     metrics_id: int,
-    response: Union[models.Response, models.StagedResponse, StreamingResponse],
+    response: Union[Response, StagedResponse, StreamingResponse],
     sent_at: Optional[datetime.datetime] = None,
 ) -> None:
     """Update a database metric entry for a received request,
@@ -71,7 +78,7 @@ async def update_metrics(
     )
     response_length = None
     if hasattr(response, "explanation") and isinstance(
-        response.explanation, models.Explanation
+        response.explanation, Explanation
     ):
         response_length = len(response.explanation.text)
     response_certainty = (
@@ -125,3 +132,189 @@ def track_request(name=None):
         raise NotImplementedError("An async coroutine is needed")
 
     return decorator
+
+
+# TODO: Refactor aggregation to use database operations, instead of timestamp formatting  # pylint: disable=fixme
+class TimeDefinition:
+    """Define time format details, given a time period."""
+
+    def __init__(self, time_period: TimePeriod):
+        self.time_period = time_period
+        self.days_diff = time_period.get_time_period().days
+        if self.time_period.hours:
+            self._time_format = "%Y-%m-%d %H"
+            self._time_delta = datetime.timedelta(hours=1)
+        elif self.time_period.days:
+            self._time_format = "%Y-%m-%d"
+            self._time_delta = datetime.timedelta(days=1)
+        elif self.time_period.weeks:
+            self._time_format = "%Y-%m-%d"
+            self._time_delta = datetime.timedelta(weeks=1)
+
+    @property
+    def time_format(self):
+        # pylint: disable=missing-function-docstring
+        return self._time_format
+
+    @property
+    def time_delta(self):
+        # pylint: disable=missing-function-docstring
+        return self._time_delta
+
+
+def create_time_series_arrays(
+    values_dict: dict[datetime.datetime, int],
+) -> tuple[list, list]:
+    """Create time series arrays from a dictionary of values.
+
+    This function generates two aligned lists:
+    1. An array of timestamps from start_time to end_time
+    2. A corresponding array of values for each timestamp
+
+    Args:
+        values_dict: Dictionary mapping timestamps to their respective values
+    Returns:
+        A tuple containing:
+            - list: Array of timestamps
+            - list: Array of corresponding values
+    """
+
+    timestamps = []
+    values = []
+
+    for timestamp, count in values_dict.items():
+        timestamps.append(timestamp)
+        values.append(count)
+
+    return timestamps, numpy.nan_to_num(values).tolist()
+
+
+async def requests_per_time(
+    period_of_time: TimePeriod,
+    endpoint: EndpointType = EndpointType.ANALYZE,
+    end_time: Optional[datetime.datetime] = None,
+) -> MetricTimeSeries:
+    """
+    Get request counts over a specified time period.
+
+    The time intervals are determined by the provided TimePeriod object, which defines
+    the granularity.
+
+    Args:
+        period_of_time: A TimePeriod object that defines the time period and interval
+                        for the analysis (e.g., hourly, daily, weekly)
+        endpoint: One of the API endpoints
+        end_time: The end time for the analysis period. If None, defaults to the current
+                  UTC time
+
+    Returns:
+        A dictionary with timestamps and associated number of requests
+    """
+    end_time = end_time or datetime.datetime.now(datetime.timezone.utc)
+    start_time = period_of_time.get_period_start_time(end_time)
+    time_def = TimeDefinition(period_of_time)
+    requests_counts = await AnalyzeRequestMetrics.get_requests_in_period(
+        start_time, end_time, time_def.time_format, endpoint
+    )
+    timestamps, counts = create_time_series_arrays(requests_counts)
+
+    return MetricTimeSeries(metric="requests", timestamps=timestamps, values=counts)
+
+
+async def average_time_per_responses(
+    period_of_time: TimePeriod,
+    endpoint: EndpointType = EndpointType.ANALYZE,
+    end_time: Optional[datetime.datetime] = None,
+) -> MetricTimeSeries:
+    """
+    Get average response time and length over a specified time period.
+
+    The time intervals are determined by the provided TimePeriod object, which defines
+    the granularity.
+
+    Args:
+        period_of_time: A TimePeriod object that defines the time period and interval
+                        for the analysis (e.g., hourly, daily, weekly)
+        endpoint: One of the API endpoints
+        end_time: The end time for the analysis period. If None, defaults to the current
+                  UTC time
+
+    Returns:
+        A dictionary of timestamps and average response times
+    """
+    end_time = end_time or datetime.datetime.now(datetime.timezone.utc)
+    start_time = period_of_time.get_period_start_time(end_time)
+    time_def = TimeDefinition(period_of_time)
+    responses_average_time = (
+        await AnalyzeRequestMetrics.get_responses_average_time_in_period(
+            start_time, end_time, time_def.time_format, endpoint
+        )
+    )
+    timestamps, average_time = create_time_series_arrays(
+        responses_average_time,
+    )
+
+    return MetricTimeSeries(metric="avg_response_time", timestamps=timestamps, values=average_time)
+
+
+async def _collect_emoji_data(
+    start_time: datetime.datetime, time_def: TimeDefinition
+) -> Dict[str, Dict[str, list]]:
+    """Collect and organize emoji feedback data
+
+    For each reaction type, a dictionary is created with time stamps
+    as keys, and aggregate counts as values.
+    """
+    reactions = await Reactions.get_since(start_time)
+    reaction_values: defaultdict[str, Dict] = defaultdict(lambda: defaultdict(int))
+
+    for comment_timestamp, reaction in reactions:
+        formatted_timestamp = comment_timestamp.strptime(
+            comment_timestamp.strftime(time_def.time_format), time_def.time_format
+        )
+
+        reaction_values[reaction.reaction_type][formatted_timestamp] += reaction.count
+
+    reaction_time_series = {
+        reaction_type: {
+            "timestamps": reaction_data.keys(),
+            "values": reaction_data.values(),
+        }
+        for reaction_type, reaction_data in reaction_values.items()
+    }
+
+    return reaction_time_series
+
+
+async def emojis_per_time(
+    period_of_time: TimePeriod,
+    end_time: Optional[datetime.datetime] = None,
+) -> list[MetricTimeSeries]:
+    """
+    Retrieve data of emoji feedback over time.
+
+    The time intervals are determined by the provided TimePeriod object, which defines
+    the granularity.
+
+    Args:
+        period_of_time: A TimePeriod object that defines the time period and interval
+                        for the analysis (e.g., hourly, daily, weekly)
+        end_time: The end time for the analysis period. If None, defaults to the current
+                  UTC time
+
+    Returns:
+        A list of `MetricTimeSeries` objects
+    """
+    time_def = TimeDefinition(period_of_time)
+    end_time = end_time or datetime.datetime.now(datetime.timezone.utc)
+    start_time = period_of_time.get_period_start_time(end_time)
+    reactions_values_dict = await _collect_emoji_data(start_time, time_def)
+
+    reaction_values: list[MetricTimeSeries] = []
+    for reaction, time_series in reactions_values_dict.items():
+        reaction_values.append(
+            MetricTimeSeries(
+                metric=f"emoji_{reaction}",
+                timestamps=time_series["timestamps"],
+                values=time_series["values"]))
+    return reaction_values

logdetective/server/models.py

@@ -401,3 +401,15 @@ class TimePeriod(BaseModel):
         if time.tzinfo is None:
             time = time.replace(tzinfo=datetime.timezone.utc)
         return time - self.get_time_period()
+
+
+class MetricTimeSeries(BaseModel):
+    """Recorded values of given metric"""
+    metric: str
+    timestamps: List[datetime.datetime]
+    values: List[float]
+
+
+class MetricResponse(BaseModel):
+    """Requested metrics"""
+    time_series: List[MetricTimeSeries]