logdetective 1.3.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

logdetective/constants.py

@@ -69,6 +69,17 @@ Analysis:
 
 """
 
+DEFAULT_SYSTEM_PROMPT = """
+You are a highly capable large language model based expert system specialized in
+packaging and delivery of software using RPM (RPM Package Manager). Your purpose is to diagnose
+RPM build failures, identifying root causes and proposing solutions if possible.
+You are truthful, concise, and helpful.
+
+You never speculate about package being built or fabricate information.
+If you do not know the answer, you acknowledge the fact and end your response.
+Your responses must be as short as possible.
+"""
+
 SNIPPET_DELIMITER = "================"
 
 DEFAULT_TEMPERATURE = 0.8
@@ -76,3 +87,7 @@ DEFAULT_TEMPERATURE = 0.8
 # Tuning for LLM-as-a-Service
 LLM_DEFAULT_MAX_QUEUE_SIZE = 50
 LLM_DEFAULT_REQUESTS_PER_MINUTE = 60
+
+# Roles for chat API
+SYSTEM_ROLE_DEFAULT = "developer"
+USER_ROLE_DEFAULT = "user"

logdetective/logdetective.py

@@ -149,6 +149,10 @@ async def run():  # pylint: disable=too-many-statements,too-many-locals
     log_summary = format_snippets(log_summary)
     LOG.info("Log summary: \n %s", log_summary)
 
+    prompt = (
+        f"{prompts_configuration.default_system_prompt}\n"
+        f"{prompts_configuration.prompt_template}")
+
     stream = True
     if args.no_stream:
         stream = False
@@ -156,7 +160,7 @@ async def run():  # pylint: disable=too-many-statements,too-many-locals
         log_summary,
         model,
         stream,
-        prompt_template=prompts_configuration.prompt_template,
+        prompt_template=prompt,
         temperature=args.temperature,
     )
     probs = []

logdetective/models.py

@@ -6,6 +6,7 @@ from logdetective.constants import (
     PROMPT_TEMPLATE_STAGED,
     SUMMARIZATION_PROMPT_TEMPLATE,
     SNIPPET_PROMPT_TEMPLATE,
+    DEFAULT_SYSTEM_PROMPT,
 )
 
 
@@ -17,6 +18,10 @@ class PromptConfig(BaseModel):
     snippet_prompt_template: str = SNIPPET_PROMPT_TEMPLATE
     prompt_template_staged: str = PROMPT_TEMPLATE_STAGED
 
+    default_system_prompt: str = DEFAULT_SYSTEM_PROMPT
+    snippet_system_prompt: str = DEFAULT_SYSTEM_PROMPT
+    staged_system_prompt: str = DEFAULT_SYSTEM_PROMPT
+
     def __init__(self, data: Optional[dict] = None):
         super().__init__()
         if data is None:
@@ -31,3 +36,12 @@ class PromptConfig(BaseModel):
         self.prompt_template_staged = data.get(
             "prompt_template_staged", PROMPT_TEMPLATE_STAGED
         )
+        self.default_system_prompt = data.get(
+            "default_system_prompt", DEFAULT_SYSTEM_PROMPT
+        )
+        self.snippet_system_prompt = data.get(
+            "snippet_system_prompt", DEFAULT_SYSTEM_PROMPT
+        )
+        self.staged_system_prompt = data.get(
+            "staged_system_prompt", DEFAULT_SYSTEM_PROMPT
+        )

logdetective/prompts.yml

@@ -59,3 +59,46 @@ prompt_template_staged: |
   {}
 
   Analysis:
+
+# System prompts
+# System prompts are meant to serve as general guide for model behavior,
+# describing role and purpose it is meant to serve.
+# Sample system prompts in this file are intentionally the same,
+# however, in some circumstances it may be beneficial have different
+# system prompts for each sub case. For example when a specialized model is deployed
+# to analyze snippets.
+
+# Default prompt is used by the CLI tool and also for final analysis
+# with /analyze and /analyze/stream API endpoints
+default_system_prompt: |
+  You are a highly capable large language model based expert system specialized in
+  packaging and delivery of software using RPM (RPM Package Manager). Your purpose is to diagnose
+  RPM build failures, identifying root causes and proposing solutions if possible.
+  You are truthful, concise, and helpful.
+
+  You never speculate about package being built or fabricate information.
+  If you do not know the answer, you acknowledge the fact and end your response.
+  Your responses must be as short as possible.
+
+# Snippet system prompt is used for analysis of individual snippets
+snippet_system_prompt: |
+  You are a highly capable large language model based expert system specialized in
+  packaging and delivery of software using RPM (RPM Package Manager). Your purpose is to diagnose
+  RPM build failures, identifying root causes and proposing solutions if possible.
+  You are truthful, concise, and helpful.
+
+  You never speculate about package being built or fabricate information.
+  If you do not know the answer, you acknowledge the fact and end your response.
+  Your responses must be as short as possible.
+
+
+# Staged system prompt is used by /analyze/staged API endpoint
+staged_system_prompt: |
+  You are a highly capable large language model based expert system specialized in
+  packaging and delivery of software using RPM (RPM Package Manager). Your purpose is to diagnose
+  RPM build failures, identifying root causes and proposing solutions if possible.
+  You are truthful, concise, and helpful.
+
+  You never speculate about package being built or fabricate information.
+  If you do not know the answer, you acknowledge the fact and end your response.
+  Your responses must be as short as possible.

logdetective/server/llm.py

@@ -1,7 +1,7 @@
 import os
 import asyncio
 import random
-from typing import List, Tuple, Union
+from typing import List, Tuple, Union, Dict
 
 import backoff
 from fastapi import HTTPException
@@ -14,6 +14,7 @@ from logdetective.constants import SNIPPET_DELIMITER
 from logdetective.extractors import DrainExtractor
 from logdetective.utils import (
     compute_certainty,
+    prompt_to_messages,
 )
 from logdetective.server.config import LOG, SERVER_CONFIG, PROMPT_CONFIG, CLIENT
 from logdetective.server.models import (
@@ -85,7 +86,7 @@ def we_give_up(details: backoff._typing.Details):
     on_giveup=we_give_up,
 )
 async def submit_text(
-    text: str,
+    messages: List[Dict[str, str]],
     inference_cfg: InferenceConfig,
     stream: bool = False,
 ) -> Union[Explanation, AsyncStream[ChatCompletionChunk]]:
@@ -100,12 +101,7 @@ async def submit_text(
 
     async with inference_cfg.get_limiter():
         response = await CLIENT.chat.completions.create(
-            messages=[
-                {
-                    "role": "user",
-                    "content": text,
-                }
-            ],
+            messages=messages,
             max_tokens=inference_cfg.max_tokens,
             logprobs=inference_cfg.log_probs,
             stream=stream,
@@ -136,7 +132,12 @@ async def perform_staged_analysis(log_text: str) -> StagedResponse:
     # Process snippets asynchronously
     awaitables = [
         submit_text(
-            PROMPT_CONFIG.snippet_prompt_template.format(s),
+            prompt_to_messages(
+                PROMPT_CONFIG.snippet_prompt_template.format(s),
+                PROMPT_CONFIG.snippet_system_prompt,
+                SERVER_CONFIG.inference.system_role,
+                SERVER_CONFIG.inference.user_role,
+            ),
             inference_cfg=SERVER_CONFIG.snippet_inference,
         )
         for s in log_summary
@@ -150,9 +151,14 @@ async def perform_staged_analysis(log_text: str) -> StagedResponse:
     final_prompt = PROMPT_CONFIG.prompt_template_staged.format(
         format_analyzed_snippets(analyzed_snippets)
     )
-
-    final_analysis = await submit_text(
+    messages = prompt_to_messages(
         final_prompt,
+        PROMPT_CONFIG.staged_system_prompt,
+        SERVER_CONFIG.inference.system_role,
+        SERVER_CONFIG.inference.user_role,
+    )
+    final_analysis = await submit_text(
+        messages,
         inference_cfg=SERVER_CONFIG.inference,
     )
 

logdetective/server/models.py

@@ -20,6 +20,8 @@ from logdetective.constants import (
     DEFAULT_TEMPERATURE,
     LLM_DEFAULT_MAX_QUEUE_SIZE,
     LLM_DEFAULT_REQUESTS_PER_MINUTE,
+    SYSTEM_ROLE_DEFAULT,
+    USER_ROLE_DEFAULT,
 )
 
 
@@ -143,6 +145,8 @@ class InferenceConfig(BaseModel):  # pylint: disable=too-many-instance-attribute
     temperature: NonNegativeFloat = DEFAULT_TEMPERATURE
     max_queue_size: int = LLM_DEFAULT_MAX_QUEUE_SIZE
     http_timeout: float = 5.0
+    user_role: str = USER_ROLE_DEFAULT
+    system_role: str = SYSTEM_ROLE_DEFAULT
     _http_session: aiohttp.ClientSession = None
     _limiter: AsyncLimiter = AsyncLimiter(LLM_DEFAULT_REQUESTS_PER_MINUTE)
 
@@ -159,7 +163,8 @@ class InferenceConfig(BaseModel):  # pylint: disable=too-many-instance-attribute
         self.model = data.get("model", "default-model")
         self.temperature = data.get("temperature", DEFAULT_TEMPERATURE)
         self.max_queue_size = data.get("max_queue_size", LLM_DEFAULT_MAX_QUEUE_SIZE)
-
+        self.user_role = data.get("user_role", USER_ROLE_DEFAULT)
+        self.system_role = data.get("system_role", SYSTEM_ROLE_DEFAULT)
         self._requests_per_minute = data.get(
             "requests_per_minute", LLM_DEFAULT_REQUESTS_PER_MINUTE
         )

logdetective/server/server.py

@@ -20,6 +20,7 @@ import logdetective.server.database.base
 from logdetective.utils import (
     compute_certainty,
     format_snippets,
+    prompt_to_messages,
 )
 
 from logdetective.server.config import SERVER_CONFIG, PROMPT_CONFIG, LOG
@@ -135,9 +136,14 @@ async def analyze_log(
     log_text = await remote_log.process_url()
     log_summary = mine_logs(log_text)
     log_summary = format_snippets(log_summary)
-
-    response = await submit_text(
+    messages = prompt_to_messages(
         PROMPT_CONFIG.prompt_template.format(log_summary),
+        PROMPT_CONFIG.default_system_prompt,
+        SERVER_CONFIG.inference.system_role,
+        SERVER_CONFIG.inference.user_role,
+    )
+    response = await submit_text(
+        messages,
         inference_cfg=SERVER_CONFIG.inference,
     )
     certainty = 0
@@ -204,10 +210,15 @@ async def analyze_log_stream(
     log_text = await remote_log.process_url()
     log_summary = mine_logs(log_text)
     log_summary = format_snippets(log_summary)
-
+    messages = prompt_to_messages(
+        PROMPT_CONFIG.prompt_template.format(log_summary),
+        PROMPT_CONFIG.default_system_prompt,
+        SERVER_CONFIG.inference.system_role,
+        SERVER_CONFIG.inference.user_role,
+    )
     try:
         stream = submit_text(
-            PROMPT_CONFIG.prompt_template.format(log_summary),
+            messages,
             inference_cfg=SERVER_CONFIG.inference,
             stream=True,
         )

logdetective/utils.py

@@ -195,3 +195,35 @@ def load_prompts(path: str | None) -> PromptConfig:
         except FileNotFoundError:
             print("Prompt configuration file not found, reverting to defaults.")
     return PromptConfig()
+
+
+def prompt_to_messages(
+        user_message: str, system_prompt: str | None = None,
+        system_role: str = "developer", user_role: str = "user") -> List[Dict[str, str]]:
+    """Turn prompt into list of message dictionaries.
+    If `system_role` and `user_role` are the same, only a single message is created,
+    as concatenation of `user_message` and `system_prompt`. This is useful for models which
+    do not have separate system role, such as mistral.
+    """
+
+    if system_role == user_role:
+        messages = [
+            {
+                "role": system_role,
+                "content": f"{system_prompt}\n{user_message}"
+            }
+        ]
+    else:
+
+        messages = [
+            {
+                "role": system_role,
+                "content": system_prompt
+            },
+            {
+                "role": user_role,
+                "content": user_message,
+            }
+        ]
+
+    return messages

{logdetective-1.3.0.dist-info → logdetective-1.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: logdetective
-Version: 1.3.0
+Version: 1.4.0
 Summary: Log using LLM AI to search for build/test failures and provide ideas for fixing these.
 License: Apache-2.0
 Author: Jiri Podivin

{logdetective-1.3.0.dist-info → logdetective-1.4.0.dist-info}/RECORD

@@ -1,12 +1,12 @@
 logdetective/__init__.py,sha256=VqRngDcuFT7JWms8Qc_MsOvajoXVOKPr-S1kqY3Pqhc,59
-logdetective/constants.py,sha256=UmYSutgy8yK-IDMQyXqLtQV-wDserDa4K1GmsMCGXHc,1949
+logdetective/constants.py,sha256=yH2vpYENfJpoYUC6KzzTSRDFOiMmjtRK-GU9vfv8o3o,2560
 logdetective/drain3.ini,sha256=ni91eCT1TwTznZwcqWoOVMQcGEnWhEDNCoTPF7cfGfY,1360
 logdetective/extractors.py,sha256=7ahzWbTtU9MveG1Q7wU9LO8OJgs85X-cHmWltUhCe9M,3491
-logdetective/logdetective.py,sha256=cC2oL4yPNo94AB2nS4v1jpZi-Qo1g0_FEchL_yQL1UU,5832
-logdetective/models.py,sha256=nrGBmMRu8i6UhFflQKAp81Y3Sd_Aaoor0i_yqSJoLT0,1115
+logdetective/logdetective.py,sha256=YwjKSgW2iW4BAxWTOgUbrVbk2Lbv76vOLFTTtuPg3hg,5928
+logdetective/models.py,sha256=0DHCtGUqZzPcduSU4Z5AzuNn3g9XBh6UhLBcA9mDG6M,1653
 logdetective/prompts-summary-first.yml,sha256=3Zfp4NNOfaFYq5xBlBjeQa5PdjYfS4v17OtJqQ-DRpU,821
 logdetective/prompts-summary-only.yml,sha256=8U9AMJV8ePW-0CoXOXlQoO92DAJDeutIT8ntSkkm6W0,470
-logdetective/prompts.yml,sha256=urPKG068TYxi58EicFVUH6FavZq_q36oM1LvfI4ddjg,1729
+logdetective/prompts.yml,sha256=2WGox--VRINjsxaJc7fql-v1sIVYWYZLEjJV_vvBm9o,3969
 logdetective/remote_log.py,sha256=1oeMIdDE_ob_2QrlXYTAA_m-36pNEicXbZwrCyzNgwo,2256
 logdetective/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 logdetective/server/compressors.py,sha256=qzrT-BPSksXY6F2L6ger04GGrgdBsGOfK2YuCFRs0Q4,5427
@@ -18,16 +18,16 @@ logdetective/server/database/models/merge_request_jobs.py,sha256=hw88wV1-3x7i53s
 logdetective/server/database/models/metrics.py,sha256=yl9fS4IPVFWDeFvPAxO6zOVu6oLF319ApvVLAgnD5yU,13928
 logdetective/server/emoji.py,sha256=W1nJLU1UnTG8FGttOs6gC7x3TcjxiBuviXklD9f2Mu8,4370
 logdetective/server/gitlab.py,sha256=wQSlvdWn6XEi1oP6HhI75bIhm6bgdpWr3zu2WXF0_oE,16473
-logdetective/server/llm.py,sha256=GkbOjRRWEbw7EhFRpblalwNbwNVQPTTjrbLOqJXKqy0,5388
+logdetective/server/llm.py,sha256=q9LdoAmsx9MpBjnjLyJ9GBU27jKViTaWbVXyMsmsCI0,5721
 logdetective/server/metric.py,sha256=B3ew_qSmtEMj6xl-FoOtS4F_bkplp-shhtfHF1cG_Io,4010
-logdetective/server/models.py,sha256=Pfvyd8CKlahIWeoVAJlQEt2TiLA5ndHEcigfm6xJwBI,15471
+logdetective/server/models.py,sha256=I45uLnq_zqn_r0FdOdop9zQPbsOWOY_M39NBBOXP134,15738
 logdetective/server/plot.py,sha256=eZs4r9gua-nW3yymSMIz1leL9mb4QKlh6FJZSeOfZ5M,14872
-logdetective/server/server.py,sha256=-JJnHj8fPzx8aCJD3q2wRwidxoHPCmwOP8FTWwc1C14,18386
+logdetective/server/server.py,sha256=V-lSG2cCTxoGwvUc8mEmLQQWS4g_W_dER2o118RufAk,18792
 logdetective/server/templates/gitlab_full_comment.md.j2,sha256=DQZ2WVFedpuXI6znbHIW4wpF9BmFS8FaUkowh8AnGhE,1627
 logdetective/server/templates/gitlab_short_comment.md.j2,sha256=fzScpayv2vpRLczP_0O0YxtA8rsKvR6gSv4ntNdWb98,1443
-logdetective/utils.py,sha256=hdExAC8FtDIxvdgIq-Ro6LVM-JZ-k_UofaMzaDAHvzM,6088
-logdetective-1.3.0.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-logdetective-1.3.0.dist-info/METADATA,sha256=9UIOXKl7Ubj5TCsM2p_enbgDDC80d7uByqwA-VpPFZQ,17709
-logdetective-1.3.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-logdetective-1.3.0.dist-info/entry_points.txt,sha256=3K_vXja6PmcA8sNdUi63WdImeiNhVZcEGPTaoJmltfA,63
-logdetective-1.3.0.dist-info/RECORD,,
+logdetective/utils.py,sha256=4VDghJs6mTz8PjkYaV794LUWWyEqBRddXKkENJzR5n4,7025
+logdetective-1.4.0.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+logdetective-1.4.0.dist-info/METADATA,sha256=UlBDdFwH4kswo5NqhvEWfz7dGcex7aXfFOUt1JI5n4Y,17709
+logdetective-1.4.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+logdetective-1.4.0.dist-info/entry_points.txt,sha256=3K_vXja6PmcA8sNdUi63WdImeiNhVZcEGPTaoJmltfA,63
+logdetective-1.4.0.dist-info/RECORD,,