logdetective 2.5.0__tar.gz → 2.7.0__tar.gz

{logdetective-2.5.0 → logdetective-2.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: logdetective
-Version: 2.5.0
+Version: 2.7.0
 Summary: Log using LLM AI to search for build/test failures and provide ideas for fixing these.
 License: Apache-2.0
 License-File: LICENSE
@@ -128,6 +128,20 @@ Note that streaming with some models (notably Meta-Llama-3) is broken and can be
 
     logdetective https://example.com/logs.txt --model QuantFactory/Meta-Llama-3-8B-Instruct-GGUF --filename_suffix Q5_K_M.gguf --no-stream
 
+Choice of LLM
+-------------
+
+While Log Detective is compatible with a wide range of LLMs, it does require an instruction tuned model to function properly.
+
+Whether or not the model has been trained to work with instructions can be determined by examining the model card, or simply by checking if it has `instruct` in its name.
+
+When deployed as a server, Log Detective uses `/chat/completions` API as defined by OpenAI. The API must support both `system` and `user` roles, in order to properly work with a system prompt.
+
+Configuration fields `system_role` and `user_role` can be used to set role names for APIs with non-standard roles.
+
+> **Note:**
+> In cases when no system role is available, it is possible to set both fields to the same value. This will concatenate system and standard prompt.
+> This may have negative impact coherence of response.
 
 Real Example
 ------------

{logdetective-2.5.0 → logdetective-2.7.0}/README.md

@@ -73,6 +73,20 @@ Note that streaming with some models (notably Meta-Llama-3) is broken and can be
 
     logdetective https://example.com/logs.txt --model QuantFactory/Meta-Llama-3-8B-Instruct-GGUF --filename_suffix Q5_K_M.gguf --no-stream
 
+Choice of LLM
+-------------
+
+While Log Detective is compatible with a wide range of LLMs, it does require an instruction tuned model to function properly.
+
+Whether or not the model has been trained to work with instructions can be determined by examining the model card, or simply by checking if it has `instruct` in its name.
+
+When deployed as a server, Log Detective uses `/chat/completions` API as defined by OpenAI. The API must support both `system` and `user` roles, in order to properly work with a system prompt.
+
+Configuration fields `system_role` and `user_role` can be used to set role names for APIs with non-standard roles.
+
+> **Note:**
+> In cases when no system role is available, it is possible to set both fields to the same value. This will concatenate system and standard prompt.
+> This may have negative impact coherence of response.
 
 Real Example
 ------------

{logdetective-2.5.0 → logdetective-2.7.0}/logdetective/logdetective.py

@@ -174,11 +174,6 @@ async def run():  # pylint: disable=too-many-statements,too-many-locals,too-many
     log_summary = format_snippets(log_summary)
     LOG.info("Log summary: \n %s", log_summary)
 
-    prompt = (
-        f"{prompts_configuration.default_system_prompt}\n"
-        f"{prompts_configuration.prompt_template}"
-    )
-
     stream = True
     if args.no_stream:
         stream = False
@@ -186,30 +181,38 @@ async def run():  # pylint: disable=too-many-statements,too-many-locals,too-many
         log_summary,
         model,
         stream,
-        prompt_template=prompt,
+        prompt_templates=prompts_configuration,
         temperature=args.temperature,
     )
     probs = []
     print("Explanation:")
     # We need to extract top token probability from the response
-    # CreateCompletionResponse structure of llama-cpp-python.
+    # CreateChatCompletionResponse structure of llama-cpp-python.
     # `compute_certainty` function expects list of dictionaries with form
     # { 'logprob': <float> } as expected from the OpenAI API.
 
     if args.no_stream:
-        print(response["choices"][0]["text"])
+        print(response["choices"][0]["message"]["content"])
         probs = [
-            {"logprob": e} for e in response["choices"][0]["logprobs"]["token_logprobs"]
+            {"logprob": e["logprob"]} for e in response["choices"][0]["logprobs"]["content"]
         ]
 
     else:
         # Stream the output
         for chunk in response:
+            # What might happen, is that first (or possibly any other) chunk may not contain
+            # fields choices[0].delta.content or choices[0].logprobs -> if so, we just skip it
+            if any([
+                'content' not in chunk["choices"][0]["delta"],
+                'logprobs' not in chunk["choices"][0]
+            ]):
+                continue
+
             if isinstance(chunk["choices"][0]["logprobs"], dict):
                 probs.append(
-                    {"logprob": chunk["choices"][0]["logprobs"]["token_logprobs"][0]}
+                    {"logprob": chunk["choices"][0]["logprobs"]["content"][0]["logprob"]}
                 )
-            delta = chunk["choices"][0]["text"]
+            delta = chunk["choices"][0]["delta"]["content"]
             print(delta, end="", flush=True)
     certainty = compute_certainty(probs)
 

{logdetective-2.5.0 → logdetective-2.7.0}/logdetective/prompts-summary-first.yml

@@ -18,5 +18,3 @@ prompt_template: |
   Snippets:
 
   {}
-
-  Analysis:

{logdetective-2.5.0 → logdetective-2.7.0}/logdetective/prompts.yml

@@ -19,7 +19,6 @@ prompt_template: |
 
   {}
 
-  Analysis:
 
 snippet_prompt_template: |
   Analyse following RPM build log snippet. Describe contents accurately, without speculation or suggestions for resolution
@@ -30,7 +29,6 @@ snippet_prompt_template: |
 
   {}
 
-  Analysis:
 
 prompt_template_staged: |
   Given following log snippets, their explanation, and nothing else, explain what failure, if any, occurred during build of this package.
@@ -47,7 +45,6 @@ prompt_template_staged: |
 
   {}
 
-  Analysis:
 
 # System prompts
 # System prompts are meant to serve as general guide for model behavior,

{logdetective-2.5.0 → logdetective-2.7.0}/logdetective/server/database/base.py

@@ -1,6 +1,6 @@
 from os import getenv
 from contextlib import asynccontextmanager
-from sqlalchemy.orm import declarative_base
+from sqlalchemy.orm import DeclarativeBase
 from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
 from logdetective import logger
 
@@ -24,7 +24,10 @@ sqlalchemy_echo = getenv("SQLALCHEMY_ECHO", "False").lower() in (
 )
 engine = create_async_engine(get_pg_url(), echo=sqlalchemy_echo)
 SessionFactory = async_sessionmaker(autoflush=True, bind=engine)  # pylint: disable=invalid-name
-Base = declarative_base()
+
+
+class Base(DeclarativeBase):
+    """Declarative base class for all ORM models."""
 
 
 @asynccontextmanager

{logdetective-2.5.0 → logdetective-2.7.0}/logdetective/server/database/models/__init__.py

@@ -1,4 +1,3 @@
-from logdetective.server.database.base import Base
 from logdetective.server.database.models.merge_request_jobs import (
     Forge,
     GitlabMergeRequestJobs,
@@ -18,8 +17,9 @@ from logdetective.server.database.models.exceptions import (
     KojiTaskAnalysisTimeoutError,
 )
 
+# pylint: disable=undefined-all-variable
+
 __all__ = [
-    Base.__name__,
     GitlabMergeRequestJobs.__name__,
     Comments.__name__,
     Reactions.__name__,

{logdetective-2.5.0 → logdetective-2.7.0}/logdetective/server/database/models/koji.py

@@ -1,6 +1,9 @@
+from __future__ import annotations
+from typing import Optional
 from datetime import datetime, timedelta, timezone
-from sqlalchemy import Column, BigInteger, DateTime, ForeignKey, Integer, String, select
-from sqlalchemy.orm import relationship
+from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, String, select
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
 from sqlalchemy.exc import OperationalError
 import backoff
 
@@ -21,25 +24,28 @@ class KojiTaskAnalysis(Base):
 
     __tablename__ = "koji_task_analysis"
 
-    id = Column(Integer, primary_key=True)
-    koji_instance = Column(String(255), nullable=False, index=True)
-    task_id = Column(BigInteger, nullable=False, index=True, unique=True)
-    log_file_name = Column(String(255), nullable=False, index=True)
-    request_received_at = Column(
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    koji_instance: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
+    task_id: Mapped[int] = mapped_column(BigInteger, nullable=False, index=True, unique=True)
+    log_file_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
+    request_received_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True),
         nullable=False,
         index=True,
         default=datetime.now(timezone.utc),
         comment="Timestamp when the request was received",
     )
-    response_id = Column(
+    response_id: Mapped[Optional[int]] = mapped_column(
         Integer,
         ForeignKey("analyze_request_metrics.id"),
         nullable=True,
         index=False,
         comment="The id of the analyze request metrics for this task",
     )
-    response = relationship("AnalyzeRequestMetrics")
+    response: Mapped[Optional["AnalyzeRequestMetrics"]] = relationship(
+        "AnalyzeRequestMetrics",
+        back_populates="koji_tasks"
+    )
 
     @classmethod
     @backoff.on_exception(backoff.expo, OperationalError, max_tries=DB_MAX_RETRIES)

{logdetective-2.5.0 → logdetective-2.7.0}/logdetective/server/database/models/merge_request_jobs.py

@@ -1,12 +1,12 @@
+from __future__ import annotations
 import enum
 import datetime
-from typing import Optional, List, Tuple, Self
+from typing import Optional, List, Tuple, Self, TYPE_CHECKING
 
 import backoff
 
 from sqlalchemy import (
     Enum,
-    Column,
     BigInteger,
     DateTime,
     String,
@@ -15,12 +15,16 @@ from sqlalchemy import (
     desc,
     select,
 )
-from sqlalchemy.orm import relationship
+from sqlalchemy.orm import Mapped, mapped_column, relationship
 from sqlalchemy.engine import Row
 from sqlalchemy.exc import OperationalError
 from logdetective.server.database.base import Base, transaction, DB_MAX_RETRIES
 
 
+if TYPE_CHECKING:
+    from .metrics import AnalyzeRequestMetrics
+
+
 class Forge(str, enum.Enum):
     """List of forges managed by logdetective"""
 
@@ -35,21 +39,26 @@ class GitlabMergeRequestJobs(Base):
 
     __tablename__ = "gitlab_merge_request_jobs"
 
-    id = Column(BigInteger, primary_key=True)
-    forge = Column(Enum(Forge), nullable=False, index=True, comment="The forge name")
-    project_id = Column(
+    id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
+    forge: Mapped[Forge] = mapped_column(
+        Enum(Forge),
+        nullable=False,
+        index=True,
+        comment="The forge name"
+    )
+    project_id: Mapped[int] = mapped_column(
         BigInteger,
         nullable=False,
         index=True,
         comment="The project gitlab id",
     )
-    mr_iid = Column(
+    mr_iid: Mapped[int] = mapped_column(
         BigInteger,
         nullable=False,
         index=False,
         comment="The merge request gitlab iid",
     )
-    job_id = Column(
+    job_id: Mapped[int] = mapped_column(
         BigInteger,
         nullable=False,
         index=True,
@@ -63,11 +72,14 @@ class GitlabMergeRequestJobs(Base):
         ),
     )
 
-    comment = relationship(
+    comment: Mapped[List["Comments"]] = relationship(
         "Comments", back_populates="merge_request_job", uselist=False
     )  # 1 comment for 1 job
 
-    request_metrics = relationship("AnalyzeRequestMetrics", back_populates="mr_job")
+    request_metrics: Mapped[List["AnalyzeRequestMetrics"]] = relationship(
+        "AnalyzeRequestMetrics",
+        back_populates="mr_job"
+    )
 
     @classmethod
     @backoff.on_exception(backoff.expo, OperationalError, max_tries=DB_MAX_RETRIES)
@@ -183,8 +195,8 @@ class Comments(Base):
 
     __tablename__ = "comments"
 
-    id = Column(BigInteger, primary_key=True)
-    merge_request_job_id = Column(
+    id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
+    merge_request_job_id: Mapped[int] = mapped_column(
         BigInteger,
         ForeignKey("gitlab_merge_request_jobs.id"),
         nullable=False,
@@ -192,14 +204,19 @@ class Comments(Base):
         index=True,
         comment="The associated merge request job (db) id",
     )
-    forge = Column(Enum(Forge), nullable=False, index=True, comment="The forge name")
-    comment_id = Column(
+    forge: Mapped[Forge] = mapped_column(
+        Enum(Forge),
+        nullable=False,
+        index=True,
+        comment="The forge name"
+    )
+    comment_id: Mapped[str] = mapped_column(
         String(50),  # e.g. 'd5a3ff139356ce33e37e73add446f16869741b50'
         nullable=False,
         index=True,
         comment="The comment gitlab id",
     )
-    created_at = Column(
+    created_at: Mapped[datetime.datetime] = mapped_column(
         DateTime(timezone=True),
         nullable=False,
         comment="Timestamp when the comment was created",
@@ -209,8 +226,11 @@ class Comments(Base):
         UniqueConstraint("forge", "comment_id", name="uix_forge_comment_id"),
     )
 
-    merge_request_job = relationship("GitlabMergeRequestJobs", back_populates="comment")
-    reactions = relationship("Reactions", back_populates="comment")
+    merge_request_job: Mapped["GitlabMergeRequestJobs"] = relationship(
+        "GitlabMergeRequestJobs",
+        back_populates="comment"
+    )
+    reactions: Mapped[list["Reactions"]] = relationship("Reactions", back_populates="comment")
 
     @classmethod
     @backoff.on_exception(backoff.expo, OperationalError, max_tries=DB_MAX_RETRIES)
@@ -408,20 +428,20 @@ class Reactions(Base):
 
     __tablename__ = "reactions"
 
-    id = Column(BigInteger, primary_key=True)
-    comment_id = Column(
+    id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
+    comment_id: Mapped[int] = mapped_column(
         BigInteger,
         ForeignKey("comments.id"),
         nullable=False,
         index=True,
         comment="The associated comment (db) id",
     )
-    reaction_type = Column(
+    reaction_type: Mapped[str] = mapped_column(
         String(127),  # e.g. 'thumbs-up'
         nullable=False,
         comment="The type of reaction",
     )
-    count = Column(
+    count: Mapped[int] = mapped_column(
         BigInteger,
         nullable=False,
         comment="The number of reactions, of this type, given in the comment",
@@ -431,7 +451,7 @@ class Reactions(Base):
         UniqueConstraint("comment_id", "reaction_type", name="uix_comment_reaction"),
     )
 
-    comment = relationship("Comments", back_populates="reactions")
+    comment: Mapped["Comments"] = relationship("Comments", back_populates="reactions")
 
     @classmethod
     @backoff.on_exception(backoff.expo, OperationalError, max_tries=DB_MAX_RETRIES)

{logdetective-2.5.0 → logdetective-2.7.0}/logdetective/server/database/models/metrics.py

@@ -1,12 +1,12 @@
+from __future__ import annotations
 import io
 import enum
 import datetime
-from typing import Optional, List, Self, Tuple
+from typing import Optional, List, Self, Tuple, TYPE_CHECKING
 
 import backoff
 
 from sqlalchemy import (
-    Column,
     Integer,
     Float,
     DateTime,
@@ -17,7 +17,7 @@ from sqlalchemy import (
     ForeignKey,
     LargeBinary,
 )
-from sqlalchemy.orm import relationship, aliased
+from sqlalchemy.orm import Mapped, mapped_column, relationship, aliased
 from sqlalchemy.exc import OperationalError
 
 from logdetective.server.database.base import Base, transaction, DB_MAX_RETRIES
@@ -27,6 +27,10 @@ from logdetective.server.database.models.merge_request_jobs import (
 )
 
 
+if TYPE_CHECKING:
+    from .koji import KojiTaskAnalysis
+
+
 class EndpointType(enum.Enum):
     """Different analyze endpoints"""
 
@@ -42,45 +46,45 @@ class AnalyzeRequestMetrics(Base):
 
     __tablename__ = "analyze_request_metrics"
 
-    id = Column(Integer, primary_key=True)
-    endpoint = Column(
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    endpoint: Mapped[EndpointType] = mapped_column(
         Enum(EndpointType),
         nullable=False,
         index=True,
         comment="The service endpoint that was called",
     )
-    request_received_at = Column(
+    request_received_at: Mapped[datetime.datetime] = mapped_column(
         DateTime(timezone=True),
         nullable=False,
         index=True,
         default=datetime.datetime.now(datetime.timezone.utc),
         comment="Timestamp when the request was received",
     )
-    compressed_log = Column(
+    compressed_log: Mapped[bytes] = mapped_column(
         LargeBinary(length=314572800),  # 300MB limit (300 * 1024 * 1024)
         nullable=False,
         index=False,
         comment="Log processed, saved in a zip format",
     )
-    compressed_response = Column(
+    compressed_response: Mapped[Optional[bytes]] = mapped_column(
         LargeBinary(length=314572800),  # 300MB limit (300 * 1024 * 1024)
         nullable=True,
         index=False,
         comment="Given response (with explanation and snippets) saved in a zip format",
     )
-    response_sent_at = Column(
+    response_sent_at: Mapped[Optional[datetime.datetime]] = mapped_column(
         DateTime(timezone=True),
         nullable=True,
         comment="Timestamp when the response was sent back",
     )
-    response_length = Column(
+    response_length: Mapped[Optional[int]] = mapped_column(
         Integer, nullable=True, comment="Length of the response in chars"
     )
-    response_certainty = Column(
+    response_certainty: Mapped[Optional[float]] = mapped_column(
         Float, nullable=True, comment="Certainty for generated response"
     )
 
-    merge_request_job_id = Column(
+    merge_request_job_id: Mapped[Optional[int]] = mapped_column(
         Integer,
         ForeignKey("gitlab_merge_request_jobs.id"),
         nullable=True,
@@ -88,7 +92,15 @@ class AnalyzeRequestMetrics(Base):
         comment="Is this an analyze request coming from a merge request?",
     )
 
-    mr_job = relationship("GitlabMergeRequestJobs", back_populates="request_metrics")
+    mr_job: Mapped[Optional["GitlabMergeRequestJobs"]] = relationship(
+        "GitlabMergeRequestJobs",
+        back_populates="request_metrics"
+    )
+
+    koji_tasks: Mapped[List["KojiTaskAnalysis"]] = relationship(
+        "KojiTaskAnalysis",
+        back_populates="response"
+    )
 
     @classmethod
     @backoff.on_exception(backoff.expo, OperationalError, max_tries=DB_MAX_RETRIES)

{logdetective-2.5.0 → logdetective-2.7.0}/logdetective/server/templates/gitlab_full_comment.md.j2

@@ -57,15 +57,15 @@ Please know that the explanation was provided by AI and may be incorrect.
         </li>
     </ul>
 </details>
----
-This comment was created by [Log Detective][log-detective].
+
+<hr>
+
+This comment was created by <a href="https://logdetective.com">Log Detective</a>.
 Was the provided feedback accurate and helpful?
 <br>
 Please vote with :thumbsup:
 or :thumbsdown: to help us improve.
 <br>
-<i>If this Log Detective report contains harmful content, please use the
-    [Gitlab reporting feature for harmful content](https://docs.gitlab.com/user/report_abuse/)
-and contact the [Log Detective developers](https://github.com/fedora-copr/logdetective/issues).</i>
-[log-detective]: https://log-detective.com/
-[contact]: https://github.com/fedora-copr
+<i>If this Log Detective report contains harmful content,
+please use the <a href="https://docs.gitlab.com/user/report_abuse/">Gitlab reporting feature for harmful content</a>
+and contact the <a href="https://github.com/fedora-copr/logdetective/issues">Log Detective developers</a>.</i>

{logdetective-2.5.0 → logdetective-2.7.0}/logdetective/server/templates/gitlab_short_comment.md.j2

@@ -46,15 +46,15 @@ Please know that the explanation was provided by AI and may be incorrect.
         </li>
     </ul>
 </details>
----
-This comment was created by [Log Detective][log-detective].
+
+<hr>
+
+This comment was created by <a href="https://logdetective.com">Log Detective</a>.
 Was the provided feedback accurate and helpful?
 <br>
 Please vote with :thumbsup:
 or :thumbsdown: to help us improve.
 <br>
-<i>If this Log Detective report contains harmful content, please use the
-    [Gitlab reporting feature for harmful content](https://docs.gitlab.com/user/report_abuse/)
-and contact the [Log Detective developers](https://github.com/fedora-copr/logdetective/issues).</i>
-[log-detective]: https://log-detective.com/
-[contact]: https://github.com/fedora-copr
+<i>If this Log Detective report contains harmful content,
+please use the <a href="https://docs.gitlab.com/user/report_abuse/">Gitlab reporting feature for harmful content</a>
+and contact the <a href="https://github.com/fedora-copr/logdetective/issues">Log Detective developers</a>.</i>

{logdetective-2.5.0 → logdetective-2.7.0}/logdetective/utils.py

@@ -8,7 +8,11 @@ import aiohttp
 import numpy as np
 import yaml
 
-from llama_cpp import Llama, CreateCompletionResponse, CreateCompletionStreamResponse
+from llama_cpp import (
+    Llama,
+    CreateChatCompletionResponse,
+    CreateChatCompletionStreamResponse,
+)
 from logdetective.constants import SNIPPET_DELIMITER
 from logdetective.models import PromptConfig, SkipSnippets
 from logdetective.remote_log import RemoteLog
@@ -123,8 +127,8 @@ def compute_certainty(probs: List[Dict]) -> float:
 
 
 def process_log(
-    log: str, model: Llama, stream: bool, prompt_template: str, temperature: float
-) -> CreateCompletionResponse | Iterator[CreateCompletionStreamResponse]:
+    log: str, model: Llama, stream: bool, prompt_templates: PromptConfig, temperature: float
+) -> CreateChatCompletionResponse | Iterator[CreateChatCompletionStreamResponse]:
     """Processes a given log using the provided language model and returns its summary.
 
     Args:
@@ -136,11 +140,23 @@ def process_log(
     Returns:
         str: The summary of the given log generated by the language model.
     """
-    response = model(
-        prompt=prompt_template.format(log),
+    messages = [
+        {
+            "role": "system",
+            "content": prompt_templates.default_system_prompt
+        },
+        {
+            "role": "user",
+            "content": prompt_templates.prompt_template.format(log)
+        },
+    ]
+
+    response = model.create_chat_completion(
+        messages=messages,
         stream=stream,
         max_tokens=0,
-        logprobs=1,
+        logprobs=True,
+        top_logprobs=1,
         temperature=temperature,
     )
 

{logdetective-2.5.0 → logdetective-2.7.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "logdetective"
-version = "2.5.0"
+version = "2.7.0"
 description = "Log using LLM AI to search for build/test failures and provide ideas for fixing these."
 authors = ["Jiri Podivin <jpodivin@gmail.com>"]
 license = "Apache-2.0"