logdetective 0.5.10__py3-none-any.whl → 0.6.0__py3-none-any.whl

logdetective/server/utils.py

@@ -37,7 +37,7 @@ def get_log(config: Config):
     if getattr(log, "initialized", False):
         return log
 
-    log.setLevel(config.log.level)
+    log.setLevel("DEBUG")
 
     # Drop the default handler, we will create it ourselves
     log.handlers = []
@@ -45,12 +45,14 @@ def get_log(config: Config):
     # STDOUT
     stream_handler = logging.StreamHandler()
     stream_handler.setFormatter(logging.Formatter(config.log.format))
+    stream_handler.setLevel(config.log.level_stream)
     log.addHandler(stream_handler)
 
     # Log to file
     if config.log.path:
         file_handler = logging.FileHandler(config.log.path)
         file_handler.setFormatter(logging.Formatter(config.log.format))
+        file_handler.setLevel(config.log.level_file)
         log.addHandler(file_handler)
 
     log.initialized = True

logdetective/utils.py

@@ -2,8 +2,9 @@ import logging
 import os
 from typing import Iterator, List, Dict, Tuple, Generator
 from urllib.parse import urlparse
+
+import aiohttp
 import numpy as np
-import requests
 import yaml
 
 from llama_cpp import Llama, CreateCompletionResponse, CreateCompletionStreamResponse
@@ -111,25 +112,46 @@ def compute_certainty(probs: List[Dict]) -> float:
 
 
 def process_log(
-    log: str, model: Llama, stream: bool, prompt_template: str
+    log: str, model: Llama, stream: bool, prompt_template: str,
+    temperature: float
 ) -> CreateCompletionResponse | Iterator[CreateCompletionStreamResponse]:
     """Processes a given log using the provided language model and returns its summary.
 
     Args:
         log (str): The input log to be processed.
         model (Llama): The language model used for processing the log.
-
+        stream (bool): Return output as Iterator.
+        prompt_template (str): Which prompt template to use.
+        temperature (float): Temperature parameter for model runtime.
     Returns:
         str: The summary of the given log generated by the language model.
     """
     response = model(
-        prompt=prompt_template.format(log), stream=stream, max_tokens=0, logprobs=1
+        prompt=prompt_template.format(log), stream=stream, max_tokens=0, logprobs=1,
+        temperature=temperature
     )
 
     return response
 
 
-def retrieve_log_content(log_path: str) -> str:
+async def get_url_content(http: aiohttp.ClientSession, url: str, timeout: int) -> str:
+    """validate log url and return log text."""
+    if validate_url(url=url):
+        LOG.debug("process url %s", url)
+        try:
+            response = await http.get(
+                url,
+                timeout=timeout,
+                raise_for_status=True
+            )
+        except aiohttp.ClientResponseError as ex:
+            raise RuntimeError(f"We couldn't obtain the logs: {ex}") from ex
+        return await response.text()
+    LOG.error("Invalid URL received ")
+    raise RuntimeError(f"Invalid log URL: {url}")
+
+
+async def retrieve_log_content(http: aiohttp.ClientSession, log_path: str) -> str:
     """Get content of the file on the log_path path.
     Path is assumed to be valid URL if it has a scheme.
     Otherwise it attempts to pull it from local filesystem."""
@@ -144,7 +166,7 @@ def retrieve_log_content(log_path: str) -> str:
             log = f.read()
 
     else:
-        log = requests.get(log_path, timeout=60).text
+        log = await get_url_content(http, log_path, timeout=60)
 
     return log
 

{logdetective-0.5.10.dist-info → logdetective-0.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: logdetective
-Version: 0.5.10
+Version: 0.6.0
 Summary: Log using LLM AI to search for build/test failures and provide ideas for fixing these.
 License: Apache-2.0
 Author: Jiri Podivin
@@ -20,6 +20,7 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development :: Debuggers
 Provides-Extra: server
 Provides-Extra: server-testing
+Requires-Dist: aiohttp (>=3.7.4)
 Requires-Dist: alembic (>=1.13.3,<2.0.0) ; extra == "server" or extra == "server-testing"
 Requires-Dist: drain3 (>=0.9.11,<0.10.0)
 Requires-Dist: fastapi (>=0.111.1) ; extra == "server" or extra == "server-testing"
@@ -32,7 +33,6 @@ Requires-Dist: psycopg2-binary (>=2.9.9,<3.0.0) ; extra == "server-testing"
 Requires-Dist: pydantic (>=2.8.2,<3.0.0)
 Requires-Dist: python-gitlab (>=4.4.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
-Requires-Dist: requests (>0.2.31)
 Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "server" or extra == "server-testing"
 Project-URL: homepage, https://github.com/fedora-copr/logdetective
 Project-URL: issues, https://github.com/fedora-copr/logdetective/issues
@@ -47,6 +47,8 @@ Log Detective
 
 A Python tool to analyze logs using a Language Model (LLM) and Drain template miner.
 
+Note: if you are looking for code of website logdetective.com it is in [github.com/fedora-copr/logdetective-website](https://github.com/fedora-copr/logdetective-website).
+
 Installation
 ------------
 
@@ -95,6 +97,17 @@ Example you want to use a different model:
     logdetective https://example.com/logs.txt --model https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/Meta-Llama-3-8B-Instruct.Q5_K_S.gguf?download=true
     logdetective https://example.com/logs.txt --model QuantFactory/Meta-Llama-3-8B-Instruct-GGUF
 
+Example of different suffix (useful for models that were quantized)
+
+    logdetective https://kojipkgs.fedoraproject.org//work/tasks/3367/131313367/build.log --model 'fedora-copr/granite-3.2-8b-instruct-GGUF' -F Q4_K.gguf
+
+Example of altered prompts:
+
+     cp ~/.local/lib/python3.13/site-packages/logdetective/prompts.yml ~/my-prompts.yml
+     vi ~/my-prompts.yml # edit the prompts there to better fit your needs
+     logdetective https://kojipkgs.fedoraproject.org//work/tasks/3367/131313367/build.log --prompts ~/my-prompts.yml
+
+
 Note that streaming with some models (notably Meta-Llama-3 is broken) is broken and can be workarounded by `no-stream` option:
 
     logdetective https://example.com/logs.txt --model QuantFactory/Meta-Llama-3-8B-Instruct-GGUF --no-stream
@@ -337,11 +350,23 @@ certbot certonly --standalone -d logdetective01.fedorainfracloud.org
 Querying statistics
 -------------------
 
-You can retrieve statistics about server requests over a specified time period
-using either the `curl` command or the `http` command (provided by the `httpie` package).
+You can retrieve statistics about server requests and responses over a specified time period
+using either a browser, the `curl` or the `http` command (provided by the `httpie` package).
 
 When no time period is specified, the query defaults to the last 2 days:
 
+You can view requests and responses statistics
+ - for the `/analyze` endpoint at http://localhost:8080/metrics/analyze
+ - for the `/analyze/staged` endpoint at http://localhost:8080/metrics/analyze/staged.
+
+You can retrieve single svg images at the following endpoints:
+ - `/metrics/analyze/requests`
+ - `/metrics/analyze/responses`
+ - `/metrics/analyze/staged/requests`
+ - `/metrics/analyze/stages/responses`
+
+Examples:
+
 ```
 http GET "localhost:8080/metrics/analyze/requests" > /tmp/plot.svg
 curl "localhost:8080/metrics/analyze/staged/requests" > /tmp/plot.svg
@@ -349,7 +374,6 @@ curl "localhost:8080/metrics/analyze/staged/requests" > /tmp/plot.svg
 
 You can specify the time period in hours, days, or weeks.
 The time period:
-
  - cannot be less than one hour
  - cannot be negative
  - ends at the current time (when the query is made)

logdetective-0.6.0.dist-info/RECORD

@@ -0,0 +1,24 @@
+logdetective/__init__.py,sha256=VqRngDcuFT7JWms8Qc_MsOvajoXVOKPr-S1kqY3Pqhc,59
+logdetective/constants.py,sha256=A5PzeqlQqDbBS_kzP2hl-lhJ0lCEqdbvW3CaQUYVxjw,1849
+logdetective/drain3.ini,sha256=ni91eCT1TwTznZwcqWoOVMQcGEnWhEDNCoTPF7cfGfY,1360
+logdetective/extractors.py,sha256=7ahzWbTtU9MveG1Q7wU9LO8OJgs85X-cHmWltUhCe9M,3491
+logdetective/logdetective.py,sha256=cC2oL4yPNo94AB2nS4v1jpZi-Qo1g0_FEchL_yQL1UU,5832
+logdetective/models.py,sha256=nrGBmMRu8i6UhFflQKAp81Y3Sd_Aaoor0i_yqSJoLT0,1115
+logdetective/prompts.yml,sha256=dMW2-bdTIqv7LF_owqRD4xinMK5ZWcNhDynnX1zoKns,1722
+logdetective/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+logdetective/server/database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+logdetective/server/database/base.py,sha256=oMJUvbWeapIUP-8Cf_DR9ptFg8CsYeaBAIjOVEzx8SM,1668
+logdetective/server/database/models.py,sha256=m_3qNBWJwLSwjJn0AmwSxXMJk75Gu1bXFtGAP_4zps4,14088
+logdetective/server/metric.py,sha256=-uM_-yqxNA-EZTCnNRdQ8g1MicmE5eC6jRFI_mBBYUg,2606
+logdetective/server/models.py,sha256=URqZcfx5yUsifZ1pOwZ_uU3Tyjcdvuq6qEnAvTexl4A,8475
+logdetective/server/plot.py,sha256=B2rOngqx7g-Z3NfttboTip3frkypdF1H7FhK8vh45mE,9655
+logdetective/server/server.py,sha256=AXduOwD6zPNFc-COw-JcTp4bPan9DfXJml52XiBHmds,29613
+logdetective/server/templates/gitlab_full_comment.md.j2,sha256=DQZ2WVFedpuXI6znbHIW4wpF9BmFS8FaUkowh8AnGhE,1627
+logdetective/server/templates/gitlab_short_comment.md.j2,sha256=fzScpayv2vpRLczP_0O0YxtA8rsKvR6gSv4ntNdWb98,1443
+logdetective/server/utils.py,sha256=QO0H1q55YLCLKxkViqex4Uu31LnakpYUKJfZHysonSc,1838
+logdetective/utils.py,sha256=IBAH6hleuhtbA3cAGhiIDb3zwKaovunBfVsG0TKzoKA,7127
+logdetective-0.6.0.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+logdetective-0.6.0.dist-info/METADATA,sha256=zBYLi3jNjqR-6EYJY4wy5CS-1-ZgEzzRGofcqzXbHco,15880
+logdetective-0.6.0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+logdetective-0.6.0.dist-info/entry_points.txt,sha256=3K_vXja6PmcA8sNdUi63WdImeiNhVZcEGPTaoJmltfA,63
+logdetective-0.6.0.dist-info/RECORD,,

logdetective-0.5.10.dist-info/RECORD

@@ -1,23 +0,0 @@
-logdetective/__init__.py,sha256=VqRngDcuFT7JWms8Qc_MsOvajoXVOKPr-S1kqY3Pqhc,59
-logdetective/constants.py,sha256=eiS6eYhEgl_Rlyi_B9j00DDp9A-UDhuFz3ACWtKf_SU,1558
-logdetective/drain3.ini,sha256=ni91eCT1TwTznZwcqWoOVMQcGEnWhEDNCoTPF7cfGfY,1360
-logdetective/extractors.py,sha256=7ahzWbTtU9MveG1Q7wU9LO8OJgs85X-cHmWltUhCe9M,3491
-logdetective/logdetective.py,sha256=SDuzeS9sMp7rs6cTZAEd0ajtyWv9XnDkEPTF82nwaYo,5390
-logdetective/models.py,sha256=nrGBmMRu8i6UhFflQKAp81Y3Sd_Aaoor0i_yqSJoLT0,1115
-logdetective/prompts.yml,sha256=3orDNqqZNadWCaNncgfk8D3Pqqef4IzfScoa_jUJzCY,1452
-logdetective/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-logdetective/server/database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-logdetective/server/database/base.py,sha256=oMJUvbWeapIUP-8Cf_DR9ptFg8CsYeaBAIjOVEzx8SM,1668
-logdetective/server/database/models.py,sha256=arIahOCT-hTmh904DXrWSkH7rlo13Ppu-OO80huX5Dc,6118
-logdetective/server/metric.py,sha256=VYMifrfIhcqgyu6YYN0c1nt8fC1iJ2_LCB7Bh2AheoE,2679
-logdetective/server/models.py,sha256=cf1ngu_-19rP_i49s5cEwIzh6SfL_ZpVy4EykCpfWck,8076
-logdetective/server/plot.py,sha256=3o-CNHjel04ekpwSB4ckV7dbiF663cfPkimQ0aP9U_8,7073
-logdetective/server/server.py,sha256=FDKx-6wsVoEwdEgcoepAT3GL0gZKjMSpB1VU-jaKt2w,24618
-logdetective/server/templates/gitlab_comment.md.j2,sha256=kheTkhQ-LfuFkr8av-Mw2a-9VYEUbDTLwaa-CKI6OkI,1622
-logdetective/server/utils.py,sha256=6y4gZCwQG4HcjWJwYdzwP46Jsm3xoNXZWH4kYmSWVZA,1741
-logdetective/utils.py,sha256=_cBBkBwZHX5qxy0K5WK2MnHA4x_oor7R-QED2VZLbCA,6226
-logdetective-0.5.10.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-logdetective-0.5.10.dist-info/METADATA,sha256=NbD3YEoEU-YAhH-VjOo95qWxyk1T1bq5wCih4N5oyqs,14738
-logdetective-0.5.10.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-logdetective-0.5.10.dist-info/entry_points.txt,sha256=3K_vXja6PmcA8sNdUi63WdImeiNhVZcEGPTaoJmltfA,63
-logdetective-0.5.10.dist-info/RECORD,,