logdetective 2.2.0__py3-none-any.whl → 2.3.0__py3-none-any.whl

logdetective/extractors.py

@@ -26,6 +26,9 @@ class Extractor:
         self.skip_snippets = skip_snippets
         self.max_snippet_len = max_snippet_len
 
+        if self.verbose:
+            LOG.setLevel(logging.DEBUG)
+
     def __call__(self, log: str) -> list[Tuple[int, str]]:
         raise NotImplementedError
 

logdetective/server/gitlab.py

@@ -11,9 +11,13 @@ import gitlab.v4
 import gitlab.v4.objects
 import jinja2
 import aiohttp
+import backoff
 
 from logdetective.server.config import SERVER_CONFIG, LOG
-from logdetective.server.exceptions import LogsTooLargeError
+from logdetective.server.exceptions import (
+    LogsTooLargeError,
+    LogDetectiveConnectionError,
+)
 from logdetective.server.llm import perform_staged_analysis
 from logdetective.server.metric import add_new_metrics, update_metrics
 from logdetective.server.models import (
@@ -29,6 +33,7 @@ from logdetective.server.database.models import (
     GitlabMergeRequestJobs,
 )
 from logdetective.server.compressors import RemoteLogCompressor
+from logdetective.server.utils import connection_error_giveup
 
 MR_REGEX = re.compile(r"refs/merge-requests/(\d+)/.*$")
 FAILURE_LOG_REGEX = re.compile(r"(\w*\.log)")
@@ -91,8 +96,8 @@ async def process_gitlab_job_event(
         log_url, preprocessed_log = await retrieve_and_preprocess_koji_logs(
             gitlab_cfg, job
         )
-    except LogsTooLargeError:
-        LOG.error("Could not retrieve logs. Too large.")
+    except (LogsTooLargeError, LogDetectiveConnectionError) as ex:
+        LOG.error("Could not retrieve logs due to %s", ex)
         raise
 
     # Submit log to Log Detective and await the results.
@@ -151,6 +156,9 @@ def is_eligible_package(project_name: str):
     return True
 
 
+@backoff.on_exception(
+    backoff.expo, ConnectionResetError, max_time=60, on_giveup=connection_error_giveup
+)
 async def retrieve_and_preprocess_koji_logs(
     gitlab_cfg: GitLabInstanceConfig,
     job: gitlab.v4.objects.ProjectJob,
@@ -256,8 +264,7 @@ async def retrieve_and_preprocess_koji_logs(
     LOG.debug("Failed architecture: %s", failed_arch)
 
     log_path = failed_arches[failed_arch].as_posix()
-
-    log_url = f"{gitlab_cfg.api_path}/projects/{job.project_id}/jobs/{job.id}/artifacts/{log_path}"  # pylint: disable=line-too-long
+    log_url = f"{gitlab_cfg.url}/{gitlab_cfg.api_path}/projects/{job.project_id}/jobs/{job.id}/artifacts/{log_path}"  # pylint: disable=line-too-long
     LOG.debug("Returning contents of %s%s", gitlab_cfg.url, log_url)
 
     # Return the log as a file-like object with .read() function

logdetective/server/models.py

@@ -307,6 +307,7 @@ class GitLabInstanceConfig(BaseModel):  # pylint: disable=too-many-instance-attr
 
     name: str = None
     url: str = None
+    # Path to API of the gitlab instance, assuming `url` as prefix.
     api_path: str = None
     api_token: str = None
 

logdetective/server/server.py

@@ -106,35 +106,35 @@ async def get_http_session(request: Request) -> aiohttp.ClientSession:
     return request.app.http
 
 
-def requires_token_when_set(authentication: Annotated[str | None, Header()] = None):
+def requires_token_when_set(authorization: Annotated[str | None, Header()] = None):
     """
-    FastAPI Depend function that expects a header named Authentication
+    FastAPI Depend function that expects a header named Authorization
 
     If LOGDETECTIVE_TOKEN env var is set, validate the client-supplied token
     otherwise ignore it
     """
     if not API_TOKEN:
-        LOG.info("LOGDETECTIVE_TOKEN env var not set, authentication disabled")
+        LOG.info("LOGDETECTIVE_TOKEN env var not set, authorization disabled")
         # no token required, means local dev environment
         return
-    if authentication:
+    if authorization:
         try:
-            token = authentication.split(" ", 1)[1]
+            token = authorization.split(" ", 1)[1]
         except (ValueError, IndexError) as ex:
             LOG.warning(
-                "Authentication header has invalid structure '%s', it should be 'Bearer TOKEN'",
-                authentication,
+                "Authorization header has invalid structure '%s', it should be 'Bearer TOKEN'",
+                authorization,
             )
             # eat the exception and raise 401 below
             raise HTTPException(
                 status_code=401,
-                detail=f"Invalid authentication, HEADER '{authentication}' not valid.",
+                detail=f"Invalid authorization, HEADER '{authorization}' not valid.",
             ) from ex
         if token == API_TOKEN:
             return
         LOG.info("Provided token '%s' does not match expected value.", token)
         raise HTTPException(status_code=401, detail=f"Token '{token}' not valid.")
-    LOG.error("No authentication header provided but LOGDETECTIVE_TOKEN env var is set")
+    LOG.error("No authorization header provided but LOGDETECTIVE_TOKEN env var is set")
     raise HTTPException(status_code=401, detail="No token provided.")
 
 

{logdetective-2.2.0.dist-info → logdetective-2.3.0.dist-info}/METADATA

@@ -1,8 +1,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: logdetective
-Version: 2.2.0
+Version: 2.3.0
 Summary: Log using LLM AI to search for build/test failures and provide ideas for fixing these.
 License: Apache-2.0
+License-File: LICENSE
 Author: Jiri Podivin
 Author-email: jpodivin@gmail.com
 Requires-Python: >=3.11,<4.0
@@ -15,6 +16,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Internet :: Log Analysis
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development :: Debuggers
@@ -93,12 +95,13 @@ Usage
 -----
 
 To analyze a log file, run the script with the following command line arguments:
-- `url` (required): The URL of the log file to be analyzed.
-- `--model` (optional, default: "Mistral-7B-Instruct-v0.2-GGUF"): The path or URL of the language model for analysis. As we are using LLama.cpp we want this to be in the `gguf` format. You can include the download link to the model here. If the model is already on your machine it will skip the download.
+- `file` (required): The path or URL of the log file to be analyzed.
+- `--model` (optional, default: "Mistral-7B-Instruct-v0.3-GGUF"): The path or Hugging space name of the language model for analysis. For models from Hugging Face, write them as `namespace/repo_name`. As we are using LLama.cpp we want this to be in the `gguf` format. If the model is already on your machine it will skip the download.
+- `--filename_suffix` (optional, default "Q4_K.gguf"): You can specify which suffix of the file to use. This option is applied when specifying model using the Hugging Face repository.
 - `--summarizer` DISABLED: LLM summarization option was removed. Argument is kept for backward compatibility only.(optional, default: "drain"): Choose between LLM and Drain template miner as the log summarizer. You can also provide the path to an existing language model file instead of using a URL.
 - `--n_lines` DISABLED: LLM summarization option was removed. Argument is kept for backward compatibility only. (optional, default: 8): The number of lines per chunk for LLM analysis. This only makes sense when you are summarizing with LLM.
-- `--n_clusters` (optional, default 8): Number of clusters for Drain to organize log chunks into. This only makes sense when you are summarizing with Drain
-- `--skip_snippets` Path to patterns for skipping snippets.
+- `--n_clusters` (optional, default 8): Number of clusters for Drain to organize log chunks into. This only makes sense when you are summarizing with Drain.
+- `--skip_snippets` Path to patterns for skipping snippets (in YAML).
 
 Example usage:
 
@@ -108,14 +111,10 @@ Or if the log file is stored locally:
 
     logdetective ./data/logs.txt
 
-Example you want to use a different model:
+Examples of using different models. Note the use of `--filename_suffix` (or `-F`) option, useful for models that were quantized:
 
-    logdetective https://example.com/logs.txt --model https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/Meta-Llama-3-8B-Instruct.Q5_K_S.gguf?download=true
-    logdetective https://example.com/logs.txt --model QuantFactory/Meta-Llama-3-8B-Instruct-GGUF
-
-Example of different suffix (useful for models that were quantized)
-
-    logdetective https://kojipkgs.fedoraproject.org//work/tasks/3367/131313367/build.log --model 'fedora-copr/granite-3.2-8b-instruct-GGUF' -F Q4_K.gguf
+    logdetective https://example.com/logs.txt --model QuantFactory/Meta-Llama-3-8B-Instruct-GGUF --filename_suffix Q5_K_S.gguf
+    logdetective https://kojipkgs.fedoraproject.org//work/tasks/3367/131313367/build.log --model 'fedora-copr/granite-3.2-8b-instruct-GGUF' -F Q4_K_M.gguf
 
 Example of altered prompts:
 
@@ -124,9 +123,9 @@ Example of altered prompts:
      logdetective https://kojipkgs.fedoraproject.org//work/tasks/3367/131313367/build.log --prompts ~/my-prompts.yml
 
 
-Note that streaming with some models (notably Meta-Llama-3 is broken) is broken and can be worked around by `no-stream` option:
+Note that streaming with some models (notably Meta-Llama-3) is broken and can be worked around by `no-stream` option:
 
-    logdetective https://example.com/logs.txt --model QuantFactory/Meta-Llama-3-8B-Instruct-GGUF --no-stream
+    logdetective https://example.com/logs.txt --model QuantFactory/Meta-Llama-3-8B-Instruct-GGUF --filename_suffix Q5_K_M.gguf --no-stream
 
 
 Real Example

{logdetective-2.2.0.dist-info → logdetective-2.3.0.dist-info}/RECORD

@@ -1,7 +1,7 @@
 logdetective/__init__.py,sha256=VqRngDcuFT7JWms8Qc_MsOvajoXVOKPr-S1kqY3Pqhc,59
 logdetective/constants.py,sha256=aCwrkBrDdS_kbNESK-Z-ewg--DSzodV2OMgwEq3UE38,2456
 logdetective/drain3.ini,sha256=ni91eCT1TwTznZwcqWoOVMQcGEnWhEDNCoTPF7cfGfY,1360
-logdetective/extractors.py,sha256=Nh5wMcLXtcYLFAHwr6naCPPOtWzLUCW2iF__UKfckUY,5927
+logdetective/extractors.py,sha256=vT-je4NkDgSj9rRtSeLpqBU52gIUnnVgJPHFbVihpCw,5993
 logdetective/logdetective.py,sha256=Ck7TL3YvdQG8zniudM8bM51LfTyVW6Ea3BarTjzjWHo,6606
 logdetective/models.py,sha256=uczmQtWFgSp_ZGssngdTM4qzPF1o64dCy0469GoSbjQ,2937
 logdetective/prompts-summary-first.yml,sha256=3Zfp4NNOfaFYq5xBlBjeQa5PdjYfS4v17OtJqQ-DRpU,821
@@ -20,20 +20,20 @@ logdetective/server/database/models/merge_request_jobs.py,sha256=q4reSC7YnEfWBPp
 logdetective/server/database/models/metrics.py,sha256=_UCaizcl9w4iX54EWvk5VvXeLcg2UfnQgXg4br3OLko,14214
 logdetective/server/emoji.py,sha256=hV4O0yfL0l1a3kWLImvBsY4AJQauKs7okYOGBEtYVz0,4795
 logdetective/server/exceptions.py,sha256=piV7wVKc-rw_pHrThbZbUjtmjuO5qUbjVNFwjdfcP3Q,864
-logdetective/server/gitlab.py,sha256=MrAprXLTN6Q15qBC_Y2y42iKdtmIfed_pfjEt0gABvc,16422
+logdetective/server/gitlab.py,sha256=hPQICeYK1B_1BS56GIExQgEhcdS5Y6IgBCkk-eXtnDY,16708
 logdetective/server/koji.py,sha256=LG1pRiKUFvYFRKzgQoUG3pUHfcEwMoaMNjUSMKw_pBA,5640
 logdetective/server/llm.py,sha256=bmA6LsV80OdO60q4WLoKuehuVDEYq-HhBAYcZeLfrv8,10150
 logdetective/server/metric.py,sha256=QrrX1FmMa7sc57av0P9UFOiCIFYVLs1opOWV3ObYo0s,4086
-logdetective/server/models.py,sha256=rsdEf3lw0fvjWKhC9evaSsfZQR-H2mg0uig4KA6ho0c,20762
+logdetective/server/models.py,sha256=jUzQUszMqgOeOrH1V2ORtejGrD1Nw1sX05bsbAyFvV4,20830
 logdetective/server/plot.py,sha256=C98U9prGoPkp8_t4v2dovdZuwOhSbxXSeB_K9Q2r3NE,14607
-logdetective/server/server.py,sha256=zap8Mz3NTFvaDJMNQDATbPYk6MhQ9o1J9gJECnGWvuQ,24694
+logdetective/server/server.py,sha256=s1QtD6FYsQX7BaVi783v7sqCEk4ccqlBJHFKUQlGgWM,24684
 logdetective/server/templates/gitlab_full_comment.md.j2,sha256=H4NPjm3l8X5d0TNtfyZZZj_gHY1Y7hWEqY6RaVA8qt0,1947
 logdetective/server/templates/gitlab_short_comment.md.j2,sha256=vPisU1c98LPKEwlKtMrtlqnEOlbykPZK96MpHAf-o88,1758
 logdetective/server/utils.py,sha256=7ub-Nz7LUP_idwi2_nEC4FBuY9otSBUVy9nw86-sjYc,3861
 logdetective/skip_snippets.yml,sha256=reGlhPPCo06nNUJWiC2LY-OJOoPdcyOB7QBTSMeh0eg,487
 logdetective/utils.py,sha256=9EyHKGNxtS1ObSepL-T3M43rKIxQJkFDA5yllLbS5Bs,9178
-logdetective-2.2.0.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-logdetective-2.2.0.dist-info/METADATA,sha256=9bIs4_L1PPIilOBBsfoLULfTTtZ8RYLCmq_0XpxJHXQ,21455
-logdetective-2.2.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-logdetective-2.2.0.dist-info/entry_points.txt,sha256=3K_vXja6PmcA8sNdUi63WdImeiNhVZcEGPTaoJmltfA,63
-logdetective-2.2.0.dist-info/RECORD,,
+logdetective-2.3.0.dist-info/METADATA,sha256=mQQpT-CjV5js3Rl0eU87hvvZcbheoUXV3niOTCjW6gY,21645
+logdetective-2.3.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+logdetective-2.3.0.dist-info/entry_points.txt,sha256=3K_vXja6PmcA8sNdUi63WdImeiNhVZcEGPTaoJmltfA,63
+logdetective-2.3.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+logdetective-2.3.0.dist-info/RECORD,,

{logdetective-2.2.0.dist-info → logdetective-2.3.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.3
+Generator: poetry-core 2.2.1
 Root-Is-Purelib: true
 Tag: py3-none-any