logdetective 0.5.11__py3-none-any.whl → 0.9.0__py3-none-any.whl

logdetective/utils.py

@@ -2,12 +2,14 @@ import logging
 import os
 from typing import Iterator, List, Dict, Tuple, Generator
 from urllib.parse import urlparse
+
+import aiohttp
 import numpy as np
-import requests
 import yaml
 
 from llama_cpp import Llama, CreateCompletionResponse, CreateCompletionStreamResponse
 from logdetective.models import PromptConfig
+from logdetective.server.remote_log import RemoteLog
 
 
 LOG = logging.getLogger("logdetective")
@@ -111,8 +113,7 @@ def compute_certainty(probs: List[Dict]) -> float:
 
 
 def process_log(
-    log: str, model: Llama, stream: bool, prompt_template: str,
-    temperature: float
+    log: str, model: Llama, stream: bool, prompt_template: str, temperature: float
 ) -> CreateCompletionResponse | Iterator[CreateCompletionStreamResponse]:
     """Processes a given log using the provided language model and returns its summary.
 
@@ -126,14 +127,17 @@ def process_log(
         str: The summary of the given log generated by the language model.
     """
     response = model(
-        prompt=prompt_template.format(log), stream=stream, max_tokens=0, logprobs=1,
-        temperature=temperature
+        prompt=prompt_template.format(log),
+        stream=stream,
+        max_tokens=0,
+        logprobs=1,
+        temperature=temperature,
     )
 
     return response
 
 
-def retrieve_log_content(log_path: str) -> str:
+async def retrieve_log_content(http: aiohttp.ClientSession, log_path: str) -> str:
     """Get content of the file on the log_path path.
     Path is assumed to be valid URL if it has a scheme.
     Otherwise it attempts to pull it from local filesystem."""
@@ -148,7 +152,8 @@ def retrieve_log_content(log_path: str) -> str:
             log = f.read()
 
     else:
-        log = requests.get(log_path, timeout=60).text
+        remote_log = RemoteLog(log_path, http)
+        log = await remote_log.get_url_content()
 
     return log
 
@@ -180,21 +185,6 @@ def format_snippets(snippets: list[str] | list[Tuple[int, str]]) -> str:
     return summary
 
 
-def validate_url(url: str) -> bool:
-    """Validate incoming URL to be at least somewhat sensible for log files
-    Only http and https protocols permitted. No result, params or query fields allowed.
-    Either netloc or path must have non-zero length.
-    """
-    result = urlparse(url)
-    if result.scheme not in ["http", "https"]:
-        return False
-    if any([result.params, result.query, result.fragment]):
-        return False
-    if not (result.path or result.netloc):
-        return False
-    return True
-
-
 def load_prompts(path: str | None) -> PromptConfig:
     """Load prompts from given yaml file if there is one.
     Alternatively use defaults."""

{logdetective-0.5.11.dist-info → logdetective-0.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: logdetective
-Version: 0.5.11
+Version: 0.9.0
 Summary: Log using LLM AI to search for build/test failures and provide ideas for fixing these.
 License: Apache-2.0
 Author: Jiri Podivin
@@ -20,7 +20,9 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development :: Debuggers
 Provides-Extra: server
 Provides-Extra: server-testing
+Requires-Dist: aiohttp (>=3.7.4)
 Requires-Dist: alembic (>=1.13.3,<2.0.0) ; extra == "server" or extra == "server-testing"
+Requires-Dist: backoff (==2.2.1) ; extra == "server" or extra == "server-testing"
 Requires-Dist: drain3 (>=0.9.11,<0.10.0)
 Requires-Dist: fastapi (>=0.111.1) ; extra == "server" or extra == "server-testing"
 Requires-Dist: huggingface-hub (>0.23.2)
@@ -32,7 +34,7 @@ Requires-Dist: psycopg2-binary (>=2.9.9,<3.0.0) ; extra == "server-testing"
 Requires-Dist: pydantic (>=2.8.2,<3.0.0)
 Requires-Dist: python-gitlab (>=4.4.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
-Requires-Dist: requests (>0.2.31)
+Requires-Dist: sentry-sdk[fastapi] (>=2.17.0,<3.0.0)
 Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "server" or extra == "server-testing"
 Project-URL: homepage, https://github.com/fedora-copr/logdetective
 Project-URL: issues, https://github.com/fedora-copr/logdetective/issues
@@ -357,13 +359,16 @@ When no time period is specified, the query defaults to the last 2 days:
 
 You can view requests and responses statistics
  - for the `/analyze` endpoint at http://localhost:8080/metrics/analyze
- - for the `/analyze/staged` endpoint at http://localhost:8080/metrics/analyze/staged.
+ - for the `/analyze-staged` endpoint at http://localhost:8080/metrics/analyze-staged.
+ - for the requests coming from gitlab: http://localhost:8080/metrics/analyze-gitlab.
 
 You can retrieve single svg images at the following endpoints:
- - `/metrics/analyze/requests`
- - `/metrics/analyze/responses`
- - `/metrics/analyze/staged/requests`
- - `/metrics/analyze/stages/responses`
+ - http://localhost:8080/metrics/analyze/requests
+ - http://localhost:8080/metrics/analyze/responses
+ - http://localhost:8080/metrics/analyze-staged/requests
+ - http://localhost:8080/metrics/analyze-staged/responses
+ - http://localhost:8080/metrics/analyze-gitlab/requests
+ - http://localhost:8080/metrics/analyze-gitlab/responses
 
 Examples:
 

logdetective-0.9.0.dist-info/RECORD

@@ -0,0 +1,28 @@
+logdetective/__init__.py,sha256=VqRngDcuFT7JWms8Qc_MsOvajoXVOKPr-S1kqY3Pqhc,59
+logdetective/constants.py,sha256=A5PzeqlQqDbBS_kzP2hl-lhJ0lCEqdbvW3CaQUYVxjw,1849
+logdetective/drain3.ini,sha256=ni91eCT1TwTznZwcqWoOVMQcGEnWhEDNCoTPF7cfGfY,1360
+logdetective/extractors.py,sha256=7ahzWbTtU9MveG1Q7wU9LO8OJgs85X-cHmWltUhCe9M,3491
+logdetective/logdetective.py,sha256=cC2oL4yPNo94AB2nS4v1jpZi-Qo1g0_FEchL_yQL1UU,5832
+logdetective/models.py,sha256=nrGBmMRu8i6UhFflQKAp81Y3Sd_Aaoor0i_yqSJoLT0,1115
+logdetective/prompts.yml,sha256=urPKG068TYxi58EicFVUH6FavZq_q36oM1LvfI4ddjg,1729
+logdetective/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+logdetective/server/compressors.py,sha256=oT-jQPX3VfYzG0rIhvrQSR6oW_5FNkBGjaKbXIHmbGQ,4261
+logdetective/server/database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+logdetective/server/database/base.py,sha256=1mcjEbhwLl4RalvT3oy6XVctjJoWIW3H9aI_sMWJBK8,1728
+logdetective/server/database/models/__init__.py,sha256=xy2hkygyw6_87zPKkG20i7g7_LXTGR__PUeojhbvv94,496
+logdetective/server/database/models/merge_request_jobs.py,sha256=hrtZdYO09p8bMjOpjP2KEQeBj3_-fsb89-Br0Myk-yo,15571
+logdetective/server/database/models/metrics.py,sha256=yl9fS4IPVFWDeFvPAxO6zOVu6oLF319ApvVLAgnD5yU,13928
+logdetective/server/metric.py,sha256=DyhbFuQIBX8mUC27ilPenphPKVZ3tSjrvRwe_Lu4Gw8,3301
+logdetective/server/models.py,sha256=XlaIElxraZnUVjE--UTgUcinMVwicQFV_l5QbvkMsY4,8674
+logdetective/server/plot.py,sha256=B2rOngqx7g-Z3NfttboTip3frkypdF1H7FhK8vh45mE,9655
+logdetective/server/remote_log.py,sha256=BHCIWCppIRoT6MJfJvXtmbrG5BpGqFuULyAaC0WrTDo,3274
+logdetective/server/server.py,sha256=MUnaJETQlJD2ISzkx835eFnwMNgWCosf7iqv3zdYSM8,33783
+logdetective/server/templates/gitlab_full_comment.md.j2,sha256=DQZ2WVFedpuXI6znbHIW4wpF9BmFS8FaUkowh8AnGhE,1627
+logdetective/server/templates/gitlab_short_comment.md.j2,sha256=fzScpayv2vpRLczP_0O0YxtA8rsKvR6gSv4ntNdWb98,1443
+logdetective/server/utils.py,sha256=QO0H1q55YLCLKxkViqex4Uu31LnakpYUKJfZHysonSc,1838
+logdetective/utils.py,sha256=zghwtFSagHTmO0qrYAfXZfUEnt-BdnqD7DEGv6SbvpE,6095
+logdetective-0.9.0.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+logdetective-0.9.0.dist-info/METADATA,sha256=DwnAmPZMQ9Mr0sulWx1_R0TsqiA4lEDFQfTDAC8rSd0,16292
+logdetective-0.9.0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+logdetective-0.9.0.dist-info/entry_points.txt,sha256=3K_vXja6PmcA8sNdUi63WdImeiNhVZcEGPTaoJmltfA,63
+logdetective-0.9.0.dist-info/RECORD,,

logdetective-0.5.11.dist-info/RECORD

@@ -1,24 +0,0 @@
-logdetective/__init__.py,sha256=VqRngDcuFT7JWms8Qc_MsOvajoXVOKPr-S1kqY3Pqhc,59
-logdetective/constants.py,sha256=A5PzeqlQqDbBS_kzP2hl-lhJ0lCEqdbvW3CaQUYVxjw,1849
-logdetective/drain3.ini,sha256=ni91eCT1TwTznZwcqWoOVMQcGEnWhEDNCoTPF7cfGfY,1360
-logdetective/extractors.py,sha256=7ahzWbTtU9MveG1Q7wU9LO8OJgs85X-cHmWltUhCe9M,3491
-logdetective/logdetective.py,sha256=Q1SfQ9sWR5sIvHJag61-F-8edwf7p1SV7QZRg9VaWcc,5604
-logdetective/models.py,sha256=nrGBmMRu8i6UhFflQKAp81Y3Sd_Aaoor0i_yqSJoLT0,1115
-logdetective/prompts.yml,sha256=dMW2-bdTIqv7LF_owqRD4xinMK5ZWcNhDynnX1zoKns,1722
-logdetective/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-logdetective/server/database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-logdetective/server/database/base.py,sha256=oMJUvbWeapIUP-8Cf_DR9ptFg8CsYeaBAIjOVEzx8SM,1668
-logdetective/server/database/models.py,sha256=m_3qNBWJwLSwjJn0AmwSxXMJk75Gu1bXFtGAP_4zps4,14088
-logdetective/server/metric.py,sha256=-uM_-yqxNA-EZTCnNRdQ8g1MicmE5eC6jRFI_mBBYUg,2606
-logdetective/server/models.py,sha256=URqZcfx5yUsifZ1pOwZ_uU3Tyjcdvuq6qEnAvTexl4A,8475
-logdetective/server/plot.py,sha256=B2rOngqx7g-Z3NfttboTip3frkypdF1H7FhK8vh45mE,9655
-logdetective/server/server.py,sha256=4NylBojHm9E3gjByVWs870T204ls39EbZmUfU0Kyq4U,28395
-logdetective/server/templates/gitlab_full_comment.md.j2,sha256=DQZ2WVFedpuXI6znbHIW4wpF9BmFS8FaUkowh8AnGhE,1627
-logdetective/server/templates/gitlab_short_comment.md.j2,sha256=fzScpayv2vpRLczP_0O0YxtA8rsKvR6gSv4ntNdWb98,1443
-logdetective/server/utils.py,sha256=QO0H1q55YLCLKxkViqex4Uu31LnakpYUKJfZHysonSc,1838
-logdetective/utils.py,sha256=nklnTipAet9P9aEiuHcnK62WT0DmNHbvO1TvNlrxlik,6463
-logdetective-0.5.11.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-logdetective-0.5.11.dist-info/METADATA,sha256=LOOzu99kJaP02U2OaFQciPdWKhlgr4Vm4tVKijTY7NM,15882
-logdetective-0.5.11.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-logdetective-0.5.11.dist-info/entry_points.txt,sha256=3K_vXja6PmcA8sNdUi63WdImeiNhVZcEGPTaoJmltfA,63
-logdetective-0.5.11.dist-info/RECORD,,