logdetective 0.5.10__py3-none-any.whl → 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- logdetective/constants.py +8 -0
- logdetective/logdetective.py +25 -9
- logdetective/prompts.yml +6 -0
- logdetective/server/database/models.py +216 -12
- logdetective/server/metric.py +4 -6
- logdetective/server/models.py +11 -3
- logdetective/server/plot.py +114 -39
- logdetective/server/server.py +227 -91
- logdetective/server/templates/{gitlab_comment.md.j2 → gitlab_full_comment.md.j2} +1 -3
- logdetective/server/templates/gitlab_short_comment.md.j2 +53 -0
- logdetective/server/utils.py +3 -1
- logdetective/utils.py +28 -6
- {logdetective-0.5.10.dist-info → logdetective-0.6.0.dist-info}/METADATA +29 -5
- logdetective-0.6.0.dist-info/RECORD +24 -0
- logdetective-0.5.10.dist-info/RECORD +0 -23
- {logdetective-0.5.10.dist-info → logdetective-0.6.0.dist-info}/LICENSE +0 -0
- {logdetective-0.5.10.dist-info → logdetective-0.6.0.dist-info}/WHEEL +0 -0
- {logdetective-0.5.10.dist-info → logdetective-0.6.0.dist-info}/entry_points.txt +0 -0
logdetective/server/server.py
CHANGED
|
@@ -3,6 +3,7 @@ import json
|
|
|
3
3
|
import os
|
|
4
4
|
import re
|
|
5
5
|
import zipfile
|
|
6
|
+
from contextlib import asynccontextmanager
|
|
6
7
|
from pathlib import Path, PurePath
|
|
7
8
|
from tempfile import TemporaryFile
|
|
8
9
|
from typing import List, Annotated, Tuple, Dict, Any
|
|
@@ -11,7 +12,7 @@ from io import BytesIO
|
|
|
11
12
|
|
|
12
13
|
import matplotlib
|
|
13
14
|
import matplotlib.pyplot
|
|
14
|
-
from fastapi import FastAPI, HTTPException, BackgroundTasks, Depends, Header
|
|
15
|
+
from fastapi import FastAPI, HTTPException, BackgroundTasks, Depends, Header, Request
|
|
15
16
|
|
|
16
17
|
from fastapi.responses import StreamingResponse
|
|
17
18
|
from fastapi.responses import Response as BasicResponse
|
|
@@ -19,14 +20,14 @@ import gitlab
|
|
|
19
20
|
import gitlab.v4
|
|
20
21
|
import gitlab.v4.objects
|
|
21
22
|
import jinja2
|
|
22
|
-
import
|
|
23
|
+
import aiohttp
|
|
23
24
|
|
|
24
25
|
from logdetective.extractors import DrainExtractor
|
|
25
26
|
from logdetective.utils import (
|
|
26
|
-
validate_url,
|
|
27
27
|
compute_certainty,
|
|
28
28
|
format_snippets,
|
|
29
29
|
load_prompts,
|
|
30
|
+
get_url_content,
|
|
30
31
|
)
|
|
31
32
|
from logdetective.server.utils import (
|
|
32
33
|
load_server_config,
|
|
@@ -61,6 +62,27 @@ FAILURE_LOG_REGEX = re.compile(r"(\w*\.log)")
|
|
|
61
62
|
LOG = get_log(SERVER_CONFIG)
|
|
62
63
|
|
|
63
64
|
|
|
65
|
+
@asynccontextmanager
|
|
66
|
+
async def lifespan(fapp: FastAPI):
|
|
67
|
+
"""
|
|
68
|
+
Establish one HTTP session
|
|
69
|
+
"""
|
|
70
|
+
fapp.http = aiohttp.ClientSession(
|
|
71
|
+
timeout=aiohttp.ClientTimeout(
|
|
72
|
+
total=int(LOG_SOURCE_REQUEST_TIMEOUT), connect=3.07
|
|
73
|
+
)
|
|
74
|
+
)
|
|
75
|
+
yield
|
|
76
|
+
await fapp.http.close()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
async def get_http_session(request: Request) -> aiohttp.ClientSession:
|
|
80
|
+
"""
|
|
81
|
+
Return the single aiohttp ClientSession for this app
|
|
82
|
+
"""
|
|
83
|
+
return request.app.http
|
|
84
|
+
|
|
85
|
+
|
|
64
86
|
def requires_token_when_set(authentication: Annotated[str | None, Header()] = None):
|
|
65
87
|
"""
|
|
66
88
|
FastAPI Depend function that expects a header named Authentication
|
|
@@ -91,33 +113,20 @@ def requires_token_when_set(authentication: Annotated[str | None, Header()] = No
|
|
|
91
113
|
raise HTTPException(status_code=401, detail=f"Token {token} not valid.")
|
|
92
114
|
|
|
93
115
|
|
|
94
|
-
app = FastAPI(dependencies=[Depends(requires_token_when_set)])
|
|
116
|
+
app = FastAPI(dependencies=[Depends(requires_token_when_set)], lifespan=lifespan)
|
|
95
117
|
app.gitlab_conn = gitlab.Gitlab(
|
|
96
118
|
url=SERVER_CONFIG.gitlab.url, private_token=SERVER_CONFIG.gitlab.api_token
|
|
97
119
|
)
|
|
98
120
|
|
|
99
121
|
|
|
100
|
-
def process_url(url: str) -> str:
|
|
122
|
+
async def process_url(http: aiohttp.ClientSession, url: str) -> str:
|
|
101
123
|
"""Validate log URL and return log text."""
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
) from ex
|
|
109
|
-
|
|
110
|
-
if not log_request.ok:
|
|
111
|
-
raise HTTPException(
|
|
112
|
-
status_code=400,
|
|
113
|
-
detail="Something went wrong while getting the logs: "
|
|
114
|
-
f"[{log_request.status_code}] {log_request.text}",
|
|
115
|
-
)
|
|
116
|
-
else:
|
|
117
|
-
LOG.error("Invalid URL received ")
|
|
118
|
-
raise HTTPException(status_code=400, detail=f"Invalid log URL: {url}")
|
|
119
|
-
|
|
120
|
-
return log_request.text
|
|
124
|
+
try:
|
|
125
|
+
return await get_url_content(http, url, timeout=int(LOG_SOURCE_REQUEST_TIMEOUT))
|
|
126
|
+
except RuntimeError as ex:
|
|
127
|
+
raise HTTPException(
|
|
128
|
+
status_code=400, detail=f"We couldn't obtain the logs: {ex}"
|
|
129
|
+
) from ex
|
|
121
130
|
|
|
122
131
|
|
|
123
132
|
def mine_logs(log: str) -> List[Tuple[int, str]]:
|
|
@@ -137,7 +146,11 @@ def mine_logs(log: str) -> List[Tuple[int, str]]:
|
|
|
137
146
|
|
|
138
147
|
|
|
139
148
|
async def submit_to_llm_endpoint(
|
|
140
|
-
|
|
149
|
+
http: aiohttp.ClientSession,
|
|
150
|
+
url: str,
|
|
151
|
+
data: Dict[str, Any],
|
|
152
|
+
headers: Dict[str, str],
|
|
153
|
+
stream: bool,
|
|
141
154
|
) -> Any:
|
|
142
155
|
"""Send request to selected API endpoint. Verifying successful request unless
|
|
143
156
|
the using the stream response.
|
|
@@ -147,46 +160,46 @@ async def submit_to_llm_endpoint(
|
|
|
147
160
|
headers:
|
|
148
161
|
stream:
|
|
149
162
|
"""
|
|
163
|
+
LOG.debug("async request %s headers=%s data=%s", url, headers, data)
|
|
150
164
|
try:
|
|
151
|
-
|
|
152
|
-
response = requests.post(
|
|
165
|
+
response = await http.post(
|
|
153
166
|
url,
|
|
154
167
|
headers=headers,
|
|
155
|
-
|
|
168
|
+
# we need to use the `json=` parameter here and let aiohttp
|
|
169
|
+
# handle the json-encoding
|
|
170
|
+
json=data,
|
|
156
171
|
timeout=int(LLM_CPP_SERVER_TIMEOUT),
|
|
157
|
-
|
|
172
|
+
# Docs says chunked takes int, but:
|
|
173
|
+
# DeprecationWarning: Chunk size is deprecated #1615
|
|
174
|
+
# So let's make sure we either put True or None here
|
|
175
|
+
chunked=True if stream else None,
|
|
176
|
+
raise_for_status=True,
|
|
158
177
|
)
|
|
159
|
-
except
|
|
160
|
-
LOG.error("Llama-cpp query failed: %s", ex)
|
|
178
|
+
except aiohttp.ClientResponseError as ex:
|
|
161
179
|
raise HTTPException(
|
|
162
|
-
status_code=400,
|
|
180
|
+
status_code=400,
|
|
181
|
+
detail="HTTP Error while getting response from inference server "
|
|
182
|
+
f"[{ex.status}] {ex.message}",
|
|
183
|
+
) from ex
|
|
184
|
+
if stream:
|
|
185
|
+
return response
|
|
186
|
+
try:
|
|
187
|
+
return json.loads(await response.text())
|
|
188
|
+
except UnicodeDecodeError as ex:
|
|
189
|
+
LOG.error("Error encountered while parsing llama server response: %s", ex)
|
|
190
|
+
raise HTTPException(
|
|
191
|
+
status_code=400,
|
|
192
|
+
detail=f"Couldn't parse the response.\nError: {ex}\nData: {response.text}",
|
|
163
193
|
) from ex
|
|
164
|
-
if not stream:
|
|
165
|
-
if not response.ok:
|
|
166
|
-
raise HTTPException(
|
|
167
|
-
status_code=400,
|
|
168
|
-
detail="Something went wrong while getting a response from the llama server: "
|
|
169
|
-
f"[{response.status_code}] {response.text}",
|
|
170
|
-
)
|
|
171
|
-
try:
|
|
172
|
-
response = json.loads(response.text)
|
|
173
|
-
except UnicodeDecodeError as ex:
|
|
174
|
-
LOG.error("Error encountered while parsing llama server response: %s", ex)
|
|
175
|
-
raise HTTPException(
|
|
176
|
-
status_code=400,
|
|
177
|
-
detail=f"Couldn't parse the response.\nError: {ex}\nData: {response.text}",
|
|
178
|
-
) from ex
|
|
179
|
-
|
|
180
|
-
return response
|
|
181
194
|
|
|
182
195
|
|
|
183
196
|
async def submit_text( # pylint: disable=R0913,R0917
|
|
197
|
+
http: aiohttp.ClientSession,
|
|
184
198
|
text: str,
|
|
185
199
|
max_tokens: int = -1,
|
|
186
200
|
log_probs: int = 1,
|
|
187
201
|
stream: bool = False,
|
|
188
202
|
model: str = "default-model",
|
|
189
|
-
api_endpoint: str = "/chat/completions",
|
|
190
203
|
) -> Explanation:
|
|
191
204
|
"""Submit prompt to LLM using a selected endpoint.
|
|
192
205
|
max_tokens: number of tokens to be produces, 0 indicates run until encountering EOS
|
|
@@ -199,16 +212,17 @@ async def submit_text( # pylint: disable=R0913,R0917
|
|
|
199
212
|
if SERVER_CONFIG.inference.api_token:
|
|
200
213
|
headers["Authorization"] = f"Bearer {SERVER_CONFIG.inference.api_token}"
|
|
201
214
|
|
|
202
|
-
if api_endpoint == "/chat/completions":
|
|
215
|
+
if SERVER_CONFIG.inference.api_endpoint == "/chat/completions":
|
|
203
216
|
return await submit_text_chat_completions(
|
|
204
|
-
text, headers, max_tokens, log_probs > 0, stream, model
|
|
217
|
+
http, text, headers, max_tokens, log_probs > 0, stream, model
|
|
205
218
|
)
|
|
206
219
|
return await submit_text_completions(
|
|
207
|
-
text, headers, max_tokens, log_probs, stream, model
|
|
220
|
+
http, text, headers, max_tokens, log_probs, stream, model
|
|
208
221
|
)
|
|
209
222
|
|
|
210
223
|
|
|
211
224
|
async def submit_text_completions( # pylint: disable=R0913,R0917
|
|
225
|
+
http: aiohttp.ClientSession,
|
|
212
226
|
text: str,
|
|
213
227
|
headers: dict,
|
|
214
228
|
max_tokens: int = -1,
|
|
@@ -227,9 +241,11 @@ async def submit_text_completions( # pylint: disable=R0913,R0917
|
|
|
227
241
|
"logprobs": log_probs,
|
|
228
242
|
"stream": stream,
|
|
229
243
|
"model": model,
|
|
244
|
+
"temperature": SERVER_CONFIG.inference.temperature,
|
|
230
245
|
}
|
|
231
246
|
|
|
232
247
|
response = await submit_to_llm_endpoint(
|
|
248
|
+
http,
|
|
233
249
|
f"{SERVER_CONFIG.inference.url}/v1/completions",
|
|
234
250
|
data,
|
|
235
251
|
headers,
|
|
@@ -242,6 +258,7 @@ async def submit_text_completions( # pylint: disable=R0913,R0917
|
|
|
242
258
|
|
|
243
259
|
|
|
244
260
|
async def submit_text_chat_completions( # pylint: disable=R0913,R0917
|
|
261
|
+
http: aiohttp.ClientSession,
|
|
245
262
|
text: str,
|
|
246
263
|
headers: dict,
|
|
247
264
|
max_tokens: int = -1,
|
|
@@ -266,9 +283,11 @@ async def submit_text_chat_completions( # pylint: disable=R0913,R0917
|
|
|
266
283
|
"logprobs": log_probs,
|
|
267
284
|
"stream": stream,
|
|
268
285
|
"model": model,
|
|
286
|
+
"temperature": SERVER_CONFIG.inference.temperature,
|
|
269
287
|
}
|
|
270
288
|
|
|
271
289
|
response = await submit_to_llm_endpoint(
|
|
290
|
+
http,
|
|
272
291
|
f"{SERVER_CONFIG.inference.url}/v1/chat/completions",
|
|
273
292
|
data,
|
|
274
293
|
headers,
|
|
@@ -288,19 +307,23 @@ async def submit_text_chat_completions( # pylint: disable=R0913,R0917
|
|
|
288
307
|
|
|
289
308
|
@app.post("/analyze", response_model=Response)
|
|
290
309
|
@track_request()
|
|
291
|
-
async def analyze_log(
|
|
310
|
+
async def analyze_log(
|
|
311
|
+
build_log: BuildLog, http: aiohttp.ClientSession = Depends(get_http_session)
|
|
312
|
+
):
|
|
292
313
|
"""Provide endpoint for log file submission and analysis.
|
|
293
314
|
Request must be in form {"url":"<YOUR_URL_HERE>"}.
|
|
294
315
|
URL must be valid for the request to be passed to the LLM server.
|
|
295
316
|
Meaning that it must contain appropriate scheme, path and netloc,
|
|
296
317
|
while lacking result, params or query fields.
|
|
297
318
|
"""
|
|
298
|
-
log_text = process_url(build_log.url)
|
|
319
|
+
log_text = await process_url(http, build_log.url)
|
|
299
320
|
log_summary = mine_logs(log_text)
|
|
300
321
|
log_summary = format_snippets(log_summary)
|
|
301
322
|
response = await submit_text(
|
|
323
|
+
http,
|
|
302
324
|
PROMPT_CONFIG.prompt_template.format(log_summary),
|
|
303
|
-
|
|
325
|
+
model=SERVER_CONFIG.inference.model,
|
|
326
|
+
max_tokens=SERVER_CONFIG.inference.max_tokens,
|
|
304
327
|
)
|
|
305
328
|
certainty = 0
|
|
306
329
|
|
|
@@ -319,19 +342,23 @@ async def analyze_log(build_log: BuildLog):
|
|
|
319
342
|
|
|
320
343
|
@app.post("/analyze/staged", response_model=StagedResponse)
|
|
321
344
|
@track_request()
|
|
322
|
-
async def analyze_log_staged(
|
|
345
|
+
async def analyze_log_staged(
|
|
346
|
+
build_log: BuildLog, http: aiohttp.ClientSession = Depends(get_http_session)
|
|
347
|
+
):
|
|
323
348
|
"""Provide endpoint for log file submission and analysis.
|
|
324
349
|
Request must be in form {"url":"<YOUR_URL_HERE>"}.
|
|
325
350
|
URL must be valid for the request to be passed to the LLM server.
|
|
326
351
|
Meaning that it must contain appropriate scheme, path and netloc,
|
|
327
352
|
while lacking result, params or query fields.
|
|
328
353
|
"""
|
|
329
|
-
log_text = process_url(build_log.url)
|
|
354
|
+
log_text = await process_url(http, build_log.url)
|
|
330
355
|
|
|
331
|
-
return await perform_staged_analysis(log_text=log_text)
|
|
356
|
+
return await perform_staged_analysis(http, log_text=log_text)
|
|
332
357
|
|
|
333
358
|
|
|
334
|
-
async def perform_staged_analysis(
|
|
359
|
+
async def perform_staged_analysis(
|
|
360
|
+
http: aiohttp.ClientSession, log_text: str
|
|
361
|
+
) -> StagedResponse:
|
|
335
362
|
"""Submit the log file snippets to the LLM and retrieve their results"""
|
|
336
363
|
log_summary = mine_logs(log_text)
|
|
337
364
|
|
|
@@ -339,8 +366,10 @@ async def perform_staged_analysis(log_text: str) -> StagedResponse:
|
|
|
339
366
|
analyzed_snippets = await asyncio.gather(
|
|
340
367
|
*[
|
|
341
368
|
submit_text(
|
|
369
|
+
http,
|
|
342
370
|
PROMPT_CONFIG.snippet_prompt_template.format(s),
|
|
343
|
-
|
|
371
|
+
model=SERVER_CONFIG.inference.model,
|
|
372
|
+
max_tokens=SERVER_CONFIG.inference.max_tokens,
|
|
344
373
|
)
|
|
345
374
|
for s in log_summary
|
|
346
375
|
]
|
|
@@ -355,7 +384,10 @@ async def perform_staged_analysis(log_text: str) -> StagedResponse:
|
|
|
355
384
|
)
|
|
356
385
|
|
|
357
386
|
final_analysis = await submit_text(
|
|
358
|
-
|
|
387
|
+
http,
|
|
388
|
+
final_prompt,
|
|
389
|
+
model=SERVER_CONFIG.inference.model,
|
|
390
|
+
max_tokens=SERVER_CONFIG.inference.max_tokens,
|
|
359
391
|
)
|
|
360
392
|
|
|
361
393
|
certainty = 0
|
|
@@ -380,14 +412,16 @@ async def perform_staged_analysis(log_text: str) -> StagedResponse:
|
|
|
380
412
|
|
|
381
413
|
@app.post("/analyze/stream", response_class=StreamingResponse)
|
|
382
414
|
@track_request()
|
|
383
|
-
async def analyze_log_stream(
|
|
415
|
+
async def analyze_log_stream(
|
|
416
|
+
build_log: BuildLog, http: aiohttp.ClientSession = Depends(get_http_session)
|
|
417
|
+
):
|
|
384
418
|
"""Stream response endpoint for Logdetective.
|
|
385
419
|
Request must be in form {"url":"<YOUR_URL_HERE>"}.
|
|
386
420
|
URL must be valid for the request to be passed to the LLM server.
|
|
387
421
|
Meaning that it must contain appropriate scheme, path and netloc,
|
|
388
422
|
while lacking result, params or query fields.
|
|
389
423
|
"""
|
|
390
|
-
log_text = process_url(build_log.url)
|
|
424
|
+
log_text = await process_url(http, build_log.url)
|
|
391
425
|
log_summary = mine_logs(log_text)
|
|
392
426
|
log_summary = format_snippets(log_summary)
|
|
393
427
|
headers = {"Content-Type": "application/json"}
|
|
@@ -396,7 +430,12 @@ async def analyze_log_stream(build_log: BuildLog):
|
|
|
396
430
|
headers["Authorization"] = f"Bearer {SERVER_CONFIG.inference.api_token}"
|
|
397
431
|
|
|
398
432
|
stream = await submit_text_chat_completions(
|
|
399
|
-
|
|
433
|
+
http,
|
|
434
|
+
PROMPT_CONFIG.prompt_template.format(log_summary),
|
|
435
|
+
stream=True,
|
|
436
|
+
headers=headers,
|
|
437
|
+
model=SERVER_CONFIG.inference.model,
|
|
438
|
+
max_tokens=SERVER_CONFIG.inference.max_tokens,
|
|
400
439
|
)
|
|
401
440
|
|
|
402
441
|
return StreamingResponse(stream)
|
|
@@ -404,31 +443,28 @@ async def analyze_log_stream(build_log: BuildLog):
|
|
|
404
443
|
|
|
405
444
|
@app.post("/webhook/gitlab/job_events")
|
|
406
445
|
async def receive_gitlab_job_event_webhook(
|
|
407
|
-
job_hook: JobHook,
|
|
446
|
+
job_hook: JobHook,
|
|
447
|
+
background_tasks: BackgroundTasks,
|
|
448
|
+
http: aiohttp.ClientSession = Depends(get_http_session),
|
|
408
449
|
):
|
|
409
450
|
"""Webhook endpoint for receiving job_events notifications from GitLab
|
|
410
451
|
https://docs.gitlab.com/user/project/integrations/webhook_events/#job-events
|
|
411
452
|
lists the full specification for the messages sent for job events."""
|
|
412
453
|
|
|
413
454
|
# Handle the message in the background so we can return 200 immediately
|
|
414
|
-
background_tasks.add_task(process_gitlab_job_event, job_hook)
|
|
455
|
+
background_tasks.add_task(process_gitlab_job_event, http, job_hook)
|
|
415
456
|
|
|
416
457
|
# No return value or body is required for a webhook.
|
|
417
458
|
# 204: No Content
|
|
418
459
|
return BasicResponse(status_code=204)
|
|
419
460
|
|
|
420
461
|
|
|
421
|
-
async def process_gitlab_job_event(job_hook):
|
|
462
|
+
async def process_gitlab_job_event(http: aiohttp.ClientSession, job_hook):
|
|
422
463
|
"""Handle a received job_event webhook from GitLab"""
|
|
423
464
|
LOG.debug("Received webhook message:\n%s", job_hook)
|
|
424
465
|
|
|
425
466
|
# Look up the project this job belongs to
|
|
426
467
|
project = await asyncio.to_thread(app.gitlab_conn.projects.get, job_hook.project_id)
|
|
427
|
-
|
|
428
|
-
# check if this project is on the opt-in list
|
|
429
|
-
if project.name not in SERVER_CONFIG.general.packages:
|
|
430
|
-
LOG.info("Ignoring unrecognized package %s", project.name)
|
|
431
|
-
return
|
|
432
468
|
LOG.info("Processing failed job for %s", project.name)
|
|
433
469
|
|
|
434
470
|
# Retrieve data about the job from the GitLab API
|
|
@@ -459,16 +495,21 @@ async def process_gitlab_job_event(job_hook):
|
|
|
459
495
|
LOG.debug("Retrieving log artifacts")
|
|
460
496
|
# Retrieve the build logs from the merge request artifacts and preprocess them
|
|
461
497
|
try:
|
|
462
|
-
log_url, preprocessed_log = await retrieve_and_preprocess_koji_logs(job)
|
|
498
|
+
log_url, preprocessed_log = await retrieve_and_preprocess_koji_logs(http, job)
|
|
463
499
|
except LogsTooLargeError:
|
|
464
500
|
LOG.error("Could not retrieve logs. Too large.")
|
|
465
501
|
raise
|
|
466
502
|
|
|
467
503
|
# Submit log to Log Detective and await the results.
|
|
468
504
|
log_text = preprocessed_log.read().decode(encoding="utf-8")
|
|
469
|
-
staged_response = await perform_staged_analysis(log_text=log_text)
|
|
505
|
+
staged_response = await perform_staged_analysis(http, log_text=log_text)
|
|
470
506
|
preprocessed_log.close()
|
|
471
507
|
|
|
508
|
+
# check if this project is on the opt-in list for posting comments.
|
|
509
|
+
if project.name not in SERVER_CONFIG.general.packages:
|
|
510
|
+
LOG.info("Not publishing comment for unrecognized package %s", project.name)
|
|
511
|
+
return
|
|
512
|
+
|
|
472
513
|
# Add the Log Detective response as a comment to the merge request
|
|
473
514
|
await comment_on_mr(project, merge_request_iid, job, log_url, staged_response)
|
|
474
515
|
|
|
@@ -477,7 +518,9 @@ class LogsTooLargeError(RuntimeError):
|
|
|
477
518
|
"""The log archive exceeds the configured maximum size"""
|
|
478
519
|
|
|
479
520
|
|
|
480
|
-
async def retrieve_and_preprocess_koji_logs(
|
|
521
|
+
async def retrieve_and_preprocess_koji_logs(
|
|
522
|
+
http: aiohttp.ClientSession, job: gitlab.v4.objects.ProjectJob
|
|
523
|
+
):
|
|
481
524
|
"""Download logs from the merge request artifacts
|
|
482
525
|
|
|
483
526
|
This function will retrieve the build logs and do some minimal
|
|
@@ -488,7 +531,7 @@ async def retrieve_and_preprocess_koji_logs(job: gitlab.v4.objects.ProjectJob):
|
|
|
488
531
|
Detective. The calling function is responsible for closing this object."""
|
|
489
532
|
|
|
490
533
|
# Make sure the file isn't too large to process.
|
|
491
|
-
if not await check_artifacts_file_size(job):
|
|
534
|
+
if not await check_artifacts_file_size(http, job):
|
|
492
535
|
raise LogsTooLargeError(
|
|
493
536
|
f"Oversized logs for job {job.id} in project {job.project_id}"
|
|
494
537
|
)
|
|
@@ -577,21 +620,28 @@ async def retrieve_and_preprocess_koji_logs(job: gitlab.v4.objects.ProjectJob):
|
|
|
577
620
|
return log_url, artifacts_zip.open(log_path)
|
|
578
621
|
|
|
579
622
|
|
|
580
|
-
async def check_artifacts_file_size(job):
|
|
623
|
+
async def check_artifacts_file_size(http: aiohttp.ClientSession, job):
|
|
581
624
|
"""Method to determine if the artifacts are too large to process"""
|
|
582
625
|
# First, make sure that the artifacts are of a reasonable size. The
|
|
583
626
|
# zipped artifact collection will be stored in memory below. The
|
|
584
627
|
# python-gitlab library doesn't expose a way to check this value directly,
|
|
585
628
|
# so we need to interact with directly with the headers.
|
|
586
629
|
artifacts_url = f"{SERVER_CONFIG.gitlab.api_url}/projects/{job.project_id}/jobs/{job.id}/artifacts" # pylint: disable=line-too-long
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
630
|
+
LOG.debug("checking artifact URL %s", artifacts_url)
|
|
631
|
+
try:
|
|
632
|
+
head_response = await http.head(
|
|
633
|
+
artifacts_url,
|
|
634
|
+
allow_redirects=True,
|
|
635
|
+
headers={"Authorization": f"Bearer {SERVER_CONFIG.gitlab.api_token}"},
|
|
636
|
+
timeout=5,
|
|
637
|
+
raise_for_status=True,
|
|
638
|
+
)
|
|
639
|
+
except aiohttp.ClientResponseError as ex:
|
|
640
|
+
raise HTTPException(
|
|
641
|
+
status_code=400,
|
|
642
|
+
detail=f"Unable to check artifact URL: [{ex.status}] {ex.message}",
|
|
643
|
+
) from ex
|
|
644
|
+
content_length = int(head_response.headers.get("content-length"))
|
|
595
645
|
LOG.debug(
|
|
596
646
|
"URL: %s, content-length: %d, max length: %d",
|
|
597
647
|
artifacts_url,
|
|
@@ -616,8 +666,8 @@ async def comment_on_mr(
|
|
|
616
666
|
response.explanation.text,
|
|
617
667
|
)
|
|
618
668
|
|
|
619
|
-
# Get the formatted comment.
|
|
620
|
-
|
|
669
|
+
# Get the formatted short comment.
|
|
670
|
+
short_comment = await generate_mr_comment(job, log_url, response, full=False)
|
|
621
671
|
|
|
622
672
|
# Look up the merge request
|
|
623
673
|
merge_request = await asyncio.to_thread(
|
|
@@ -625,11 +675,33 @@ async def comment_on_mr(
|
|
|
625
675
|
)
|
|
626
676
|
|
|
627
677
|
# Submit a new comment to the Merge Request using the Gitlab API
|
|
628
|
-
await asyncio.to_thread(
|
|
678
|
+
discussion = await asyncio.to_thread(
|
|
679
|
+
merge_request.discussions.create, {"body": short_comment}
|
|
680
|
+
)
|
|
681
|
+
|
|
682
|
+
# Get the ID of the first note
|
|
683
|
+
note_id = discussion.attributes["notes"][0]["id"]
|
|
684
|
+
note = discussion.notes.get(note_id)
|
|
685
|
+
|
|
686
|
+
# Update the comment with the full details
|
|
687
|
+
# We do this in a second step so we don't bombard the user's email
|
|
688
|
+
# notifications with a massive message. Gitlab doesn't send email for
|
|
689
|
+
# comment edits.
|
|
690
|
+
full_comment = await generate_mr_comment(job, log_url, response, full=True)
|
|
691
|
+
note.body = full_comment
|
|
692
|
+
|
|
693
|
+
# Pause for five seconds before sending the snippet data, otherwise
|
|
694
|
+
# Gitlab may bundle the edited message together with the creation
|
|
695
|
+
# message in email.
|
|
696
|
+
await asyncio.sleep(5)
|
|
697
|
+
await asyncio.to_thread(note.save)
|
|
629
698
|
|
|
630
699
|
|
|
631
700
|
async def generate_mr_comment(
|
|
632
|
-
job: gitlab.v4.objects.ProjectJob,
|
|
701
|
+
job: gitlab.v4.objects.ProjectJob,
|
|
702
|
+
log_url: str,
|
|
703
|
+
response: StagedResponse,
|
|
704
|
+
full: bool = True,
|
|
633
705
|
) -> str:
|
|
634
706
|
"""Use a template to generate a comment string to submit to Gitlab"""
|
|
635
707
|
|
|
@@ -637,7 +709,11 @@ async def generate_mr_comment(
|
|
|
637
709
|
script_path = Path(__file__).resolve().parent
|
|
638
710
|
template_path = Path(script_path, "templates")
|
|
639
711
|
jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_path))
|
|
640
|
-
|
|
712
|
+
|
|
713
|
+
if full:
|
|
714
|
+
tpl = jinja_env.get_template("gitlab_full_comment.md.j2")
|
|
715
|
+
else:
|
|
716
|
+
tpl = jinja_env.get_template("gitlab_short_comment.md.j2")
|
|
641
717
|
|
|
642
718
|
artifacts_url = f"{job.project_url}/-/jobs/{job.id}/artifacts/download"
|
|
643
719
|
|
|
@@ -676,6 +752,35 @@ def _svg_figure_response(fig: matplotlib.figure.Figure):
|
|
|
676
752
|
)
|
|
677
753
|
|
|
678
754
|
|
|
755
|
+
def _multiple_svg_figures_response(figures: list[matplotlib.figure.Figure]):
|
|
756
|
+
"""Create a response with multiple svg figures."""
|
|
757
|
+
svg_contents = []
|
|
758
|
+
for i, fig in enumerate(figures):
|
|
759
|
+
buf = BytesIO()
|
|
760
|
+
fig.savefig(buf, format="svg", bbox_inches="tight")
|
|
761
|
+
matplotlib.pyplot.close(fig)
|
|
762
|
+
buf.seek(0)
|
|
763
|
+
svg_contents.append(buf.read().decode("utf-8"))
|
|
764
|
+
|
|
765
|
+
html_content = "<html><body>\n"
|
|
766
|
+
for i, svg in enumerate(svg_contents):
|
|
767
|
+
html_content += f"<div id='figure-{i}'>\n{svg}\n</div>\n"
|
|
768
|
+
html_content += "</body></html>"
|
|
769
|
+
|
|
770
|
+
return BasicResponse(content=html_content, media_type="text/html")
|
|
771
|
+
|
|
772
|
+
|
|
773
|
+
@app.get("/metrics/analyze", response_class=StreamingResponse)
|
|
774
|
+
async def show_analyze_metrics(period_since_now: TimePeriod = Depends(TimePeriod)):
|
|
775
|
+
"""Show statistics for requests and responses in the given period of time
|
|
776
|
+
for the /analyze API endpoint."""
|
|
777
|
+
fig_requests = plot.requests_per_time(period_since_now, EndpointType.ANALYZE)
|
|
778
|
+
fig_responses = plot.average_time_per_responses(
|
|
779
|
+
period_since_now, EndpointType.ANALYZE
|
|
780
|
+
)
|
|
781
|
+
return _multiple_svg_figures_response([fig_requests, fig_responses])
|
|
782
|
+
|
|
783
|
+
|
|
679
784
|
@app.get("/metrics/analyze/requests", response_class=StreamingResponse)
|
|
680
785
|
async def show_analyze_requests(period_since_now: TimePeriod = Depends(TimePeriod)):
|
|
681
786
|
"""Show statistics for the requests received in the given period of time
|
|
@@ -684,6 +789,27 @@ async def show_analyze_requests(period_since_now: TimePeriod = Depends(TimePerio
|
|
|
684
789
|
return _svg_figure_response(fig)
|
|
685
790
|
|
|
686
791
|
|
|
792
|
+
@app.get("/metrics/analyze/responses", response_class=StreamingResponse)
|
|
793
|
+
async def show_analyze_responses(period_since_now: TimePeriod = Depends(TimePeriod)):
|
|
794
|
+
"""Show statistics for responses given in the specified period of time
|
|
795
|
+
for the /analyze API endpoint."""
|
|
796
|
+
fig = plot.average_time_per_responses(period_since_now, EndpointType.ANALYZE)
|
|
797
|
+
return _svg_figure_response(fig)
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
@app.get("/metrics/analyze/staged", response_class=StreamingResponse)
|
|
801
|
+
async def show_analyze_staged_metrics(
|
|
802
|
+
period_since_now: TimePeriod = Depends(TimePeriod),
|
|
803
|
+
):
|
|
804
|
+
"""Show statistics for requests and responses in the given period of time
|
|
805
|
+
for the /analyze/staged API endpoint."""
|
|
806
|
+
fig_requests = plot.requests_per_time(period_since_now, EndpointType.ANALYZE_STAGED)
|
|
807
|
+
fig_responses = plot.average_time_per_responses(
|
|
808
|
+
period_since_now, EndpointType.ANALYZE_STAGED
|
|
809
|
+
)
|
|
810
|
+
return _multiple_svg_figures_response([fig_requests, fig_responses])
|
|
811
|
+
|
|
812
|
+
|
|
687
813
|
@app.get("/metrics/analyze/staged/requests", response_class=StreamingResponse)
|
|
688
814
|
async def show_analyze_staged_requests(
|
|
689
815
|
period_since_now: TimePeriod = Depends(TimePeriod),
|
|
@@ -692,3 +818,13 @@ async def show_analyze_staged_requests(
|
|
|
692
818
|
for the /analyze/staged API endpoint."""
|
|
693
819
|
fig = plot.requests_per_time(period_since_now, EndpointType.ANALYZE_STAGED)
|
|
694
820
|
return _svg_figure_response(fig)
|
|
821
|
+
|
|
822
|
+
|
|
823
|
+
@app.get("/metrics/analyze/staged/responses", response_class=StreamingResponse)
|
|
824
|
+
async def show_analyze_staged_responses(
|
|
825
|
+
period_since_now: TimePeriod = Depends(TimePeriod),
|
|
826
|
+
):
|
|
827
|
+
"""Show statistics for responses given in the specified period of time
|
|
828
|
+
for the /analyze/staged API endpoint."""
|
|
829
|
+
fig = plot.average_time_per_responses(period_since_now, EndpointType.ANALYZE_STAGED)
|
|
830
|
+
return _svg_figure_response(fig)
|
|
@@ -9,9 +9,7 @@ In this case, we are {{ certainty }}% certain of the response {{ emoji_face }}.
|
|
|
9
9
|
<ul>
|
|
10
10
|
{% for snippet in snippets %}
|
|
11
11
|
<li>
|
|
12
|
-
<code>
|
|
13
|
-
Line {{ snippet.line_number }}: {{ snippet.text }}
|
|
14
|
-
</code>
|
|
12
|
+
<b>Line {{ snippet.line_number }}:</b> <code>{{ snippet.text }}</code>
|
|
15
13
|
{{ snippet.explanation }}
|
|
16
14
|
</li>
|
|
17
15
|
{% endfor %}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
The package {{ package }} failed to build, here is a possible explanation why.
|
|
2
|
+
|
|
3
|
+
Please know that the explanation was provided by AI and may be incorrect.
|
|
4
|
+
In this case, we are {{ certainty }}% certain of the response {{ emoji_face }}.
|
|
5
|
+
|
|
6
|
+
{{ explanation }}
|
|
7
|
+
|
|
8
|
+
<details>
|
|
9
|
+
<summary>Logs</summary>
|
|
10
|
+
<p>
|
|
11
|
+
Log Detective analyzed the following logs files to provide an explanation:
|
|
12
|
+
</p>
|
|
13
|
+
|
|
14
|
+
<ul>
|
|
15
|
+
<li><a href="{{ log_url }}">{{ log_url }}</a></li>
|
|
16
|
+
</ul>
|
|
17
|
+
|
|
18
|
+
<p>
|
|
19
|
+
Additional logs are available from:
|
|
20
|
+
<ul>
|
|
21
|
+
<li><a href="{{ artifacts_url }}">artifacts.zip</a></li>
|
|
22
|
+
</ul>
|
|
23
|
+
</p>
|
|
24
|
+
|
|
25
|
+
<p>
|
|
26
|
+
Please know that these log files are automatically removed after some
|
|
27
|
+
time, so you might need a backup.
|
|
28
|
+
</p>
|
|
29
|
+
</details>
|
|
30
|
+
|
|
31
|
+
<details>
|
|
32
|
+
<summary>Help</summary>
|
|
33
|
+
<p>Don't hesitate to reach out.</p>
|
|
34
|
+
|
|
35
|
+
<ul>
|
|
36
|
+
<li><a href="https://github.com/fedora-copr/logdetective">Upstream</a></li>
|
|
37
|
+
<li><a href="https://github.com/fedora-copr/logdetective/issues">Issue tracker</a></li>
|
|
38
|
+
<li><a href="https://redhat.enterprise.slack.com/archives/C06DWNVKKDE">Slack</a></li>
|
|
39
|
+
<li><a href="https://log-detective.com/documentation">Documentation</a></li>
|
|
40
|
+
</ul>
|
|
41
|
+
</details>
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
This comment was created by [Log Detective][log-detective].
|
|
46
|
+
|
|
47
|
+
Was the provided feedback accurate and helpful? <br>Please vote with :thumbsup:
|
|
48
|
+
or :thumbsdown: to help us improve.<br>
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
[log-detective]: https://log-detective.com/
|
|
53
|
+
[contact]: https://github.com/fedora-copr
|