logdetective 2.10.0__py3-none-any.whl → 2.12.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- logdetective/server/config.py +1 -1
- logdetective/server/emoji.py +46 -48
- logdetective/server/gitlab.py +21 -8
- logdetective/server/llm.py +38 -12
- logdetective/server/models.py +66 -259
- logdetective/server/server.py +199 -32
- {logdetective-2.10.0.dist-info → logdetective-2.12.0.dist-info}/METADATA +2 -2
- {logdetective-2.10.0.dist-info → logdetective-2.12.0.dist-info}/RECORD +11 -11
- {logdetective-2.10.0.dist-info → logdetective-2.12.0.dist-info}/WHEEL +0 -0
- {logdetective-2.10.0.dist-info → logdetective-2.12.0.dist-info}/entry_points.txt +0 -0
- {logdetective-2.10.0.dist-info → logdetective-2.12.0.dist-info}/licenses/LICENSE +0 -0
logdetective/server/server.py
CHANGED
|
@@ -2,13 +2,17 @@ import os
|
|
|
2
2
|
import asyncio
|
|
3
3
|
import datetime
|
|
4
4
|
from enum import Enum
|
|
5
|
+
from collections import defaultdict
|
|
5
6
|
from contextlib import asynccontextmanager
|
|
6
7
|
from typing import Annotated
|
|
7
8
|
from io import BytesIO
|
|
8
9
|
|
|
10
|
+
from aiolimiter import AsyncLimiter
|
|
9
11
|
import matplotlib
|
|
10
12
|
import matplotlib.figure
|
|
11
13
|
import matplotlib.pyplot
|
|
14
|
+
from koji import ClientSession
|
|
15
|
+
from gitlab import Gitlab
|
|
12
16
|
from fastapi import (
|
|
13
17
|
FastAPI,
|
|
14
18
|
HTTPException,
|
|
@@ -18,12 +22,12 @@ from fastapi import (
|
|
|
18
22
|
Path,
|
|
19
23
|
Request,
|
|
20
24
|
)
|
|
21
|
-
|
|
22
25
|
from fastapi.responses import StreamingResponse
|
|
23
26
|
from fastapi.responses import Response as BasicResponse
|
|
24
27
|
import aiohttp
|
|
25
28
|
import sentry_sdk
|
|
26
29
|
|
|
30
|
+
from logdetective.extractors import DrainExtractor, CSGrepExtractor, Extractor
|
|
27
31
|
from logdetective.server.exceptions import KojiInvalidTaskID
|
|
28
32
|
|
|
29
33
|
from logdetective.server.database.models.koji import KojiTaskAnalysis
|
|
@@ -42,13 +46,14 @@ from logdetective.server.koji import (
|
|
|
42
46
|
from logdetective.remote_log import RemoteLog
|
|
43
47
|
from logdetective.server.llm import (
|
|
44
48
|
perform_staged_analysis,
|
|
45
|
-
|
|
46
|
-
|
|
49
|
+
perform_analysis,
|
|
50
|
+
perform_analysis_stream,
|
|
47
51
|
)
|
|
48
52
|
from logdetective.server.gitlab import process_gitlab_job_event
|
|
49
53
|
from logdetective.server.metric import track_request, add_new_metrics, update_metrics
|
|
50
54
|
from logdetective.server.models import (
|
|
51
55
|
BuildLog,
|
|
56
|
+
Config,
|
|
52
57
|
EmojiHook,
|
|
53
58
|
JobHook,
|
|
54
59
|
KojiInstanceConfig,
|
|
@@ -56,6 +61,7 @@ from logdetective.server.models import (
|
|
|
56
61
|
Response,
|
|
57
62
|
StagedResponse,
|
|
58
63
|
TimePeriod,
|
|
64
|
+
ExtractorConfig,
|
|
59
65
|
)
|
|
60
66
|
from logdetective.server import plot as plot_engine
|
|
61
67
|
from logdetective.server.database.models import (
|
|
@@ -78,6 +84,89 @@ if sentry_dsn := SERVER_CONFIG.general.sentry_dsn:
|
|
|
78
84
|
sentry_sdk.init(dsn=str(sentry_dsn), traces_sample_rate=1.0)
|
|
79
85
|
|
|
80
86
|
|
|
87
|
+
def initialize_extractors(extractor_config: ExtractorConfig) -> list[Extractor]:
|
|
88
|
+
"""Set up extractors based on provided ExtractorConfig."""
|
|
89
|
+
extractors: list[Extractor] = [
|
|
90
|
+
DrainExtractor(
|
|
91
|
+
verbose=extractor_config.verbose,
|
|
92
|
+
max_snippet_len=extractor_config.max_snippet_len,
|
|
93
|
+
max_clusters=extractor_config.max_clusters,
|
|
94
|
+
)
|
|
95
|
+
]
|
|
96
|
+
|
|
97
|
+
if extractor_config.csgrep:
|
|
98
|
+
extractors.append(
|
|
99
|
+
CSGrepExtractor(
|
|
100
|
+
verbose=extractor_config.verbose,
|
|
101
|
+
max_snippet_len=extractor_config.max_snippet_len,
|
|
102
|
+
)
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
return extractors
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class ConnectionManager:
|
|
109
|
+
"""
|
|
110
|
+
Manager for all connections and sesssions.
|
|
111
|
+
"""
|
|
112
|
+
|
|
113
|
+
koji_connections: dict[str, ClientSession] = {}
|
|
114
|
+
gitlab_connections: dict[str, Gitlab] = {}
|
|
115
|
+
gitlab_http_sessions: dict[str, aiohttp.ClientSession] = {}
|
|
116
|
+
|
|
117
|
+
async def initialize(self, service_config: Config):
|
|
118
|
+
"""Initialize all managed objects"""
|
|
119
|
+
|
|
120
|
+
for connection, config in service_config.gitlab.instances.items():
|
|
121
|
+
self.gitlab_connections[connection] = Gitlab(
|
|
122
|
+
url=config.url,
|
|
123
|
+
private_token=config.api_token,
|
|
124
|
+
timeout=config.timeout,
|
|
125
|
+
)
|
|
126
|
+
self.gitlab_http_sessions[connection] = aiohttp.ClientSession(
|
|
127
|
+
base_url=config.url,
|
|
128
|
+
headers={"Authorization": f"Bearer {config.api_token}"},
|
|
129
|
+
timeout=aiohttp.ClientTimeout(
|
|
130
|
+
total=config.timeout,
|
|
131
|
+
connect=3.07,
|
|
132
|
+
),
|
|
133
|
+
)
|
|
134
|
+
for connection, config in service_config.koji.instances.items():
|
|
135
|
+
self.koji_connections[connection] = ClientSession(baseurl=config.xmlrpc_url)
|
|
136
|
+
|
|
137
|
+
async def close(self):
|
|
138
|
+
"""Close all managed http sessions"""
|
|
139
|
+
for session in self.gitlab_http_sessions.values():
|
|
140
|
+
|
|
141
|
+
await session.close()
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class KojiCallbackManager:
|
|
145
|
+
"""Manages callbacks used by Koji, with callbacks referenced by task id.
|
|
146
|
+
|
|
147
|
+
Multiple callbacks can be assigned to a single task."""
|
|
148
|
+
|
|
149
|
+
_callbacks: defaultdict[int, set[str]]
|
|
150
|
+
|
|
151
|
+
def __init__(self) -> None:
|
|
152
|
+
self._callbacks = defaultdict(set)
|
|
153
|
+
|
|
154
|
+
def register_callback(self, task_id: int, callback: str):
|
|
155
|
+
"""Register a callback for a task"""
|
|
156
|
+
self._callbacks[task_id].add(callback)
|
|
157
|
+
|
|
158
|
+
def clear_callbacks(self, task_id: int):
|
|
159
|
+
"""Unregister a callback for a task"""
|
|
160
|
+
try:
|
|
161
|
+
del self._callbacks[task_id]
|
|
162
|
+
except KeyError:
|
|
163
|
+
pass
|
|
164
|
+
|
|
165
|
+
def get_callbacks(self, task_id: int) -> set[str]:
|
|
166
|
+
"""Get the callbacks for a task"""
|
|
167
|
+
return self._callbacks[task_id]
|
|
168
|
+
|
|
169
|
+
|
|
81
170
|
@asynccontextmanager
|
|
82
171
|
async def lifespan(fapp: FastAPI):
|
|
83
172
|
"""
|
|
@@ -89,14 +178,31 @@ async def lifespan(fapp: FastAPI):
|
|
|
89
178
|
)
|
|
90
179
|
)
|
|
91
180
|
|
|
181
|
+
# General limiter for async requests
|
|
182
|
+
fapp.state.llm_request_limiter = AsyncLimiter(
|
|
183
|
+
max_rate=SERVER_CONFIG.inference.requests_per_minute
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
# Manager for connections and sessions
|
|
187
|
+
fapp.state.connection_manager = ConnectionManager()
|
|
188
|
+
|
|
189
|
+
await fapp.state.connection_manager.initialize(service_config=SERVER_CONFIG)
|
|
190
|
+
|
|
191
|
+
# Set up extractors
|
|
192
|
+
fapp.state.extractors = initialize_extractors(SERVER_CONFIG.extractor)
|
|
193
|
+
|
|
194
|
+
# Koji callbacks
|
|
195
|
+
fapp.state.koji_callback_manager = KojiCallbackManager()
|
|
196
|
+
|
|
92
197
|
# Ensure that the database is initialized.
|
|
93
198
|
await logdetective.server.database.base.init()
|
|
94
199
|
|
|
95
200
|
# Start the background task scheduler for collecting emojis
|
|
96
|
-
asyncio.create_task(schedule_collect_emojis_task())
|
|
201
|
+
asyncio.create_task(schedule_collect_emojis_task(fapp.state.connection_manager))
|
|
97
202
|
|
|
98
203
|
yield
|
|
99
204
|
|
|
205
|
+
await fapp.state.connection_manager.close()
|
|
100
206
|
await fapp.http.close()
|
|
101
207
|
|
|
102
208
|
|
|
@@ -144,20 +250,24 @@ app = FastAPI(
|
|
|
144
250
|
contact={
|
|
145
251
|
"name": "Log Detective developers",
|
|
146
252
|
"url": "https://github.com/fedora-copr/logdetective",
|
|
147
|
-
"email": "copr-devel@lists.fedorahosted.org"
|
|
253
|
+
"email": "copr-devel@lists.fedorahosted.org",
|
|
148
254
|
},
|
|
149
255
|
license_info={
|
|
150
256
|
"name": "Apache-2.0",
|
|
151
257
|
"url": "https://www.apache.org/licenses/LICENSE-2.0.html",
|
|
152
258
|
},
|
|
153
259
|
version=get_version(),
|
|
154
|
-
dependencies=[Depends(requires_token_when_set)],
|
|
260
|
+
dependencies=[Depends(requires_token_when_set)],
|
|
261
|
+
lifespan=lifespan,
|
|
262
|
+
)
|
|
155
263
|
|
|
156
264
|
|
|
157
265
|
@app.post("/analyze", response_model=Response)
|
|
158
266
|
@track_request()
|
|
159
267
|
async def analyze_log(
|
|
160
|
-
build_log: BuildLog,
|
|
268
|
+
build_log: BuildLog,
|
|
269
|
+
request: Request,
|
|
270
|
+
http_session: aiohttp.ClientSession = Depends(get_http_session),
|
|
161
271
|
):
|
|
162
272
|
"""Provide endpoint for log file submission and analysis.
|
|
163
273
|
Request must be in form {"url":"<YOUR_URL_HERE>"}.
|
|
@@ -168,13 +278,19 @@ async def analyze_log(
|
|
|
168
278
|
remote_log = RemoteLog(build_log.url, http_session)
|
|
169
279
|
log_text = await remote_log.process_url()
|
|
170
280
|
|
|
171
|
-
return await
|
|
281
|
+
return await perform_analysis(
|
|
282
|
+
log_text,
|
|
283
|
+
async_request_limiter=request.app.state.llm_request_limiter,
|
|
284
|
+
extractors=request.app.state.extractors
|
|
285
|
+
)
|
|
172
286
|
|
|
173
287
|
|
|
174
288
|
@app.post("/analyze/staged", response_model=StagedResponse)
|
|
175
289
|
@track_request()
|
|
176
290
|
async def analyze_log_staged(
|
|
177
|
-
build_log: BuildLog,
|
|
291
|
+
build_log: BuildLog,
|
|
292
|
+
request: Request,
|
|
293
|
+
http_session: aiohttp.ClientSession = Depends(get_http_session),
|
|
178
294
|
):
|
|
179
295
|
"""Provide endpoint for log file submission and analysis.
|
|
180
296
|
Request must be in form {"url":"<YOUR_URL_HERE>"}.
|
|
@@ -185,7 +301,11 @@ async def analyze_log_staged(
|
|
|
185
301
|
remote_log = RemoteLog(build_log.url, http_session)
|
|
186
302
|
log_text = await remote_log.process_url()
|
|
187
303
|
|
|
188
|
-
return await perform_staged_analysis(
|
|
304
|
+
return await perform_staged_analysis(
|
|
305
|
+
log_text,
|
|
306
|
+
async_request_limiter=request.app.state.llm_request_limiter,
|
|
307
|
+
extractors=request.app.state.extractors,
|
|
308
|
+
)
|
|
189
309
|
|
|
190
310
|
|
|
191
311
|
@app.get(
|
|
@@ -245,10 +365,11 @@ async def get_koji_task_analysis(
|
|
|
245
365
|
async def analyze_rpmbuild_koji(
|
|
246
366
|
koji_instance: Annotated[str, Path(title="The Koji instance to use")],
|
|
247
367
|
task_id: Annotated[int, Path(title="The task ID to analyze")],
|
|
368
|
+
request: Request,
|
|
248
369
|
x_koji_token: Annotated[str, Header()] = "",
|
|
249
370
|
x_koji_callback: Annotated[str, Header()] = "",
|
|
250
371
|
background_tasks: BackgroundTasks = BackgroundTasks(),
|
|
251
|
-
):
|
|
372
|
+
): # pylint: disable=too-many-arguments disable=too-many-positional-arguments
|
|
252
373
|
"""Provide endpoint for retrieving log file analysis of a Koji task"""
|
|
253
374
|
|
|
254
375
|
try:
|
|
@@ -276,16 +397,23 @@ async def analyze_rpmbuild_koji(
|
|
|
276
397
|
# Task not yet analyzed or it timed out, so we need to start the
|
|
277
398
|
# analysis in the background and return a 202 (Accepted) error.
|
|
278
399
|
|
|
400
|
+
koji_connection = request.app.state.connection_manager.koji_connections[
|
|
401
|
+
koji_instance
|
|
402
|
+
]
|
|
279
403
|
background_tasks.add_task(
|
|
280
404
|
analyze_koji_task,
|
|
281
405
|
task_id,
|
|
282
406
|
koji_instance_config,
|
|
407
|
+
koji_connection,
|
|
408
|
+
request.app.state.llm_request_limiter,
|
|
409
|
+
request.app.state.extractors,
|
|
410
|
+
request.app.state.koji_callback_manager
|
|
283
411
|
)
|
|
284
412
|
|
|
285
413
|
# If a callback URL is provided, we need to add it to the callbacks
|
|
286
414
|
# table so that we can notify it when the analysis is complete.
|
|
287
415
|
if x_koji_callback:
|
|
288
|
-
|
|
416
|
+
request.app.state.koji_callback_manager.register_callback(task_id, x_koji_callback)
|
|
289
417
|
|
|
290
418
|
response = BasicResponse(
|
|
291
419
|
status_code=202, content=f"Beginning analysis of task {task_id}"
|
|
@@ -301,13 +429,19 @@ async def analyze_rpmbuild_koji(
|
|
|
301
429
|
return response
|
|
302
430
|
|
|
303
431
|
|
|
304
|
-
async def analyze_koji_task(
|
|
432
|
+
async def analyze_koji_task(
|
|
433
|
+
task_id: int,
|
|
434
|
+
koji_instance_config: KojiInstanceConfig,
|
|
435
|
+
koji_connection: ClientSession,
|
|
436
|
+
async_request_limiter: AsyncLimiter,
|
|
437
|
+
extractors: list[Extractor],
|
|
438
|
+
koji_callback_manager: KojiCallbackManager,
|
|
439
|
+
): # pylint: disable=too-many-arguments disable=too-many-positional-arguments
|
|
305
440
|
"""Analyze a koji task and return the response"""
|
|
306
441
|
|
|
307
442
|
# Get the log text from the koji task
|
|
308
|
-
koji_conn = koji_instance_config.get_connection()
|
|
309
443
|
log_file_name, log_text = await get_failed_log_from_koji_task(
|
|
310
|
-
|
|
444
|
+
koji_connection, task_id, max_size=SERVER_CONFIG.koji.max_artifact_size
|
|
311
445
|
)
|
|
312
446
|
|
|
313
447
|
# We need to handle the metric tracking manually here, because we need
|
|
@@ -327,7 +461,11 @@ async def analyze_koji_task(task_id: int, koji_instance_config: KojiInstanceConf
|
|
|
327
461
|
task_id=task_id,
|
|
328
462
|
log_file_name=log_file_name,
|
|
329
463
|
)
|
|
330
|
-
response = await perform_staged_analysis(
|
|
464
|
+
response = await perform_staged_analysis(
|
|
465
|
+
log_text,
|
|
466
|
+
async_request_limiter=async_request_limiter,
|
|
467
|
+
extractors=extractors
|
|
468
|
+
)
|
|
331
469
|
|
|
332
470
|
# Now that we have the response, we can update the metrics and mark the
|
|
333
471
|
# koji task analysis as completed.
|
|
@@ -335,12 +473,12 @@ async def analyze_koji_task(task_id: int, koji_instance_config: KojiInstanceConf
|
|
|
335
473
|
await KojiTaskAnalysis.add_response(task_id, metrics_id)
|
|
336
474
|
|
|
337
475
|
# Notify any callbacks that the analysis is complete.
|
|
338
|
-
for callback in
|
|
476
|
+
for callback in koji_callback_manager.get_callbacks(task_id):
|
|
339
477
|
LOG.info("Notifying callback %s of task %d completion", callback, task_id)
|
|
340
478
|
asyncio.create_task(send_koji_callback(callback, task_id))
|
|
341
479
|
|
|
342
480
|
# Now that it's sent, we can clear the callbacks for this task.
|
|
343
|
-
|
|
481
|
+
koji_callback_manager.clear_callbacks(task_id)
|
|
344
482
|
|
|
345
483
|
return response
|
|
346
484
|
|
|
@@ -353,18 +491,18 @@ async def send_koji_callback(callback: str, task_id: int):
|
|
|
353
491
|
|
|
354
492
|
|
|
355
493
|
@app.get("/queue/print")
|
|
356
|
-
async def queue_print(msg: str):
|
|
494
|
+
async def queue_print(msg: str, request: Request):
|
|
357
495
|
"""Debug endpoint to test the LLM request queue"""
|
|
358
496
|
LOG.info("Will print %s", msg)
|
|
359
497
|
|
|
360
|
-
result = await async_log(msg)
|
|
498
|
+
result = await async_log(msg, request)
|
|
361
499
|
|
|
362
500
|
LOG.info("Printed %s and returned it", result)
|
|
363
501
|
|
|
364
502
|
|
|
365
|
-
async def async_log(msg):
|
|
503
|
+
async def async_log(msg: str, request: Request):
|
|
366
504
|
"""Debug function to test the LLM request queue"""
|
|
367
|
-
async with
|
|
505
|
+
async with request.app.state.llm_request_limiter:
|
|
368
506
|
LOG.critical(msg)
|
|
369
507
|
return msg
|
|
370
508
|
|
|
@@ -378,7 +516,9 @@ async def get_version_wrapper():
|
|
|
378
516
|
@app.post("/analyze/stream", response_class=StreamingResponse)
|
|
379
517
|
@track_request()
|
|
380
518
|
async def analyze_log_stream(
|
|
381
|
-
build_log: BuildLog,
|
|
519
|
+
build_log: BuildLog,
|
|
520
|
+
request: Request,
|
|
521
|
+
http_session: aiohttp.ClientSession = Depends(get_http_session),
|
|
382
522
|
):
|
|
383
523
|
"""Stream response endpoint for Logdetective.
|
|
384
524
|
Request must be in form {"url":"<YOUR_URL_HERE>"}.
|
|
@@ -389,7 +529,11 @@ async def analyze_log_stream(
|
|
|
389
529
|
remote_log = RemoteLog(build_log.url, http_session)
|
|
390
530
|
log_text = await remote_log.process_url()
|
|
391
531
|
try:
|
|
392
|
-
stream =
|
|
532
|
+
stream = perform_analysis_stream(
|
|
533
|
+
log_text,
|
|
534
|
+
async_request_limiter=request.app.state.llm_request_limiter,
|
|
535
|
+
extractors=request.app.state.extractors,
|
|
536
|
+
)
|
|
393
537
|
except aiohttp.ClientResponseError as ex:
|
|
394
538
|
raise HTTPException(
|
|
395
539
|
status_code=400,
|
|
@@ -421,6 +565,7 @@ def is_valid_webhook_secret(forge, x_gitlab_token):
|
|
|
421
565
|
async def receive_gitlab_job_event_webhook(
|
|
422
566
|
job_hook: JobHook,
|
|
423
567
|
background_tasks: BackgroundTasks,
|
|
568
|
+
request: Request,
|
|
424
569
|
x_gitlab_instance: Annotated[str | None, Header()],
|
|
425
570
|
x_gitlab_token: Annotated[str | None, Header()] = None,
|
|
426
571
|
):
|
|
@@ -441,11 +586,21 @@ async def receive_gitlab_job_event_webhook(
|
|
|
441
586
|
|
|
442
587
|
# Handle the message in the background so we can return 204 immediately
|
|
443
588
|
gitlab_cfg = SERVER_CONFIG.gitlab.instances[forge.value]
|
|
589
|
+
gitlab_connection = request.app.state.connection_manager.gitlab_connections[
|
|
590
|
+
forge.value
|
|
591
|
+
]
|
|
592
|
+
gitlab_http_session = request.app.state.connection_manager.gitlab_http_sessions[
|
|
593
|
+
forge.value
|
|
594
|
+
]
|
|
444
595
|
background_tasks.add_task(
|
|
445
596
|
process_gitlab_job_event,
|
|
446
597
|
gitlab_cfg,
|
|
598
|
+
gitlab_connection,
|
|
599
|
+
gitlab_http_session,
|
|
447
600
|
forge,
|
|
448
601
|
job_hook,
|
|
602
|
+
request.app.state.llm_request_limiter,
|
|
603
|
+
request.app.state.extractors
|
|
449
604
|
)
|
|
450
605
|
|
|
451
606
|
# No return value or body is required for a webhook.
|
|
@@ -467,6 +622,7 @@ async def receive_gitlab_emoji_event_webhook(
|
|
|
467
622
|
x_gitlab_token: Annotated[str | None, Header()],
|
|
468
623
|
emoji_hook: EmojiHook,
|
|
469
624
|
background_tasks: BackgroundTasks,
|
|
625
|
+
request: Request,
|
|
470
626
|
):
|
|
471
627
|
"""Webhook endpoint for receiving emoji event notifications from Gitlab
|
|
472
628
|
https://docs.gitlab.com/user/project/integrations/webhook_events/#emoji-events
|
|
@@ -519,10 +675,14 @@ async def receive_gitlab_emoji_event_webhook(
|
|
|
519
675
|
# Inform the lookup table that we are processing this emoji
|
|
520
676
|
emoji_lookup[key] = False
|
|
521
677
|
|
|
678
|
+
gitlab_connection = request.app.state.connection_manager.gitlab_connections[
|
|
679
|
+
x_gitlab_instance
|
|
680
|
+
]
|
|
522
681
|
# Create a background task to process the emojis on this Merge Request.
|
|
523
682
|
background_tasks.add_task(
|
|
524
683
|
schedule_emoji_collection_for_mr,
|
|
525
684
|
forge,
|
|
685
|
+
gitlab_connection,
|
|
526
686
|
emoji_hook.merge_request.target_project_id,
|
|
527
687
|
emoji_hook.merge_request.iid,
|
|
528
688
|
background_tasks,
|
|
@@ -534,17 +694,21 @@ async def receive_gitlab_emoji_event_webhook(
|
|
|
534
694
|
|
|
535
695
|
|
|
536
696
|
async def schedule_emoji_collection_for_mr(
|
|
537
|
-
forge: Forge,
|
|
697
|
+
forge: Forge,
|
|
698
|
+
gitlab_connection: Gitlab,
|
|
699
|
+
project_id: int,
|
|
700
|
+
mr_iid: int,
|
|
701
|
+
background_tasks: BackgroundTasks,
|
|
538
702
|
):
|
|
539
703
|
"""Background task to update the database on emoji reactions"""
|
|
540
704
|
|
|
541
705
|
key = (forge, project_id, mr_iid)
|
|
542
706
|
|
|
543
707
|
# FIXME: Look up the connection from the Forge # pylint: disable=fixme
|
|
544
|
-
gitlab_conn = SERVER_CONFIG.gitlab.instances[forge.value].get_connection()
|
|
708
|
+
# gitlab_conn = SERVER_CONFIG.gitlab.instances[forge.value].get_connection()
|
|
545
709
|
|
|
546
710
|
LOG.debug("Looking up emojis for %s, %d, %d", forge, project_id, mr_iid)
|
|
547
|
-
await collect_emojis_for_mr(project_id, mr_iid,
|
|
711
|
+
await collect_emojis_for_mr(project_id, mr_iid, gitlab_connection)
|
|
548
712
|
|
|
549
713
|
# Check whether we've been asked to re-schedule this lookup because
|
|
550
714
|
# another request came in while it was processing.
|
|
@@ -555,6 +719,7 @@ async def schedule_emoji_collection_for_mr(
|
|
|
555
719
|
background_tasks.add_task(
|
|
556
720
|
schedule_emoji_collection_for_mr,
|
|
557
721
|
forge,
|
|
722
|
+
gitlab_connection,
|
|
558
723
|
project_id,
|
|
559
724
|
mr_iid,
|
|
560
725
|
background_tasks,
|
|
@@ -677,25 +842,27 @@ async def get_metrics(
|
|
|
677
842
|
return await handler()
|
|
678
843
|
|
|
679
844
|
|
|
680
|
-
async def collect_emoji_task():
|
|
845
|
+
async def collect_emoji_task(connection_manager: ConnectionManager):
|
|
681
846
|
"""Collect emoji feedback.
|
|
682
847
|
Query only comments created in the last year.
|
|
683
848
|
"""
|
|
684
849
|
|
|
685
|
-
for instance in SERVER_CONFIG.gitlab.instances.
|
|
850
|
+
for instance_url, instance in SERVER_CONFIG.gitlab.instances.items():
|
|
686
851
|
LOG.info(
|
|
687
852
|
"Collect emoji feedback for %s started at %s",
|
|
688
853
|
instance.url,
|
|
689
854
|
datetime.datetime.now(datetime.timezone.utc),
|
|
690
855
|
)
|
|
691
|
-
await collect_emojis(
|
|
856
|
+
await collect_emojis(
|
|
857
|
+
connection_manager.gitlab_connections[instance_url], TimePeriod(weeks=54)
|
|
858
|
+
)
|
|
692
859
|
LOG.info(
|
|
693
860
|
"Collect emoji feedback finished at %s",
|
|
694
861
|
datetime.datetime.now(datetime.timezone.utc),
|
|
695
862
|
)
|
|
696
863
|
|
|
697
864
|
|
|
698
|
-
async def schedule_collect_emojis_task():
|
|
865
|
+
async def schedule_collect_emojis_task(connection_manager: ConnectionManager):
|
|
699
866
|
"""Schedule the collect_emojis_task to run on a configured interval"""
|
|
700
867
|
while True:
|
|
701
868
|
seconds_until_run = SERVER_CONFIG.general.collect_emojis_interval
|
|
@@ -703,6 +870,6 @@ async def schedule_collect_emojis_task():
|
|
|
703
870
|
await asyncio.sleep(seconds_until_run)
|
|
704
871
|
|
|
705
872
|
try:
|
|
706
|
-
await collect_emoji_task()
|
|
873
|
+
await collect_emoji_task(connection_manager=connection_manager)
|
|
707
874
|
except Exception as e: # pylint: disable=broad-exception-caught
|
|
708
875
|
LOG.exception("Error in collect_emoji_task: %s", e)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: logdetective
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.12.0
|
|
4
4
|
Summary: Log using LLM AI to search for build/test failures and provide ideas for fixing these.
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
License-File: LICENSE
|
|
@@ -33,7 +33,7 @@ Requires-Dist: backoff (==2.2.1) ; extra == "server" or extra == "server-testing
|
|
|
33
33
|
Requires-Dist: drain3 (>=0.9.11,<0.10.0)
|
|
34
34
|
Requires-Dist: fastapi (>=0.111.1,<1.0.0) ; extra == "server" or extra == "server-testing"
|
|
35
35
|
Requires-Dist: flexmock (>=0.12.2,<0.13.0) ; extra == "testing"
|
|
36
|
-
Requires-Dist: huggingface-hub (
|
|
36
|
+
Requires-Dist: huggingface-hub (>=0.23.0,<1.4.0)
|
|
37
37
|
Requires-Dist: koji (>=1.35.0,<2.0.0) ; extra == "server" or extra == "server-testing"
|
|
38
38
|
Requires-Dist: llama-cpp-python (>0.2.56,!=0.2.86,<1.0.0)
|
|
39
39
|
Requires-Dist: matplotlib (>=3.8.4,<4.0.0) ; extra == "server" or extra == "server-testing"
|
|
@@ -10,7 +10,7 @@ logdetective/prompts.yml,sha256=i3z6Jcb4ScVi7LsxOpDlKiXrcvql3qO_JnLzkAKMn1c,3870
|
|
|
10
10
|
logdetective/remote_log.py,sha256=28QvdQiy7RBnd86EKCq_A75P21gSNlCbgxJe5XAe9MA,2258
|
|
11
11
|
logdetective/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
12
|
logdetective/server/compressors.py,sha256=y4aFYJ_9CbYdKuAI39Kc9GQSdPN8cSJ2c_VAz3T47EE,5249
|
|
13
|
-
logdetective/server/config.py,sha256=
|
|
13
|
+
logdetective/server/config.py,sha256=dYoqvexnMo8LBXhXezMIEqUwzTsRD-eWvRIFIYNv388,2540
|
|
14
14
|
logdetective/server/database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
15
15
|
logdetective/server/database/base.py,sha256=HSV2tgye7iYTDzJD1Q5X7_nlLuTMIFP-hRVQMYxngHQ,2073
|
|
16
16
|
logdetective/server/database/models/__init__.py,sha256=zoZMCt1_7tewDa6eEIIX_xrdN-tLegSiPNg5NiYaV3o,850
|
|
@@ -18,23 +18,23 @@ logdetective/server/database/models/exceptions.py,sha256=4ED7FSSA1liV9-7VIN2BwUi
|
|
|
18
18
|
logdetective/server/database/models/koji.py,sha256=HNWxHYDxf4JN9K2ue8-V8dH-0XY5ZmxqH7Y9lAIbILA,6436
|
|
19
19
|
logdetective/server/database/models/merge_request_jobs.py,sha256=MxiAVKQIsQMbFylBsmYBmVXYvid-4_5mwwXLfWdp6_w,19965
|
|
20
20
|
logdetective/server/database/models/metrics.py,sha256=4xsUdbtlp5PI1-iJQc5Dd8EPDgVVplD9hJRWeRDn43k,15443
|
|
21
|
-
logdetective/server/emoji.py,sha256=
|
|
21
|
+
logdetective/server/emoji.py,sha256=zSaYtLpSkpRCXpjMWnHR1bYwkmobMJASZ7YNalrd85U,5274
|
|
22
22
|
logdetective/server/exceptions.py,sha256=WN715KLL3ya6FiZ95v70VSbNuVhGuHFzxm2OeEPWQCw,981
|
|
23
|
-
logdetective/server/gitlab.py,sha256=
|
|
23
|
+
logdetective/server/gitlab.py,sha256=X9JSotUUlG9bOWYbUNKt9KqLUAj6Uocd2KNpfn35ccU,17192
|
|
24
24
|
logdetective/server/koji.py,sha256=LG1pRiKUFvYFRKzgQoUG3pUHfcEwMoaMNjUSMKw_pBA,5640
|
|
25
|
-
logdetective/server/llm.py,sha256=
|
|
25
|
+
logdetective/server/llm.py,sha256=wHMxRbAjI0q3osR5mRDR1kqww_6Pkc7JpF1mh9e6Mg8,10855
|
|
26
26
|
logdetective/server/metric.py,sha256=wLOpgcAch3rwhPA5P2YWUeMNAPsvRGseRjH5HlTb7JM,4529
|
|
27
|
-
logdetective/server/models.py,sha256=
|
|
27
|
+
logdetective/server/models.py,sha256=iJ-5UgScKKSRL8fRCsM23Z34P3p98LaduwWO-q9rudo,13041
|
|
28
28
|
logdetective/server/plot.py,sha256=8LERgY3vQckaHZV2PZfOrZT8CjCAiji57QCmRW24Rfo,14697
|
|
29
|
-
logdetective/server/server.py,sha256=
|
|
29
|
+
logdetective/server/server.py,sha256=AM10P72tc_7N0GhH_N7msFhLr7ZGNgIfgTxt2sjasVE,30982
|
|
30
30
|
logdetective/server/templates/base_response.html.j2,sha256=BJGGV_Xb0Lnue8kq32oG9lI5CQDf9vce7HMYsP-Pvb4,2040
|
|
31
31
|
logdetective/server/templates/gitlab_full_comment.md.j2,sha256=4UujUzl3lmdbNEADsxn3HVrjfUiUu2FvUlp9MDFGXQI,2321
|
|
32
32
|
logdetective/server/templates/gitlab_short_comment.md.j2,sha256=2krnMlGqqju2V_6pE0UqUR1P674OFaeX5BMyY5htTOQ,2022
|
|
33
33
|
logdetective/server/utils.py,sha256=0BZ8WmzXNEtkUty1kOyFbBxDZWL0Icc8BUrxuHw9uvs,4015
|
|
34
34
|
logdetective/skip_snippets.yml,sha256=reGlhPPCo06nNUJWiC2LY-OJOoPdcyOB7QBTSMeh0eg,487
|
|
35
35
|
logdetective/utils.py,sha256=yalhySOF_Gzmqx_Ft9qad3TplAfZ6LOmauGXEJfKWiE,9803
|
|
36
|
-
logdetective-2.
|
|
37
|
-
logdetective-2.
|
|
38
|
-
logdetective-2.
|
|
39
|
-
logdetective-2.
|
|
40
|
-
logdetective-2.
|
|
36
|
+
logdetective-2.12.0.dist-info/METADATA,sha256=q8qwE4AyHr0WfJZwNMbCb3-X0mBQfreXhuNtYxSfOSM,23273
|
|
37
|
+
logdetective-2.12.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
|
|
38
|
+
logdetective-2.12.0.dist-info/entry_points.txt,sha256=3K_vXja6PmcA8sNdUi63WdImeiNhVZcEGPTaoJmltfA,63
|
|
39
|
+
logdetective-2.12.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
|
40
|
+
logdetective-2.12.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|