logdetective 2.11.0__py3-none-any.whl → 2.13.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,7 +15,7 @@ def load_server_config(path: str | None) -> Config:
15
15
  if path is not None:
16
16
  try:
17
17
  with open(path, "r") as config_file:
18
- return Config(yaml.safe_load(config_file))
18
+ return Config.model_validate(yaml.safe_load(config_file))
19
19
  except FileNotFoundError:
20
20
  # This is not an error, we will fall back to default
21
21
  print("Unable to find server config file, using default then.")
@@ -22,7 +22,7 @@ sqlalchemy_echo = getenv("SQLALCHEMY_ECHO", "False").lower() in (
22
22
  "y",
23
23
  "1",
24
24
  )
25
- engine = create_async_engine(get_pg_url(), echo=sqlalchemy_echo)
25
+ engine = create_async_engine(get_pg_url(), echo=sqlalchemy_echo, pool_pre_ping=True)
26
26
  SessionFactory = async_sessionmaker(autoflush=True, bind=engine) # pylint: disable=invalid-name
27
27
 
28
28
 
@@ -4,6 +4,7 @@ import zipfile
4
4
  from pathlib import Path, PurePath
5
5
  from tempfile import TemporaryFile
6
6
 
7
+ from aiolimiter import AsyncLimiter
7
8
  from fastapi import HTTPException
8
9
 
9
10
  import gitlab
@@ -13,6 +14,7 @@ import jinja2
13
14
  import aiohttp
14
15
  import backoff
15
16
 
17
+ from logdetective.extractors import Extractor
16
18
  from logdetective.server.config import SERVER_CONFIG, LOG
17
19
  from logdetective.server.exceptions import (
18
20
  LogsTooLargeError,
@@ -41,15 +43,20 @@ FAILURE_LOG_REGEX = re.compile(r"(\w*\.log)")
41
43
 
42
44
  async def process_gitlab_job_event(
43
45
  gitlab_cfg: GitLabInstanceConfig,
46
+ gitlab_connection: gitlab.Gitlab,
47
+ http_session: aiohttp.ClientSession,
44
48
  forge: Forge,
45
49
  job_hook: JobHook,
46
- ): # pylint: disable=too-many-locals
50
+ async_request_limiter: AsyncLimiter,
51
+ extractors: list[Extractor],
52
+ ): # pylint: disable=too-many-locals disable=too-many-arguments disable=too-many-positional-arguments
47
53
  """Handle a received job_event webhook from GitLab"""
48
54
  LOG.debug("Received webhook message from %s:\n%s", forge.value, job_hook)
49
55
 
50
56
  # Look up the project this job belongs to
51
- gitlab_conn = gitlab_cfg.get_connection()
52
- project = await asyncio.to_thread(gitlab_conn.projects.get, job_hook.project_id)
57
+ project = await asyncio.to_thread(
58
+ gitlab_connection.projects.get, job_hook.project_id
59
+ )
53
60
  LOG.info("Processing failed job for %s", project.name)
54
61
 
55
62
  # Retrieve data about the job from the GitLab API
@@ -94,7 +101,7 @@ async def process_gitlab_job_event(
94
101
  # Retrieve the build logs from the merge request artifacts and preprocess them
95
102
  try:
96
103
  log_url, preprocessed_log = await retrieve_and_preprocess_koji_logs(
97
- gitlab_cfg, job
104
+ gitlab_cfg, job, http_session
98
105
  )
99
106
  except (LogsTooLargeError, LogDetectiveConnectionError) as ex:
100
107
  LOG.error("Could not retrieve logs due to %s", ex)
@@ -105,10 +112,14 @@ async def process_gitlab_job_event(
105
112
  metrics_id = await add_new_metrics(
106
113
  api_name=EndpointType.ANALYZE_GITLAB_JOB,
107
114
  url=log_url,
108
- http_session=gitlab_cfg.get_http_session(),
115
+ http_session=http_session,
109
116
  compressed_log_content=RemoteLogCompressor.zip_text(log_text),
110
117
  )
111
- staged_response = await perform_staged_analysis(log_text=log_text)
118
+ staged_response = await perform_staged_analysis(
119
+ log_text=log_text,
120
+ async_request_limiter=async_request_limiter,
121
+ extractors=extractors,
122
+ )
112
123
  await update_metrics(metrics_id, staged_response)
113
124
  preprocessed_log.close()
114
125
 
@@ -162,6 +173,7 @@ def is_eligible_package(project_name: str):
162
173
  async def retrieve_and_preprocess_koji_logs(
163
174
  gitlab_cfg: GitLabInstanceConfig,
164
175
  job: gitlab.v4.objects.ProjectJob,
176
+ http_session: aiohttp.ClientSession,
165
177
  ): # pylint: disable=too-many-branches,too-many-locals
166
178
  """Download logs from the merge request artifacts
167
179
 
@@ -173,7 +185,7 @@ async def retrieve_and_preprocess_koji_logs(
173
185
  Detective. The calling function is responsible for closing this object."""
174
186
 
175
187
  # Make sure the file isn't too large to process.
176
- if not await check_artifacts_file_size(gitlab_cfg, job):
188
+ if not await check_artifacts_file_size(gitlab_cfg, job, http_session):
177
189
  raise LogsTooLargeError(
178
190
  f"Oversized logs for job {job.id} in project {job.project_id}"
179
191
  )
@@ -274,6 +286,7 @@ async def retrieve_and_preprocess_koji_logs(
274
286
  async def check_artifacts_file_size(
275
287
  gitlab_cfg: GitLabInstanceConfig,
276
288
  job: gitlab.v4.objects.ProjectJob,
289
+ http_session: aiohttp.ClientSession,
277
290
  ):
278
291
  """Method to determine if the artifacts are too large to process"""
279
292
  # First, make sure that the artifacts are of a reasonable size. The
@@ -285,7 +298,7 @@ async def check_artifacts_file_size(
285
298
  )
286
299
  LOG.debug("checking artifact URL %s%s", gitlab_cfg.url, artifacts_path)
287
300
  try:
288
- head_response = await gitlab_cfg.get_http_session().head(
301
+ head_response = await http_session.head(
289
302
  artifacts_path,
290
303
  allow_redirects=True,
291
304
  raise_for_status=True,
@@ -9,9 +9,11 @@ from fastapi import HTTPException
9
9
  from pydantic import ValidationError
10
10
 
11
11
  import aiohttp
12
+ from aiolimiter import AsyncLimiter
12
13
  from openai import AsyncStream
13
14
  from openai.types.chat import ChatCompletionChunk
14
15
 
16
+ from logdetective.extractors import Extractor
15
17
  from logdetective.utils import (
16
18
  compute_certainty,
17
19
  prompt_to_messages,
@@ -41,7 +43,6 @@ from logdetective.server.utils import (
41
43
  construct_final_prompt,
42
44
  )
43
45
 
44
-
45
46
  LLM_CPP_SERVER_TIMEOUT = os.environ.get("LLAMA_CPP_SERVER_TIMEOUT", 600)
46
47
 
47
48
 
@@ -57,6 +58,7 @@ LLM_CPP_SERVER_TIMEOUT = os.environ.get("LLAMA_CPP_SERVER_TIMEOUT", 600)
57
58
  async def call_llm(
58
59
  messages: List[Dict[str, str]],
59
60
  inference_cfg: InferenceConfig,
61
+ async_request_limiter: AsyncLimiter,
60
62
  stream: bool = False,
61
63
  structured_output: dict | None = None,
62
64
  ) -> Explanation:
@@ -87,7 +89,7 @@ async def call_llm(
87
89
  }
88
90
  kwargs["response_format"] = response_format
89
91
 
90
- async with inference_cfg.get_limiter():
92
+ async with async_request_limiter:
91
93
  response = await CLIENT.chat.completions.create(
92
94
  messages=messages,
93
95
  max_tokens=inference_cfg.max_tokens,
@@ -126,6 +128,7 @@ async def call_llm(
126
128
  async def call_llm_stream(
127
129
  messages: List[Dict[str, str]],
128
130
  inference_cfg: InferenceConfig,
131
+ async_request_limiter: AsyncLimiter,
129
132
  stream: bool = False,
130
133
  ) -> AsyncStream[ChatCompletionChunk]:
131
134
  """Submit prompt to LLM and recieve stream of tokens as a result.
@@ -136,7 +139,7 @@ async def call_llm_stream(
136
139
 
137
140
  LOG.info("Submitting to /v1/chat/completions endpoint")
138
141
 
139
- async with inference_cfg.get_limiter():
142
+ async with async_request_limiter:
140
143
  response = await CLIENT.chat.completions.create(
141
144
  messages=messages,
142
145
  max_tokens=inference_cfg.max_tokens,
@@ -150,7 +153,9 @@ async def call_llm_stream(
150
153
 
151
154
 
152
155
  async def analyze_snippets(
153
- log_summary: List[Tuple[int, str]], structured_output: dict | None = None
156
+ log_summary: List[Tuple[int, str]],
157
+ async_request_limiter: AsyncLimiter,
158
+ structured_output: dict | None = None,
154
159
  ) -> List[SnippetAnalysis | RatedSnippetAnalysis]:
155
160
  """Submit log file snippets to the LLM and gather results"""
156
161
  # Process snippets asynchronously
@@ -162,6 +167,7 @@ async def analyze_snippets(
162
167
  SERVER_CONFIG.inference.system_role,
163
168
  SERVER_CONFIG.inference.user_role,
164
169
  ),
170
+ async_request_limiter=async_request_limiter,
165
171
  inference_cfg=SERVER_CONFIG.snippet_inference,
166
172
  structured_output=structured_output,
167
173
  )
@@ -184,9 +190,13 @@ async def analyze_snippets(
184
190
  return analyzed_snippets
185
191
 
186
192
 
187
- async def perfrom_analysis(log_text: str) -> Response:
193
+ async def perform_analysis(
194
+ log_text: str,
195
+ async_request_limiter: AsyncLimiter,
196
+ extractors: List[Extractor],
197
+ ) -> Response:
188
198
  """Sumbit log file snippets in aggregate to LLM and retrieve results"""
189
- log_summary = mine_logs(log_text, SERVER_CONFIG.extractor.get_extractors())
199
+ log_summary = mine_logs(log_text, extractors)
190
200
  log_summary = format_snippets(log_summary)
191
201
 
192
202
  final_prompt = construct_final_prompt(log_summary, PROMPT_CONFIG.prompt_template)
@@ -199,6 +209,7 @@ async def perfrom_analysis(log_text: str) -> Response:
199
209
  )
200
210
  response = await call_llm(
201
211
  messages,
212
+ async_request_limiter=async_request_limiter,
202
213
  inference_cfg=SERVER_CONFIG.inference,
203
214
  )
204
215
  certainty = 0
@@ -216,9 +227,13 @@ async def perfrom_analysis(log_text: str) -> Response:
216
227
  return Response(explanation=response, response_certainty=certainty)
217
228
 
218
229
 
219
- async def perform_analyis_stream(log_text: str) -> AsyncStream:
230
+ async def perform_analysis_stream(
231
+ log_text: str,
232
+ async_request_limiter: AsyncLimiter,
233
+ extractors: List[Extractor],
234
+ ) -> AsyncStream:
220
235
  """Submit log file snippets in aggregate and return a stream of tokens"""
221
- log_summary = mine_logs(log_text, SERVER_CONFIG.extractor.get_extractors())
236
+ log_summary = mine_logs(log_text, extractors)
222
237
  log_summary = format_snippets(log_summary)
223
238
 
224
239
  final_prompt = construct_final_prompt(log_summary, PROMPT_CONFIG.prompt_template)
@@ -232,6 +247,7 @@ async def perform_analyis_stream(log_text: str) -> AsyncStream:
232
247
 
233
248
  stream = call_llm_stream(
234
249
  messages,
250
+ async_request_limiter=async_request_limiter,
235
251
  inference_cfg=SERVER_CONFIG.inference,
236
252
  )
237
253
 
@@ -241,13 +257,18 @@ async def perform_analyis_stream(log_text: str) -> AsyncStream:
241
257
  return stream
242
258
 
243
259
 
244
- async def perform_staged_analysis(log_text: str) -> StagedResponse:
260
+ async def perform_staged_analysis(
261
+ log_text: str,
262
+ async_request_limiter: AsyncLimiter,
263
+ extractors: List[Extractor],
264
+ ) -> StagedResponse:
245
265
  """Submit the log file snippets to the LLM and retrieve their results"""
246
- log_summary = mine_logs(log_text, SERVER_CONFIG.extractor.get_extractors())
266
+ log_summary = mine_logs(log_text, extractors)
247
267
  start = time.time()
248
268
  if SERVER_CONFIG.general.top_k_snippets:
249
269
  rated_snippets = await analyze_snippets(
250
270
  log_summary=log_summary,
271
+ async_request_limiter=async_request_limiter,
251
272
  structured_output=RatedSnippetAnalysis.model_json_schema(),
252
273
  )
253
274
 
@@ -266,7 +287,9 @@ async def perform_staged_analysis(log_text: str) -> StagedResponse:
266
287
  len(rated_snippets),
267
288
  )
268
289
  else:
269
- processed_snippets = await analyze_snippets(log_summary=log_summary)
290
+ processed_snippets = await analyze_snippets(
291
+ log_summary=log_summary, async_request_limiter=async_request_limiter
292
+ )
270
293
 
271
294
  # Extract original text and line number from `log_summary`
272
295
  processed_snippets = [
@@ -276,7 +299,9 @@ async def perform_staged_analysis(log_text: str) -> StagedResponse:
276
299
  delta = time.time() - start
277
300
  LOG.info("Snippet analysis performed in %f s", delta)
278
301
  log_summary = format_analyzed_snippets(processed_snippets)
279
- final_prompt = construct_final_prompt(log_summary, PROMPT_CONFIG.prompt_template_staged)
302
+ final_prompt = construct_final_prompt(
303
+ log_summary, PROMPT_CONFIG.prompt_template_staged
304
+ )
280
305
 
281
306
  messages = prompt_to_messages(
282
307
  final_prompt,
@@ -286,6 +311,7 @@ async def perform_staged_analysis(log_text: str) -> StagedResponse:
286
311
  )
287
312
  final_analysis = await call_llm(
288
313
  messages,
314
+ async_request_limiter=async_request_limiter,
289
315
  inference_cfg=SERVER_CONFIG.inference,
290
316
  )
291
317
 
@@ -1,8 +1,6 @@
1
- import asyncio
2
- from collections import defaultdict
3
1
  import datetime
4
2
  from logging import BASIC_FORMAT
5
- from typing import List, Dict, Optional
3
+ from typing import List, Dict, Optional, Any
6
4
  from pydantic import (
7
5
  BaseModel,
8
6
  Field,
@@ -10,14 +8,8 @@ from pydantic import (
10
8
  field_validator,
11
9
  NonNegativeFloat,
12
10
  HttpUrl,
13
- PrivateAttr,
14
11
  )
15
12
 
16
- import aiohttp
17
-
18
- from aiolimiter import AsyncLimiter
19
- from gitlab import Gitlab
20
- import koji
21
13
 
22
14
  from logdetective.constants import (
23
15
  DEFAULT_TEMPERATURE,
@@ -26,8 +18,6 @@ from logdetective.constants import (
26
18
  SYSTEM_ROLE_DEFAULT,
27
19
  USER_ROLE_DEFAULT,
28
20
  )
29
-
30
- from logdetective.extractors import Extractor, DrainExtractor, CSGrepExtractor
31
21
  from logdetective.utils import check_csgrep
32
22
 
33
23
 
@@ -177,40 +167,14 @@ class InferenceConfig(BaseModel): # pylint: disable=too-many-instance-attribute
177
167
  # OpenAI client library requires a string to be specified for API token
178
168
  # even if it is not checked on the server side
179
169
  api_token: str = "None"
180
- model: str = ""
170
+ model: str = "default-model"
181
171
  temperature: NonNegativeFloat = DEFAULT_TEMPERATURE
182
172
  max_queue_size: int = LLM_DEFAULT_MAX_QUEUE_SIZE
183
173
  http_timeout: float = 5.0
184
174
  user_role: str = USER_ROLE_DEFAULT
185
175
  system_role: str = SYSTEM_ROLE_DEFAULT
186
176
  llm_api_timeout: float = 15.0
187
- _limiter: AsyncLimiter = PrivateAttr(
188
- default_factory=lambda: AsyncLimiter(LLM_DEFAULT_REQUESTS_PER_MINUTE))
189
-
190
- def __init__(self, data: Optional[dict] = None):
191
- super().__init__()
192
- if data is None:
193
- return
194
-
195
- self.max_tokens = data.get("max_tokens", -1)
196
- self.log_probs = data.get("log_probs", True)
197
- self.url = data.get("url", "")
198
- self.http_timeout = data.get("http_timeout", 5.0)
199
- self.api_token = data.get("api_token", "None")
200
- self.model = data.get("model", "default-model")
201
- self.temperature = data.get("temperature", DEFAULT_TEMPERATURE)
202
- self.max_queue_size = data.get("max_queue_size", LLM_DEFAULT_MAX_QUEUE_SIZE)
203
- self.user_role = data.get("user_role", USER_ROLE_DEFAULT)
204
- self.system_role = data.get("system_role", SYSTEM_ROLE_DEFAULT)
205
- self._requests_per_minute = data.get(
206
- "requests_per_minute", LLM_DEFAULT_REQUESTS_PER_MINUTE
207
- )
208
- self.llm_api_timeout = data.get("llm_api_timeout", 15.0)
209
- self._limiter = AsyncLimiter(self._requests_per_minute)
210
-
211
- def get_limiter(self):
212
- """Return the limiter object so it can be used as a context manager"""
213
- return self._limiter
177
+ requests_per_minute: int = LLM_DEFAULT_REQUESTS_PER_MINUTE
214
178
 
215
179
 
216
180
  class ExtractorConfig(BaseModel):
@@ -221,64 +185,25 @@ class ExtractorConfig(BaseModel):
221
185
  max_snippet_len: int = 2000
222
186
  csgrep: bool = False
223
187
 
224
- _extractors: List[Extractor] = PrivateAttr(default_factory=list)
225
-
226
- def _setup_extractors(self):
227
- """Initialize extractors with common settings."""
228
- self._extractors = [
229
- DrainExtractor(
230
- verbose=self.verbose,
231
- max_snippet_len=self.max_snippet_len,
232
- max_clusters=self.max_clusters,
233
- )
234
- ]
235
-
236
- if self.csgrep:
237
- self._extractors.append(
238
- CSGrepExtractor(
239
- verbose=self.verbose,
240
- max_snippet_len=self.max_snippet_len,
241
- )
242
- )
243
-
244
- def __init__(self, data: Optional[dict] = None):
245
- super().__init__(data=data)
246
-
247
- if data is None:
248
- self._setup_extractors()
249
- return
250
-
251
- self.max_clusters = data.get("max_clusters", 8)
252
- self.verbose = data.get("verbose", False)
253
- self.max_snippet_len = data.get("max_snippet_len", 2000)
254
- self.csgrep = data.get("csgrep", False)
255
-
256
- self._setup_extractors()
257
-
258
- def get_extractors(self) -> List[Extractor]:
259
- """Return list of initialized extractors, each will be applied in turn
260
- on original log text to retrieve snippets."""
261
- return self._extractors
262
-
263
- @field_validator("csgrep", mode="after")
188
+ @field_validator("csgrep", mode="before")
264
189
  @classmethod
265
- def validate_csgrep(cls, value: bool) -> bool:
266
- """Verify that csgrep is available if requested."""
267
- if not check_csgrep():
190
+ def verify_csgrep(cls, v: bool):
191
+ """Verify presence of csgrep binary if csgrep extractor is requested."""
192
+ if v and not check_csgrep():
268
193
  raise ValueError(
269
194
  "Requested csgrep extractor but `csgrep` binary is not in the PATH"
270
195
  )
271
- return value
196
+ return v
272
197
 
273
198
 
274
199
  class GitLabInstanceConfig(BaseModel): # pylint: disable=too-many-instance-attributes
275
200
  """Model for GitLab configuration of logdetective server."""
276
201
 
277
- name: str = None
278
- url: str = None
202
+ name: str
203
+ url: str = "https://gitlab.com"
279
204
  # Path to API of the gitlab instance, assuming `url` as prefix.
280
- api_path: str = None
281
- api_token: str = None
205
+ api_path: str = "/api/v4"
206
+ api_token: Optional[str] = None
282
207
 
283
208
  # This is a list to support key rotation.
284
209
  # When the key is being changed, we will add the new key as a new entry in
@@ -289,69 +214,17 @@ class GitLabInstanceConfig(BaseModel): # pylint: disable=too-many-instance-attr
289
214
  webhook_secrets: Optional[List[str]] = None
290
215
 
291
216
  timeout: float = 5.0
292
- _conn: Gitlab | None = PrivateAttr(default=None)
293
- _http_session: aiohttp.ClientSession | None = PrivateAttr(default=None)
294
217
 
295
218
  # Maximum size of artifacts.zip in MiB. (default: 300 MiB)
296
219
  max_artifact_size: int = 300 * 1024 * 1024
297
220
 
298
- def __init__(self, name: str, data: Optional[dict] = None):
299
- super().__init__()
300
- if data is None:
301
- return
302
-
303
- self.name = name
304
- self.url = data.get("url", "https://gitlab.com")
305
- self.api_path = data.get("api_path", "/api/v4")
306
- self.api_token = data.get("api_token", None)
307
- self.webhook_secrets = data.get("webhook_secrets", None)
308
- self.max_artifact_size = int(data.get("max_artifact_size", 300)) * 1024 * 1024
309
-
310
- self.timeout = data.get("timeout", 5.0)
311
- self._conn = Gitlab(
312
- url=self.url,
313
- private_token=self.api_token,
314
- timeout=self.timeout,
315
- )
316
-
317
- def get_connection(self):
318
- """Get the Gitlab connection object"""
319
- return self._conn
320
-
321
- def get_http_session(self):
322
- """Return the internal HTTP session so it can be used to contect the
323
- Gitlab server. May be used as a context manager."""
324
-
325
- # Create the session on the first attempt. We need to do this "lazily"
326
- # because it needs to happen once the event loop is running, even
327
- # though the initialization itself is synchronous.
328
- if not self._http_session:
329
- self._http_session = aiohttp.ClientSession(
330
- base_url=self.url,
331
- headers={"Authorization": f"Bearer {self.api_token}"},
332
- timeout=aiohttp.ClientTimeout(
333
- total=self.timeout,
334
- connect=3.07,
335
- ),
336
- )
337
-
338
- return self._http_session
339
-
340
- def __del__(self):
341
- # Close connection when this object is destroyed
342
- if self._http_session:
343
- try:
344
- loop = asyncio.get_running_loop()
345
- loop.create_task(self._http_session.close())
346
- except RuntimeError:
347
- # No loop running, so create one to close the session
348
- loop = asyncio.new_event_loop()
349
- loop.run_until_complete(self._http_session.close())
350
- loop.close()
351
- except Exception: # pylint: disable=broad-exception-caught
352
- # We should only get here if we're shutting down, so we don't
353
- # really care if the close() completes cleanly.
354
- pass
221
+ @field_validator("max_artifact_size", mode="before")
222
+ @classmethod
223
+ def megabytes_to_bytes(cls, v: Any):
224
+ """Convert max_artifact_size from megabytes to bytes."""
225
+ if isinstance(v, int):
226
+ return v * 1024 * 1024
227
+ return 300 * 1024 * 1024
355
228
 
356
229
 
357
230
  class GitLabConfig(BaseModel):
@@ -359,63 +232,28 @@ class GitLabConfig(BaseModel):
359
232
 
360
233
  instances: Dict[str, GitLabInstanceConfig] = {}
361
234
 
362
- def __init__(self, data: Optional[dict] = None):
363
- super().__init__()
364
- if data is None:
365
- return
235
+ @model_validator(mode="before")
236
+ @classmethod
237
+ def set_gitlab_instance_configs(cls, data: Any):
238
+ """Initialize configuration for each GitLab instance"""
239
+ if not isinstance(data, dict):
240
+ return data
366
241
 
242
+ instances = {}
367
243
  for instance_name, instance_data in data.items():
368
- instance = GitLabInstanceConfig(instance_name, instance_data)
369
- self.instances[instance.url] = instance
244
+ instance = GitLabInstanceConfig(name=instance_name, **instance_data)
245
+ instances[instance.url] = instance
246
+
247
+ return {"instances": instances}
370
248
 
371
249
 
372
250
  class KojiInstanceConfig(BaseModel):
373
251
  """Model for Koji configuration of logdetective server."""
374
252
 
375
253
  name: str = ""
376
- xmlrpc_url: str = ""
254
+ xmlrpc_url: str = "https://koji.fedoraproject.org/kojihub"
377
255
  tokens: List[str] = []
378
256
 
379
- _conn: Optional[koji.ClientSession] = PrivateAttr(default=None)
380
- _callbacks: defaultdict[int, set[str]] = PrivateAttr(default_factory=lambda: defaultdict(set))
381
-
382
- def __init__(self, name: str, data: Optional[dict] = None):
383
- super().__init__()
384
-
385
- self.name = name
386
- if data is None:
387
- # Set some reasonable defaults
388
- self.xmlrpc_url = "https://koji.fedoraproject.org/kojihub"
389
- self.tokens = []
390
- self.max_artifact_size = 1024 * 1024
391
- return
392
-
393
- self.xmlrpc_url = data.get(
394
- "xmlrpc_url", "https://koji.fedoraproject.org/kojihub"
395
- )
396
- self.tokens = data.get("tokens", [])
397
-
398
- def get_connection(self):
399
- """Get the Koji connection object"""
400
- if not self._conn:
401
- self._conn = koji.ClientSession(self.xmlrpc_url)
402
- return self._conn
403
-
404
- def register_callback(self, task_id: int, callback: str):
405
- """Register a callback for a task"""
406
- self._callbacks[task_id].add(callback)
407
-
408
- def clear_callbacks(self, task_id: int):
409
- """Unregister a callback for a task"""
410
- try:
411
- del self._callbacks[task_id]
412
- except KeyError:
413
- pass
414
-
415
- def get_callbacks(self, task_id: int) -> set[str]:
416
- """Get the callbacks for a task"""
417
- return self._callbacks[task_id]
418
-
419
257
 
420
258
  class KojiConfig(BaseModel):
421
259
  """Model for Koji configuration of logdetective server."""
@@ -424,23 +262,26 @@ class KojiConfig(BaseModel):
424
262
  analysis_timeout: int = 15
425
263
  max_artifact_size: int = 300 * 1024 * 1024
426
264
 
427
- def __init__(self, data: Optional[dict] = None):
428
- super().__init__()
429
- if data is None:
430
- return
431
-
432
- # Handle analysis_timeout with default 15
433
- self.analysis_timeout = data.get("analysis_timeout", 15)
434
-
435
- # Handle max_artifact_size with default 300
436
- self.max_artifact_size = data.get("max_artifact_size", 300) * 1024 * 1024
265
+ @field_validator("max_artifact_size", mode="before")
266
+ @classmethod
267
+ def megabytes_to_bytes(cls, v: Any):
268
+ """Convert max_artifact_size from megabytes to bytes."""
269
+ if isinstance(v, int):
270
+ return v * 1024 * 1024
271
+ return 300 * 1024 * 1024
437
272
 
438
- # Handle instances dictionary
439
- instances_data = data.get("instances", {})
440
- for instance_name, instance_data in instances_data.items():
441
- self.instances[instance_name] = KojiInstanceConfig(
442
- instance_name, instance_data
443
- )
273
+ @model_validator(mode="before")
274
+ @classmethod
275
+ def set_koji_instance_configs(cls, data: Any):
276
+ """Initialize configuration for each Koji instance."""
277
+ if isinstance(data, dict):
278
+ instances = {}
279
+ for instance_name, instance_data in data.get("instances", {}).items():
280
+ instances[instance_name] = KojiInstanceConfig(
281
+ name=instance_name, **instance_data
282
+ )
283
+ data["instances"] = instances
284
+ return data
444
285
 
445
286
 
446
287
  class LogConfig(BaseModel):
@@ -452,17 +293,6 @@ class LogConfig(BaseModel):
452
293
  path: str | None = None
453
294
  format: str = BASIC_FORMAT
454
295
 
455
- def __init__(self, data: Optional[dict] = None):
456
- super().__init__()
457
- if data is None:
458
- return
459
-
460
- self.name = data.get("name", "logdetective")
461
- self.level_stream = data.get("level_stream", "INFO").upper()
462
- self.level_file = data.get("level_file", "INFO").upper()
463
- self.path = data.get("path")
464
- self.format = data.get("format", BASIC_FORMAT)
465
-
466
296
 
467
297
  class GeneralConfig(BaseModel):
468
298
  """General config options for Log Detective"""
@@ -474,50 +304,27 @@ class GeneralConfig(BaseModel):
474
304
  collect_emojis_interval: int = 60 * 60 # seconds
475
305
  top_k_snippets: int = 0
476
306
 
477
- def __init__(self, data: Optional[dict] = None):
478
- super().__init__()
479
- if data is None:
480
- return
481
-
482
- self.packages = data.get("packages", [])
483
- self.excluded_packages = data.get("excluded_packages", [])
484
- self.devmode = data.get("devmode", False)
485
- self.sentry_dsn = data.get("sentry_dsn")
486
- self.collect_emojis_interval = data.get(
487
- "collect_emojis_interval", 60 * 60
488
- ) # seconds
489
- self.top_k_snippets = data.get("top_k_snippets", 0)
490
-
491
307
 
492
308
  class Config(BaseModel):
493
309
  """Model for configuration of logdetective server."""
494
310
 
495
- log: LogConfig = LogConfig()
496
- inference: InferenceConfig = InferenceConfig()
497
- snippet_inference: InferenceConfig = InferenceConfig()
311
+ log: LogConfig = Field(default_factory=LogConfig)
312
+ inference: InferenceConfig = Field(default_factory=InferenceConfig)
313
+ snippet_inference: InferenceConfig = Field(default_factory=InferenceConfig)
498
314
  # TODO(jpodivin): Extend to work with multiple extractor configs
499
- extractor: ExtractorConfig = ExtractorConfig()
500
- gitlab: GitLabConfig = GitLabConfig()
501
- koji: KojiConfig = KojiConfig()
502
- general: GeneralConfig = GeneralConfig()
503
-
504
- def __init__(self, data: Optional[dict] = None):
505
- super().__init__()
506
-
507
- if data is None:
508
- return
509
-
510
- self.log = LogConfig(data.get("log"))
511
- self.inference = InferenceConfig(data.get("inference"))
512
- self.extractor = ExtractorConfig(data.get("extractor"))
513
- self.gitlab = GitLabConfig(data.get("gitlab"))
514
- self.koji = KojiConfig(data.get("koji"))
515
- self.general = GeneralConfig(data.get("general"))
516
-
517
- if snippet_inference := data.get("snippet_inference", None):
518
- self.snippet_inference = InferenceConfig(snippet_inference)
519
- else:
520
- self.snippet_inference = self.inference
315
+ extractor: ExtractorConfig = Field(default_factory=ExtractorConfig)
316
+ gitlab: GitLabConfig = Field(default_factory=GitLabConfig)
317
+ koji: KojiConfig = Field(default_factory=KojiConfig)
318
+ general: GeneralConfig = Field(default_factory=GeneralConfig)
319
+
320
+ @model_validator(mode="before")
321
+ @classmethod
322
+ def default_snippet_inference(cls, data: Any):
323
+ """Use base inference configuration, if specific snippet configuration isn't provided."""
324
+ if isinstance(data, dict):
325
+ if "snippet_inference" not in data and "inference" in data:
326
+ data["snippet_inference"] = data["inference"]
327
+ return data
521
328
 
522
329
 
523
330
  class TimePeriod(BaseModel):
@@ -2,13 +2,17 @@ import os
2
2
  import asyncio
3
3
  import datetime
4
4
  from enum import Enum
5
+ from collections import defaultdict
5
6
  from contextlib import asynccontextmanager
6
7
  from typing import Annotated
7
8
  from io import BytesIO
8
9
 
10
+ from aiolimiter import AsyncLimiter
9
11
  import matplotlib
10
12
  import matplotlib.figure
11
13
  import matplotlib.pyplot
14
+ from koji import ClientSession
15
+ from gitlab import Gitlab
12
16
  from fastapi import (
13
17
  FastAPI,
14
18
  HTTPException,
@@ -18,12 +22,12 @@ from fastapi import (
18
22
  Path,
19
23
  Request,
20
24
  )
21
-
22
25
  from fastapi.responses import StreamingResponse
23
26
  from fastapi.responses import Response as BasicResponse
24
27
  import aiohttp
25
28
  import sentry_sdk
26
29
 
30
+ from logdetective.extractors import DrainExtractor, CSGrepExtractor, Extractor
27
31
  from logdetective.server.exceptions import KojiInvalidTaskID
28
32
 
29
33
  from logdetective.server.database.models.koji import KojiTaskAnalysis
@@ -42,13 +46,14 @@ from logdetective.server.koji import (
42
46
  from logdetective.remote_log import RemoteLog
43
47
  from logdetective.server.llm import (
44
48
  perform_staged_analysis,
45
- perfrom_analysis,
46
- perform_analyis_stream,
49
+ perform_analysis,
50
+ perform_analysis_stream,
47
51
  )
48
52
  from logdetective.server.gitlab import process_gitlab_job_event
49
53
  from logdetective.server.metric import track_request, add_new_metrics, update_metrics
50
54
  from logdetective.server.models import (
51
55
  BuildLog,
56
+ Config,
52
57
  EmojiHook,
53
58
  JobHook,
54
59
  KojiInstanceConfig,
@@ -56,6 +61,7 @@ from logdetective.server.models import (
56
61
  Response,
57
62
  StagedResponse,
58
63
  TimePeriod,
64
+ ExtractorConfig,
59
65
  )
60
66
  from logdetective.server import plot as plot_engine
61
67
  from logdetective.server.database.models import (
@@ -78,6 +84,89 @@ if sentry_dsn := SERVER_CONFIG.general.sentry_dsn:
78
84
  sentry_sdk.init(dsn=str(sentry_dsn), traces_sample_rate=1.0)
79
85
 
80
86
 
87
+ def initialize_extractors(extractor_config: ExtractorConfig) -> list[Extractor]:
88
+ """Set up extractors based on provided ExtractorConfig."""
89
+ extractors: list[Extractor] = [
90
+ DrainExtractor(
91
+ verbose=extractor_config.verbose,
92
+ max_snippet_len=extractor_config.max_snippet_len,
93
+ max_clusters=extractor_config.max_clusters,
94
+ )
95
+ ]
96
+
97
+ if extractor_config.csgrep:
98
+ extractors.append(
99
+ CSGrepExtractor(
100
+ verbose=extractor_config.verbose,
101
+ max_snippet_len=extractor_config.max_snippet_len,
102
+ )
103
+ )
104
+
105
+ return extractors
106
+
107
+
108
+ class ConnectionManager:
109
+ """
110
+ Manager for all connections and sesssions.
111
+ """
112
+
113
+ koji_connections: dict[str, ClientSession] = {}
114
+ gitlab_connections: dict[str, Gitlab] = {}
115
+ gitlab_http_sessions: dict[str, aiohttp.ClientSession] = {}
116
+
117
+ async def initialize(self, service_config: Config):
118
+ """Initialize all managed objects"""
119
+
120
+ for connection, config in service_config.gitlab.instances.items():
121
+ self.gitlab_connections[connection] = Gitlab(
122
+ url=config.url,
123
+ private_token=config.api_token,
124
+ timeout=config.timeout,
125
+ )
126
+ self.gitlab_http_sessions[connection] = aiohttp.ClientSession(
127
+ base_url=config.url,
128
+ headers={"Authorization": f"Bearer {config.api_token}"},
129
+ timeout=aiohttp.ClientTimeout(
130
+ total=config.timeout,
131
+ connect=3.07,
132
+ ),
133
+ )
134
+ for connection, config in service_config.koji.instances.items():
135
+ self.koji_connections[connection] = ClientSession(baseurl=config.xmlrpc_url)
136
+
137
+ async def close(self):
138
+ """Close all managed http sessions"""
139
+ for session in self.gitlab_http_sessions.values():
140
+
141
+ await session.close()
142
+
143
+
144
+ class KojiCallbackManager:
145
+ """Manages callbacks used by Koji, with callbacks referenced by task id.
146
+
147
+ Multiple callbacks can be assigned to a single task."""
148
+
149
+ _callbacks: defaultdict[int, set[str]]
150
+
151
+ def __init__(self) -> None:
152
+ self._callbacks = defaultdict(set)
153
+
154
+ def register_callback(self, task_id: int, callback: str):
155
+ """Register a callback for a task"""
156
+ self._callbacks[task_id].add(callback)
157
+
158
+ def clear_callbacks(self, task_id: int):
159
+ """Unregister a callback for a task"""
160
+ try:
161
+ del self._callbacks[task_id]
162
+ except KeyError:
163
+ pass
164
+
165
+ def get_callbacks(self, task_id: int) -> set[str]:
166
+ """Get the callbacks for a task"""
167
+ return self._callbacks[task_id]
168
+
169
+
81
170
  @asynccontextmanager
82
171
  async def lifespan(fapp: FastAPI):
83
172
  """
@@ -89,14 +178,31 @@ async def lifespan(fapp: FastAPI):
89
178
  )
90
179
  )
91
180
 
181
+ # General limiter for async requests
182
+ fapp.state.llm_request_limiter = AsyncLimiter(
183
+ max_rate=SERVER_CONFIG.inference.requests_per_minute
184
+ )
185
+
186
+ # Manager for connections and sessions
187
+ fapp.state.connection_manager = ConnectionManager()
188
+
189
+ await fapp.state.connection_manager.initialize(service_config=SERVER_CONFIG)
190
+
191
+ # Set up extractors
192
+ fapp.state.extractors = initialize_extractors(SERVER_CONFIG.extractor)
193
+
194
+ # Koji callbacks
195
+ fapp.state.koji_callback_manager = KojiCallbackManager()
196
+
92
197
  # Ensure that the database is initialized.
93
198
  await logdetective.server.database.base.init()
94
199
 
95
200
  # Start the background task scheduler for collecting emojis
96
- asyncio.create_task(schedule_collect_emojis_task())
201
+ asyncio.create_task(schedule_collect_emojis_task(fapp.state.connection_manager))
97
202
 
98
203
  yield
99
204
 
205
+ await fapp.state.connection_manager.close()
100
206
  await fapp.http.close()
101
207
 
102
208
 
@@ -144,20 +250,24 @@ app = FastAPI(
144
250
  contact={
145
251
  "name": "Log Detective developers",
146
252
  "url": "https://github.com/fedora-copr/logdetective",
147
- "email": "copr-devel@lists.fedorahosted.org"
253
+ "email": "copr-devel@lists.fedorahosted.org",
148
254
  },
149
255
  license_info={
150
256
  "name": "Apache-2.0",
151
257
  "url": "https://www.apache.org/licenses/LICENSE-2.0.html",
152
258
  },
153
259
  version=get_version(),
154
- dependencies=[Depends(requires_token_when_set)], lifespan=lifespan)
260
+ dependencies=[Depends(requires_token_when_set)],
261
+ lifespan=lifespan,
262
+ )
155
263
 
156
264
 
157
265
  @app.post("/analyze", response_model=Response)
158
266
  @track_request()
159
267
  async def analyze_log(
160
- build_log: BuildLog, http_session: aiohttp.ClientSession = Depends(get_http_session)
268
+ build_log: BuildLog,
269
+ request: Request,
270
+ http_session: aiohttp.ClientSession = Depends(get_http_session),
161
271
  ):
162
272
  """Provide endpoint for log file submission and analysis.
163
273
  Request must be in form {"url":"<YOUR_URL_HERE>"}.
@@ -168,13 +278,19 @@ async def analyze_log(
168
278
  remote_log = RemoteLog(build_log.url, http_session)
169
279
  log_text = await remote_log.process_url()
170
280
 
171
- return await perfrom_analysis(log_text)
281
+ return await perform_analysis(
282
+ log_text,
283
+ async_request_limiter=request.app.state.llm_request_limiter,
284
+ extractors=request.app.state.extractors
285
+ )
172
286
 
173
287
 
174
288
  @app.post("/analyze/staged", response_model=StagedResponse)
175
289
  @track_request()
176
290
  async def analyze_log_staged(
177
- build_log: BuildLog, http_session: aiohttp.ClientSession = Depends(get_http_session)
291
+ build_log: BuildLog,
292
+ request: Request,
293
+ http_session: aiohttp.ClientSession = Depends(get_http_session),
178
294
  ):
179
295
  """Provide endpoint for log file submission and analysis.
180
296
  Request must be in form {"url":"<YOUR_URL_HERE>"}.
@@ -185,7 +301,11 @@ async def analyze_log_staged(
185
301
  remote_log = RemoteLog(build_log.url, http_session)
186
302
  log_text = await remote_log.process_url()
187
303
 
188
- return await perform_staged_analysis(log_text)
304
+ return await perform_staged_analysis(
305
+ log_text,
306
+ async_request_limiter=request.app.state.llm_request_limiter,
307
+ extractors=request.app.state.extractors,
308
+ )
189
309
 
190
310
 
191
311
  @app.get(
@@ -245,10 +365,11 @@ async def get_koji_task_analysis(
245
365
  async def analyze_rpmbuild_koji(
246
366
  koji_instance: Annotated[str, Path(title="The Koji instance to use")],
247
367
  task_id: Annotated[int, Path(title="The task ID to analyze")],
368
+ request: Request,
248
369
  x_koji_token: Annotated[str, Header()] = "",
249
370
  x_koji_callback: Annotated[str, Header()] = "",
250
371
  background_tasks: BackgroundTasks = BackgroundTasks(),
251
- ):
372
+ ): # pylint: disable=too-many-arguments disable=too-many-positional-arguments
252
373
  """Provide endpoint for retrieving log file analysis of a Koji task"""
253
374
 
254
375
  try:
@@ -276,16 +397,23 @@ async def analyze_rpmbuild_koji(
276
397
  # Task not yet analyzed or it timed out, so we need to start the
277
398
  # analysis in the background and return a 202 (Accepted) error.
278
399
 
400
+ koji_connection = request.app.state.connection_manager.koji_connections[
401
+ koji_instance
402
+ ]
279
403
  background_tasks.add_task(
280
404
  analyze_koji_task,
281
405
  task_id,
282
406
  koji_instance_config,
407
+ koji_connection,
408
+ request.app.state.llm_request_limiter,
409
+ request.app.state.extractors,
410
+ request.app.state.koji_callback_manager
283
411
  )
284
412
 
285
413
  # If a callback URL is provided, we need to add it to the callbacks
286
414
  # table so that we can notify it when the analysis is complete.
287
415
  if x_koji_callback:
288
- koji_instance_config.register_callback(task_id, x_koji_callback)
416
+ request.app.state.koji_callback_manager.register_callback(task_id, x_koji_callback)
289
417
 
290
418
  response = BasicResponse(
291
419
  status_code=202, content=f"Beginning analysis of task {task_id}"
@@ -301,13 +429,19 @@ async def analyze_rpmbuild_koji(
301
429
  return response
302
430
 
303
431
 
304
- async def analyze_koji_task(task_id: int, koji_instance_config: KojiInstanceConfig):
432
+ async def analyze_koji_task(
433
+ task_id: int,
434
+ koji_instance_config: KojiInstanceConfig,
435
+ koji_connection: ClientSession,
436
+ async_request_limiter: AsyncLimiter,
437
+ extractors: list[Extractor],
438
+ koji_callback_manager: KojiCallbackManager,
439
+ ): # pylint: disable=too-many-arguments disable=too-many-positional-arguments
305
440
  """Analyze a koji task and return the response"""
306
441
 
307
442
  # Get the log text from the koji task
308
- koji_conn = koji_instance_config.get_connection()
309
443
  log_file_name, log_text = await get_failed_log_from_koji_task(
310
- koji_conn, task_id, max_size=SERVER_CONFIG.koji.max_artifact_size
444
+ koji_connection, task_id, max_size=SERVER_CONFIG.koji.max_artifact_size
311
445
  )
312
446
 
313
447
  # We need to handle the metric tracking manually here, because we need
@@ -327,7 +461,11 @@ async def analyze_koji_task(task_id: int, koji_instance_config: KojiInstanceConf
327
461
  task_id=task_id,
328
462
  log_file_name=log_file_name,
329
463
  )
330
- response = await perform_staged_analysis(log_text)
464
+ response = await perform_staged_analysis(
465
+ log_text,
466
+ async_request_limiter=async_request_limiter,
467
+ extractors=extractors
468
+ )
331
469
 
332
470
  # Now that we have the response, we can update the metrics and mark the
333
471
  # koji task analysis as completed.
@@ -335,12 +473,12 @@ async def analyze_koji_task(task_id: int, koji_instance_config: KojiInstanceConf
335
473
  await KojiTaskAnalysis.add_response(task_id, metrics_id)
336
474
 
337
475
  # Notify any callbacks that the analysis is complete.
338
- for callback in koji_instance_config.get_callbacks(task_id):
476
+ for callback in koji_callback_manager.get_callbacks(task_id):
339
477
  LOG.info("Notifying callback %s of task %d completion", callback, task_id)
340
478
  asyncio.create_task(send_koji_callback(callback, task_id))
341
479
 
342
480
  # Now that it's sent, we can clear the callbacks for this task.
343
- koji_instance_config.clear_callbacks(task_id)
481
+ koji_callback_manager.clear_callbacks(task_id)
344
482
 
345
483
  return response
346
484
 
@@ -353,18 +491,18 @@ async def send_koji_callback(callback: str, task_id: int):
353
491
 
354
492
 
355
493
  @app.get("/queue/print")
356
- async def queue_print(msg: str):
494
+ async def queue_print(msg: str, request: Request):
357
495
  """Debug endpoint to test the LLM request queue"""
358
496
  LOG.info("Will print %s", msg)
359
497
 
360
- result = await async_log(msg)
498
+ result = await async_log(msg, request)
361
499
 
362
500
  LOG.info("Printed %s and returned it", result)
363
501
 
364
502
 
365
- async def async_log(msg):
503
+ async def async_log(msg: str, request: Request):
366
504
  """Debug function to test the LLM request queue"""
367
- async with SERVER_CONFIG.inference.get_limiter():
505
+ async with request.app.state.llm_request_limiter:
368
506
  LOG.critical(msg)
369
507
  return msg
370
508
 
@@ -378,7 +516,9 @@ async def get_version_wrapper():
378
516
  @app.post("/analyze/stream", response_class=StreamingResponse)
379
517
  @track_request()
380
518
  async def analyze_log_stream(
381
- build_log: BuildLog, http_session: aiohttp.ClientSession = Depends(get_http_session)
519
+ build_log: BuildLog,
520
+ request: Request,
521
+ http_session: aiohttp.ClientSession = Depends(get_http_session),
382
522
  ):
383
523
  """Stream response endpoint for Logdetective.
384
524
  Request must be in form {"url":"<YOUR_URL_HERE>"}.
@@ -389,7 +529,11 @@ async def analyze_log_stream(
389
529
  remote_log = RemoteLog(build_log.url, http_session)
390
530
  log_text = await remote_log.process_url()
391
531
  try:
392
- stream = perform_analyis_stream(log_text)
532
+ stream = perform_analysis_stream(
533
+ log_text,
534
+ async_request_limiter=request.app.state.llm_request_limiter,
535
+ extractors=request.app.state.extractors,
536
+ )
393
537
  except aiohttp.ClientResponseError as ex:
394
538
  raise HTTPException(
395
539
  status_code=400,
@@ -421,6 +565,7 @@ def is_valid_webhook_secret(forge, x_gitlab_token):
421
565
  async def receive_gitlab_job_event_webhook(
422
566
  job_hook: JobHook,
423
567
  background_tasks: BackgroundTasks,
568
+ request: Request,
424
569
  x_gitlab_instance: Annotated[str | None, Header()],
425
570
  x_gitlab_token: Annotated[str | None, Header()] = None,
426
571
  ):
@@ -441,11 +586,21 @@ async def receive_gitlab_job_event_webhook(
441
586
 
442
587
  # Handle the message in the background so we can return 204 immediately
443
588
  gitlab_cfg = SERVER_CONFIG.gitlab.instances[forge.value]
589
+ gitlab_connection = request.app.state.connection_manager.gitlab_connections[
590
+ forge.value
591
+ ]
592
+ gitlab_http_session = request.app.state.connection_manager.gitlab_http_sessions[
593
+ forge.value
594
+ ]
444
595
  background_tasks.add_task(
445
596
  process_gitlab_job_event,
446
597
  gitlab_cfg,
598
+ gitlab_connection,
599
+ gitlab_http_session,
447
600
  forge,
448
601
  job_hook,
602
+ request.app.state.llm_request_limiter,
603
+ request.app.state.extractors
449
604
  )
450
605
 
451
606
  # No return value or body is required for a webhook.
@@ -467,6 +622,7 @@ async def receive_gitlab_emoji_event_webhook(
467
622
  x_gitlab_token: Annotated[str | None, Header()],
468
623
  emoji_hook: EmojiHook,
469
624
  background_tasks: BackgroundTasks,
625
+ request: Request,
470
626
  ):
471
627
  """Webhook endpoint for receiving emoji event notifications from Gitlab
472
628
  https://docs.gitlab.com/user/project/integrations/webhook_events/#emoji-events
@@ -519,10 +675,14 @@ async def receive_gitlab_emoji_event_webhook(
519
675
  # Inform the lookup table that we are processing this emoji
520
676
  emoji_lookup[key] = False
521
677
 
678
+ gitlab_connection = request.app.state.connection_manager.gitlab_connections[
679
+ x_gitlab_instance
680
+ ]
522
681
  # Create a background task to process the emojis on this Merge Request.
523
682
  background_tasks.add_task(
524
683
  schedule_emoji_collection_for_mr,
525
684
  forge,
685
+ gitlab_connection,
526
686
  emoji_hook.merge_request.target_project_id,
527
687
  emoji_hook.merge_request.iid,
528
688
  background_tasks,
@@ -534,17 +694,21 @@ async def receive_gitlab_emoji_event_webhook(
534
694
 
535
695
 
536
696
  async def schedule_emoji_collection_for_mr(
537
- forge: Forge, project_id: int, mr_iid: int, background_tasks: BackgroundTasks
697
+ forge: Forge,
698
+ gitlab_connection: Gitlab,
699
+ project_id: int,
700
+ mr_iid: int,
701
+ background_tasks: BackgroundTasks,
538
702
  ):
539
703
  """Background task to update the database on emoji reactions"""
540
704
 
541
705
  key = (forge, project_id, mr_iid)
542
706
 
543
707
  # FIXME: Look up the connection from the Forge # pylint: disable=fixme
544
- gitlab_conn = SERVER_CONFIG.gitlab.instances[forge.value].get_connection()
708
+ # gitlab_conn = SERVER_CONFIG.gitlab.instances[forge.value].get_connection()
545
709
 
546
710
  LOG.debug("Looking up emojis for %s, %d, %d", forge, project_id, mr_iid)
547
- await collect_emojis_for_mr(project_id, mr_iid, gitlab_conn)
711
+ await collect_emojis_for_mr(project_id, mr_iid, gitlab_connection)
548
712
 
549
713
  # Check whether we've been asked to re-schedule this lookup because
550
714
  # another request came in while it was processing.
@@ -555,6 +719,7 @@ async def schedule_emoji_collection_for_mr(
555
719
  background_tasks.add_task(
556
720
  schedule_emoji_collection_for_mr,
557
721
  forge,
722
+ gitlab_connection,
558
723
  project_id,
559
724
  mr_iid,
560
725
  background_tasks,
@@ -677,25 +842,27 @@ async def get_metrics(
677
842
  return await handler()
678
843
 
679
844
 
680
- async def collect_emoji_task():
845
+ async def collect_emoji_task(connection_manager: ConnectionManager):
681
846
  """Collect emoji feedback.
682
847
  Query only comments created in the last year.
683
848
  """
684
849
 
685
- for instance in SERVER_CONFIG.gitlab.instances.values():
850
+ for instance_url, instance in SERVER_CONFIG.gitlab.instances.items():
686
851
  LOG.info(
687
852
  "Collect emoji feedback for %s started at %s",
688
853
  instance.url,
689
854
  datetime.datetime.now(datetime.timezone.utc),
690
855
  )
691
- await collect_emojis(instance.get_connection(), TimePeriod(weeks=54))
856
+ await collect_emojis(
857
+ connection_manager.gitlab_connections[instance_url], TimePeriod(weeks=54)
858
+ )
692
859
  LOG.info(
693
860
  "Collect emoji feedback finished at %s",
694
861
  datetime.datetime.now(datetime.timezone.utc),
695
862
  )
696
863
 
697
864
 
698
- async def schedule_collect_emojis_task():
865
+ async def schedule_collect_emojis_task(connection_manager: ConnectionManager):
699
866
  """Schedule the collect_emojis_task to run on a configured interval"""
700
867
  while True:
701
868
  seconds_until_run = SERVER_CONFIG.general.collect_emojis_interval
@@ -703,6 +870,6 @@ async def schedule_collect_emojis_task():
703
870
  await asyncio.sleep(seconds_until_run)
704
871
 
705
872
  try:
706
- await collect_emoji_task()
873
+ await collect_emoji_task(connection_manager=connection_manager)
707
874
  except Exception as e: # pylint: disable=broad-exception-caught
708
875
  LOG.exception("Error in collect_emoji_task: %s", e)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: logdetective
3
- Version: 2.11.0
3
+ Version: 2.13.0
4
4
  Summary: Log using LLM AI to search for build/test failures and provide ideas for fixing these.
5
5
  License: Apache-2.0
6
6
  License-File: LICENSE
@@ -24,7 +24,7 @@ Provides-Extra: server
24
24
  Provides-Extra: server-testing
25
25
  Provides-Extra: testing
26
26
  Requires-Dist: aiohttp (>=3.7.4,<4.0.0)
27
- Requires-Dist: aiolimiter (>=1.0.0,<2.0.0) ; extra == "server"
27
+ Requires-Dist: aiolimiter (>=1.0.0,<2.0.0) ; extra == "server" or extra == "server-testing"
28
28
  Requires-Dist: aioresponses (>=0.7.8,<0.8.0) ; extra == "testing"
29
29
  Requires-Dist: alembic (>=1.13.3,<2.0.0) ; extra == "server" or extra == "server-testing"
30
30
  Requires-Dist: asciidoc[testing] (>=10.2.1,<11.0.0) ; extra == "testing"
@@ -10,9 +10,9 @@ logdetective/prompts.yml,sha256=i3z6Jcb4ScVi7LsxOpDlKiXrcvql3qO_JnLzkAKMn1c,3870
10
10
  logdetective/remote_log.py,sha256=28QvdQiy7RBnd86EKCq_A75P21gSNlCbgxJe5XAe9MA,2258
11
11
  logdetective/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
12
12
  logdetective/server/compressors.py,sha256=y4aFYJ_9CbYdKuAI39Kc9GQSdPN8cSJ2c_VAz3T47EE,5249
13
- logdetective/server/config.py,sha256=cKUmNCJyNyEid0bPTiUjr8CQuBYBab5bC79Axk2h0z8,2525
13
+ logdetective/server/config.py,sha256=dYoqvexnMo8LBXhXezMIEqUwzTsRD-eWvRIFIYNv388,2540
14
14
  logdetective/server/database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
15
- logdetective/server/database/base.py,sha256=HSV2tgye7iYTDzJD1Q5X7_nlLuTMIFP-hRVQMYxngHQ,2073
15
+ logdetective/server/database/base.py,sha256=bqMkhL2D96i_QiSnO5u1FqxYuJJu0m0wXLkqj_A9WBs,2093
16
16
  logdetective/server/database/models/__init__.py,sha256=zoZMCt1_7tewDa6eEIIX_xrdN-tLegSiPNg5NiYaV3o,850
17
17
  logdetective/server/database/models/exceptions.py,sha256=4ED7FSSA1liV9-7VIN2BwUiz6XlmP97Y1loKnsoNdD8,507
18
18
  logdetective/server/database/models/koji.py,sha256=HNWxHYDxf4JN9K2ue8-V8dH-0XY5ZmxqH7Y9lAIbILA,6436
@@ -20,21 +20,21 @@ logdetective/server/database/models/merge_request_jobs.py,sha256=MxiAVKQIsQMbFyl
20
20
  logdetective/server/database/models/metrics.py,sha256=4xsUdbtlp5PI1-iJQc5Dd8EPDgVVplD9hJRWeRDn43k,15443
21
21
  logdetective/server/emoji.py,sha256=zSaYtLpSkpRCXpjMWnHR1bYwkmobMJASZ7YNalrd85U,5274
22
22
  logdetective/server/exceptions.py,sha256=WN715KLL3ya6FiZ95v70VSbNuVhGuHFzxm2OeEPWQCw,981
23
- logdetective/server/gitlab.py,sha256=putpnf8PfGsCZJsqWZA1rMovRGnyagoQmgpKLqtA-aQ,16743
23
+ logdetective/server/gitlab.py,sha256=X9JSotUUlG9bOWYbUNKt9KqLUAj6Uocd2KNpfn35ccU,17192
24
24
  logdetective/server/koji.py,sha256=LG1pRiKUFvYFRKzgQoUG3pUHfcEwMoaMNjUSMKw_pBA,5640
25
- logdetective/server/llm.py,sha256=bmA6LsV80OdO60q4WLoKuehuVDEYq-HhBAYcZeLfrv8,10150
25
+ logdetective/server/llm.py,sha256=wHMxRbAjI0q3osR5mRDR1kqww_6Pkc7JpF1mh9e6Mg8,10855
26
26
  logdetective/server/metric.py,sha256=wLOpgcAch3rwhPA5P2YWUeMNAPsvRGseRjH5HlTb7JM,4529
27
- logdetective/server/models.py,sha256=AJyycAEEl2o6TH4eAqVMlt5woqAB5M8ze2L575leA_I,19835
27
+ logdetective/server/models.py,sha256=iJ-5UgScKKSRL8fRCsM23Z34P3p98LaduwWO-q9rudo,13041
28
28
  logdetective/server/plot.py,sha256=8LERgY3vQckaHZV2PZfOrZT8CjCAiji57QCmRW24Rfo,14697
29
- logdetective/server/server.py,sha256=JueU-5c8t9h1CZy4gtoEeT8VSEirpeS0K3wrfqTPvAc,25381
29
+ logdetective/server/server.py,sha256=AM10P72tc_7N0GhH_N7msFhLr7ZGNgIfgTxt2sjasVE,30982
30
30
  logdetective/server/templates/base_response.html.j2,sha256=BJGGV_Xb0Lnue8kq32oG9lI5CQDf9vce7HMYsP-Pvb4,2040
31
31
  logdetective/server/templates/gitlab_full_comment.md.j2,sha256=4UujUzl3lmdbNEADsxn3HVrjfUiUu2FvUlp9MDFGXQI,2321
32
32
  logdetective/server/templates/gitlab_short_comment.md.j2,sha256=2krnMlGqqju2V_6pE0UqUR1P674OFaeX5BMyY5htTOQ,2022
33
33
  logdetective/server/utils.py,sha256=0BZ8WmzXNEtkUty1kOyFbBxDZWL0Icc8BUrxuHw9uvs,4015
34
34
  logdetective/skip_snippets.yml,sha256=reGlhPPCo06nNUJWiC2LY-OJOoPdcyOB7QBTSMeh0eg,487
35
35
  logdetective/utils.py,sha256=yalhySOF_Gzmqx_Ft9qad3TplAfZ6LOmauGXEJfKWiE,9803
36
- logdetective-2.11.0.dist-info/METADATA,sha256=SdXBkYlSoiVXhgPiM23luYQa0Y_BCX_el_mxTdJc0Zw,23273
37
- logdetective-2.11.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
38
- logdetective-2.11.0.dist-info/entry_points.txt,sha256=3K_vXja6PmcA8sNdUi63WdImeiNhVZcEGPTaoJmltfA,63
39
- logdetective-2.11.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
40
- logdetective-2.11.0.dist-info/RECORD,,
36
+ logdetective-2.13.0.dist-info/METADATA,sha256=uwiSy7i6qIvLUxz-J5hCqzlLWWqmdAsi0IIvrGgQmMs,23302
37
+ logdetective-2.13.0.dist-info/WHEEL,sha256=3ny-bZhpXrU6vSQ1UPG34FoxZBp3lVcvK0LkgUz6VLk,88
38
+ logdetective-2.13.0.dist-info/entry_points.txt,sha256=3K_vXja6PmcA8sNdUi63WdImeiNhVZcEGPTaoJmltfA,63
39
+ logdetective-2.13.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
40
+ logdetective-2.13.0.dist-info/RECORD,,
@@ -1,4 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: poetry-core 2.2.1
2
+ Generator: poetry-core 2.3.0
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any