mailshift 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mailshift/__init__.py +1 -0
- mailshift/config/__init__.py +1 -0
- mailshift/config/config.py +279 -0
- mailshift/core/__init__.py +1 -0
- mailshift/core/analyzers/__init__.py +1 -0
- mailshift/core/analyzers/base.py +8 -0
- mailshift/core/analyzers/fast.py +123 -0
- mailshift/core/analyzers/pro.py +529 -0
- mailshift/core/engine.py +842 -0
- mailshift/core/session.py +268 -0
- mailshift/db/__init__.py +1 -0
- mailshift/db/database.py +199 -0
- mailshift/main.py +902 -0
- mailshift/models/__init__.py +1 -0
- mailshift/models/models.py +47 -0
- mailshift/ui/__init__.py +1 -0
- mailshift/ui/cli.py +919 -0
- mailshift/ui/styles.py +96 -0
- mailshift/utils/__init__.py +1 -0
- mailshift/utils/hardware.py +572 -0
- mailshift/utils/history.py +358 -0
- mailshift/utils/logger.py +78 -0
- mailshift/utils/paths.py +8 -0
- mailshift/utils/power_user_settings.py +85 -0
- mailshift/utils/unsubscribe.py +119 -0
- mailshift/utils/updater.py +136 -0
- mailshift/utils/worker_profile_store.py +284 -0
- mailshift-1.0.0.dist-info/METADATA +258 -0
- mailshift-1.0.0.dist-info/RECORD +32 -0
- mailshift-1.0.0.dist-info/WHEEL +4 -0
- mailshift-1.0.0.dist-info/entry_points.txt +2 -0
- mailshift-1.0.0.dist-info/licenses/LICENSE +674 -0
mailshift/core/engine.py
ADDED
|
@@ -0,0 +1,842 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import email
|
|
4
|
+
import email.header
|
|
5
|
+
import imaplib
|
|
6
|
+
import re
|
|
7
|
+
import socket
|
|
8
|
+
import ssl
|
|
9
|
+
import time
|
|
10
|
+
import threading
|
|
11
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError
|
|
12
|
+
from email.message import Message
|
|
13
|
+
from threading import Lock
|
|
14
|
+
from typing import Callable, Optional, TypeAlias
|
|
15
|
+
|
|
16
|
+
from bs4 import BeautifulSoup
|
|
17
|
+
|
|
18
|
+
from ..models.models import MailMeta, ScanResult, ScanStats
|
|
19
|
+
from ..config.config import AppConfig, IMAPConfig, Mode, Provider, RateLimitConfig
|
|
20
|
+
from ..utils.logger import log
|
|
21
|
+
|
|
22
|
+
# ---------------------------------------------------------------------------
|
|
23
|
+
# Type Aliases (Clean Code Optimizasyonu)
|
|
24
|
+
# ---------------------------------------------------------------------------
|
|
25
|
+
IMAPConnection: TypeAlias = imaplib.IMAP4 | imaplib.IMAP4_SSL
|
|
26
|
+
|
|
27
|
+
# ---------------------------------------------------------------------------
|
|
28
|
+
# Constants
|
|
29
|
+
# ---------------------------------------------------------------------------
|
|
30
|
+
# 25 KB altındaki maillerin gövdesi, header ile aynı anda çekilir.
|
|
31
|
+
# (Double fetch maliyetini düşürmek için eklendi)
|
|
32
|
+
SMALL_MAIL_THRESHOLD_BYTES = 25 * 1024
|
|
33
|
+
|
|
34
|
+
_UID_RE = re.compile(r"UID\s+(\d+)", re.IGNORECASE)
|
|
35
|
+
_SIZE_RE = re.compile(r"RFC822\.SIZE\s+(\d+)", re.IGNORECASE)
|
|
36
|
+
_UNSUB_URL_RE = re.compile(r"<(https?://[^>]+)>", re.IGNORECASE)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# ---------------------------------------------------------------------------
|
|
40
|
+
# Utilities
|
|
41
|
+
# ---------------------------------------------------------------------------
|
|
42
|
+
|
|
43
|
+
def chunk_list(lst: list, n: int):
|
|
44
|
+
"""Yield successive *n*-sized chunks from *lst*."""
|
|
45
|
+
for i in range(0, len(lst), n):
|
|
46
|
+
yield lst[i : i + n]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _connect(cfg: IMAPConfig, timeout: int = 30) -> IMAPConnection:
|
|
50
|
+
"""Open an IMAP connection with an explicit socket timeout."""
|
|
51
|
+
socket.setdefaulttimeout(timeout)
|
|
52
|
+
conn = (
|
|
53
|
+
imaplib.IMAP4_SSL(cfg.host, cfg.port, ssl_context=ssl.create_default_context())
|
|
54
|
+
if cfg.use_ssl
|
|
55
|
+
else imaplib.IMAP4(cfg.host, cfg.port)
|
|
56
|
+
)
|
|
57
|
+
conn.login(cfg.username, cfg.password.get_secret_value())
|
|
58
|
+
return conn
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _decode_header_value(raw: str | bytes | None) -> str:
|
|
62
|
+
if not raw:
|
|
63
|
+
return ""
|
|
64
|
+
raw = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw
|
|
65
|
+
return " ".join(
|
|
66
|
+
p.decode(c or "utf-8", errors="replace") if isinstance(p, bytes) else str(p)
|
|
67
|
+
for p, c in email.header.decode_header(raw)
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _extract_unsubscribe_url(msg: Message) -> str:
|
|
72
|
+
"""Extract the first HTTP(S) URL from the List-Unsubscribe header."""
|
|
73
|
+
raw = msg.get("List-Unsubscribe", "")
|
|
74
|
+
if not raw:
|
|
75
|
+
return ""
|
|
76
|
+
match = _UNSUB_URL_RE.search(raw)
|
|
77
|
+
return match.group(1) if match else ""
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _has_attachment(msg: Message) -> bool:
|
|
81
|
+
parts = msg.walk() if msg.is_multipart() else [msg]
|
|
82
|
+
return any(
|
|
83
|
+
"attachment" in str(p.get("Content-Disposition", "")).lower() for p in parts
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _extract_body_preview(msg: Message, max_chars: int = 500) -> str:
|
|
88
|
+
text_body, html_body = "", ""
|
|
89
|
+
for part in msg.walk() if msg.is_multipart() else [msg]:
|
|
90
|
+
if "attachment" in str(part.get("Content-Disposition", "")).lower():
|
|
91
|
+
continue
|
|
92
|
+
ct = part.get_content_type()
|
|
93
|
+
if ct in ("text/plain", "text/html") and (
|
|
94
|
+
payload := part.get_payload(decode=True)
|
|
95
|
+
):
|
|
96
|
+
decoded = payload.decode(
|
|
97
|
+
part.get_content_charset() or "utf-8", errors="replace"
|
|
98
|
+
)
|
|
99
|
+
if ct == "text/plain" and not text_body:
|
|
100
|
+
text_body = decoded
|
|
101
|
+
elif ct == "text/html" and not html_body:
|
|
102
|
+
html_body = decoded
|
|
103
|
+
|
|
104
|
+
# HTML Parsing Optimizasyonu: CPU darboğazını önlemek için lxml kullanıyoruz
|
|
105
|
+
try:
|
|
106
|
+
final_text = text_body or BeautifulSoup(
|
|
107
|
+
html_body, "lxml"
|
|
108
|
+
).get_text(separator=" ", strip=True)
|
|
109
|
+
except Exception:
|
|
110
|
+
# lxml kurulu değilse veya patlarsa varsayılan html.parser'a düş
|
|
111
|
+
final_text = text_body or BeautifulSoup(
|
|
112
|
+
html_body, "html.parser"
|
|
113
|
+
).get_text(separator=" ", strip=True)
|
|
114
|
+
|
|
115
|
+
return final_text[:max_chars]
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# ---------------------------------------------------------------------------
|
|
119
|
+
# Retry helper
|
|
120
|
+
# ---------------------------------------------------------------------------
|
|
121
|
+
|
|
122
|
+
def _with_retry(
|
|
123
|
+
fn: Callable,
|
|
124
|
+
rl: RateLimitConfig,
|
|
125
|
+
label: str = "",
|
|
126
|
+
on_error: Optional[Callable[[Exception, int], None]] = None,
|
|
127
|
+
):
|
|
128
|
+
"""Call *fn()* up to *rl.max_retries* times with exponential back-off."""
|
|
129
|
+
delay = rl.retry_backoff
|
|
130
|
+
last_exc: Exception | None = None
|
|
131
|
+
for attempt in range(1, rl.max_retries + 1):
|
|
132
|
+
try:
|
|
133
|
+
return fn()
|
|
134
|
+
except Exception as exc:
|
|
135
|
+
last_exc = exc
|
|
136
|
+
if on_error:
|
|
137
|
+
try:
|
|
138
|
+
on_error(exc, attempt)
|
|
139
|
+
except Exception as hook_exc:
|
|
140
|
+
log.warning(
|
|
141
|
+
f"Retry hook failed for '{label}' on attempt {attempt}: "
|
|
142
|
+
f"{hook_exc}"
|
|
143
|
+
)
|
|
144
|
+
if attempt < rl.max_retries:
|
|
145
|
+
log.warning(
|
|
146
|
+
f"Retry {attempt}/{rl.max_retries} for '{label}' "
|
|
147
|
+
f"after {delay:.1f}s | {exc}"
|
|
148
|
+
)
|
|
149
|
+
time.sleep(delay)
|
|
150
|
+
delay *= 2 # exponential back-off
|
|
151
|
+
else:
|
|
152
|
+
log.error(
|
|
153
|
+
f"All {rl.max_retries} retries exhausted for '{label}': {exc}"
|
|
154
|
+
)
|
|
155
|
+
raise last_exc # type: ignore[misc]
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
# ---------------------------------------------------------------------------
|
|
159
|
+
# Bulk IMAP fetch
|
|
160
|
+
# ---------------------------------------------------------------------------
|
|
161
|
+
|
|
162
|
+
def _fetch_mails_bulk(
|
|
163
|
+
conn: IMAPConnection,
|
|
164
|
+
uids: list[str],
|
|
165
|
+
fetch_body: bool = False,
|
|
166
|
+
max_body_chars: int = 500,
|
|
167
|
+
) -> list[MailMeta]:
|
|
168
|
+
"""Fetch a single pre-sized chunk of UIDs; raises on IMAP error."""
|
|
169
|
+
if not uids:
|
|
170
|
+
return []
|
|
171
|
+
uid_str = ",".join(uids)
|
|
172
|
+
query = "(RFC822.SIZE RFC822)" if fetch_body else "(RFC822.SIZE RFC822.HEADER)"
|
|
173
|
+
|
|
174
|
+
status, data = conn.uid("fetch", uid_str, query)
|
|
175
|
+
if status != "OK" or not data:
|
|
176
|
+
raise RuntimeError(f"FETCH returned status={status}")
|
|
177
|
+
|
|
178
|
+
results: list[MailMeta] = []
|
|
179
|
+
for item in data:
|
|
180
|
+
if not isinstance(item, tuple):
|
|
181
|
+
continue
|
|
182
|
+
try:
|
|
183
|
+
msg_info = item[0].decode(errors="replace")
|
|
184
|
+
uid_match = _UID_RE.search(msg_info)
|
|
185
|
+
parsed_uid = uid_match.group(1) if uid_match else None
|
|
186
|
+
|
|
187
|
+
size_match = _SIZE_RE.search(msg_info)
|
|
188
|
+
size_bytes = int(size_match.group(1)) if size_match else 0
|
|
189
|
+
|
|
190
|
+
msg = email.message_from_bytes(item[1])
|
|
191
|
+
|
|
192
|
+
if parsed_uid:
|
|
193
|
+
results.append(
|
|
194
|
+
MailMeta(
|
|
195
|
+
uid=parsed_uid,
|
|
196
|
+
subject=_decode_header_value(msg.get("Subject")),
|
|
197
|
+
sender=_decode_header_value(msg.get("From")),
|
|
198
|
+
date=_decode_header_value(msg.get("Date")),
|
|
199
|
+
size_bytes=size_bytes,
|
|
200
|
+
body_preview=(
|
|
201
|
+
_extract_body_preview(msg, max_body_chars)
|
|
202
|
+
if fetch_body
|
|
203
|
+
else ""
|
|
204
|
+
),
|
|
205
|
+
has_attachment=_has_attachment(msg),
|
|
206
|
+
unsubscribe_url=_extract_unsubscribe_url(msg),
|
|
207
|
+
)
|
|
208
|
+
)
|
|
209
|
+
except Exception:
|
|
210
|
+
continue
|
|
211
|
+
return results
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
# ---------------------------------------------------------------------------
|
|
215
|
+
# MailEngine
|
|
216
|
+
# ---------------------------------------------------------------------------
|
|
217
|
+
|
|
218
|
+
class MailEngine:
|
|
219
|
+
def __init__(self, cfg: AppConfig) -> None:
|
|
220
|
+
self.cfg = cfg
|
|
221
|
+
self._rl: RateLimitConfig = cfg.rate_limit
|
|
222
|
+
self._conn: Optional[IMAPConnection] = None
|
|
223
|
+
|
|
224
|
+
# ------------------------------------------------------------------
|
|
225
|
+
# Connection management
|
|
226
|
+
# ------------------------------------------------------------------
|
|
227
|
+
|
|
228
|
+
def connect(self) -> None:
|
|
229
|
+
log.info(
|
|
230
|
+
f"Connecting to IMAP server at "
|
|
231
|
+
f"{self.cfg.imap.host}:{self.cfg.imap.port} "
|
|
232
|
+
f"(timeout={self._rl.connect_timeout}s)"
|
|
233
|
+
)
|
|
234
|
+
self._conn = _connect(self.cfg.imap, timeout=self._rl.connect_timeout)
|
|
235
|
+
self._conn.select("INBOX")
|
|
236
|
+
log.info("IMAP connection established and INBOX selected")
|
|
237
|
+
|
|
238
|
+
def disconnect(self) -> None:
|
|
239
|
+
if self._conn:
|
|
240
|
+
log.info("Disconnecting from IMAP server")
|
|
241
|
+
try:
|
|
242
|
+
self._conn.logout()
|
|
243
|
+
except Exception as e:
|
|
244
|
+
log.warning(f"Error disconnecting: {e}")
|
|
245
|
+
self._conn = None
|
|
246
|
+
|
|
247
|
+
def _is_connection_error(self, exc: Exception) -> bool:
|
|
248
|
+
if isinstance(exc, (imaplib.IMAP4.abort, ssl.SSLError, socket.timeout, OSError)):
|
|
249
|
+
return True
|
|
250
|
+
msg = str(exc).lower()
|
|
251
|
+
return any(
|
|
252
|
+
token in msg
|
|
253
|
+
for token in (
|
|
254
|
+
"eof",
|
|
255
|
+
"socket",
|
|
256
|
+
"connection",
|
|
257
|
+
"broken pipe",
|
|
258
|
+
"timed out",
|
|
259
|
+
"connection reset",
|
|
260
|
+
"imap",
|
|
261
|
+
)
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
def _recover_connection(self, exc: Exception, label: str, attempt: int) -> None:
|
|
265
|
+
if not self._is_connection_error(exc):
|
|
266
|
+
return
|
|
267
|
+
self._force_reconnect(label, attempt)
|
|
268
|
+
|
|
269
|
+
def _force_reconnect(self, label: str, attempt: int) -> None:
|
|
270
|
+
log.warning(f"Forcing IMAP reconnect during '{label}' (attempt {attempt})")
|
|
271
|
+
old_conn = self._conn
|
|
272
|
+
self._conn = None
|
|
273
|
+
|
|
274
|
+
if old_conn:
|
|
275
|
+
try:
|
|
276
|
+
old_conn.logout()
|
|
277
|
+
except Exception:
|
|
278
|
+
pass
|
|
279
|
+
|
|
280
|
+
self._conn = _connect(self.cfg.imap, timeout=self._rl.connect_timeout)
|
|
281
|
+
self._conn.select("INBOX")
|
|
282
|
+
log.info("IMAP reconnect successful; INBOX re-selected")
|
|
283
|
+
|
|
284
|
+
def __enter__(self) -> MailEngine:
|
|
285
|
+
self.connect()
|
|
286
|
+
return self
|
|
287
|
+
|
|
288
|
+
def __exit__(self, *_) -> None:
|
|
289
|
+
self.disconnect()
|
|
290
|
+
|
|
291
|
+
# ------------------------------------------------------------------
|
|
292
|
+
# UID listing
|
|
293
|
+
# ------------------------------------------------------------------
|
|
294
|
+
|
|
295
|
+
def _build_search_criteria(self) -> list[str]:
|
|
296
|
+
criteria = ["ALL"]
|
|
297
|
+
if self.cfg.since:
|
|
298
|
+
criteria.extend(["SINCE", self.cfg.since])
|
|
299
|
+
if self.cfg.before:
|
|
300
|
+
criteria.extend(["BEFORE", self.cfg.before])
|
|
301
|
+
return criteria
|
|
302
|
+
|
|
303
|
+
def list_uids(self) -> list[str]:
|
|
304
|
+
assert self._conn, "Not connected"
|
|
305
|
+
search_criteria = self._build_search_criteria()
|
|
306
|
+
log.info(f"Searching messages in INBOX with criteria: {' '.join(search_criteria)}")
|
|
307
|
+
status, data = self._conn.uid("search", None, *search_criteria)
|
|
308
|
+
if status != "OK" or not data or not data[0]:
|
|
309
|
+
log.warning("No messages found or search failed")
|
|
310
|
+
return []
|
|
311
|
+
uids = data[0].decode().split()[::-1]
|
|
312
|
+
log.info(f"Found {len(uids)} total messages in INBOX")
|
|
313
|
+
return uids[: self.cfg.scan_limit] if self.cfg.scan_limit else uids
|
|
314
|
+
|
|
315
|
+
# ------------------------------------------------------------------
|
|
316
|
+
# Header / body fetching — rate-limit + retry + checkpoint
|
|
317
|
+
# ------------------------------------------------------------------
|
|
318
|
+
|
|
319
|
+
def fetch_headers_concurrent(
|
|
320
|
+
self,
|
|
321
|
+
uids: list[str],
|
|
322
|
+
progress_cb: Optional[Callable[[MailMeta], None]] = None,
|
|
323
|
+
resume: bool = False,
|
|
324
|
+
) -> list[MailMeta]:
|
|
325
|
+
assert self._conn, "Not connected"
|
|
326
|
+
from ..db.database import mark_uids_fetched, get_fetched_uids, save_mails_cache
|
|
327
|
+
|
|
328
|
+
rl = self._rl
|
|
329
|
+
|
|
330
|
+
if resume:
|
|
331
|
+
done = get_fetched_uids()
|
|
332
|
+
pending = [u for u in uids if u not in done]
|
|
333
|
+
log.info(
|
|
334
|
+
f"Resume mode: {len(done)} already fetched, "
|
|
335
|
+
f"{len(pending)} remaining"
|
|
336
|
+
)
|
|
337
|
+
else:
|
|
338
|
+
pending = uids
|
|
339
|
+
|
|
340
|
+
results: list[MailMeta] = []
|
|
341
|
+
chunks = list(chunk_list(pending, rl.fetch_chunk_size))
|
|
342
|
+
total_chunks = len(chunks)
|
|
343
|
+
chunk_delay = max(0.0, rl.chunk_delay)
|
|
344
|
+
|
|
345
|
+
log.info(
|
|
346
|
+
f"Fetching headers for {len(pending)} messages "
|
|
347
|
+
f"in {total_chunks} chunks of {rl.fetch_chunk_size}"
|
|
348
|
+
)
|
|
349
|
+
|
|
350
|
+
for chunk_idx, chunk in enumerate(chunks, start=1):
|
|
351
|
+
chunk_started = time.perf_counter()
|
|
352
|
+
had_retry = False
|
|
353
|
+
|
|
354
|
+
def _fetch(c=chunk):
|
|
355
|
+
return _fetch_mails_bulk(
|
|
356
|
+
self._conn,
|
|
357
|
+
c,
|
|
358
|
+
fetch_body=False,
|
|
359
|
+
max_body_chars=self.cfg.ollama.max_body_chars,
|
|
360
|
+
)
|
|
361
|
+
|
|
362
|
+
def _on_fetch_error(exc: Exception, attempt: int, lbl: str) -> None:
|
|
363
|
+
nonlocal had_retry
|
|
364
|
+
had_retry = True
|
|
365
|
+
self._recover_connection(exc, lbl, attempt)
|
|
366
|
+
|
|
367
|
+
try:
|
|
368
|
+
chunk_results = _with_retry(
|
|
369
|
+
_fetch, rl, label=f"fetch chunk {chunk_idx}/{total_chunks}",
|
|
370
|
+
on_error=lambda exc, attempt, lbl=(
|
|
371
|
+
f"fetch chunk {chunk_idx}/{total_chunks}"
|
|
372
|
+
): _on_fetch_error(exc, attempt, lbl),
|
|
373
|
+
)
|
|
374
|
+
except Exception as exc:
|
|
375
|
+
log.error(f"Chunk {chunk_idx}/{total_chunks} failed permanently: {exc}")
|
|
376
|
+
chunk_results = []
|
|
377
|
+
|
|
378
|
+
# Dinamik Fetch Optimizasyonu: Küçük boyutlu maillerin body'sini hemen çek
|
|
379
|
+
small_uids = [
|
|
380
|
+
m.uid for m in chunk_results
|
|
381
|
+
if 0 < m.size_bytes <= SMALL_MAIL_THRESHOLD_BYTES
|
|
382
|
+
]
|
|
383
|
+
if small_uids:
|
|
384
|
+
try:
|
|
385
|
+
small_bodies = _fetch_mails_bulk(
|
|
386
|
+
self._conn, small_uids, fetch_body=True,
|
|
387
|
+
max_body_chars=self.cfg.ollama.max_body_chars
|
|
388
|
+
)
|
|
389
|
+
body_map = {m.uid: m.body_preview for m in small_bodies}
|
|
390
|
+
for meta in chunk_results:
|
|
391
|
+
if meta.uid in body_map:
|
|
392
|
+
meta.body_preview = body_map[meta.uid]
|
|
393
|
+
except Exception as e:
|
|
394
|
+
log.warning(f"Failed dynamic body fetch for chunk {chunk_idx}: {e}")
|
|
395
|
+
|
|
396
|
+
chunk_elapsed = time.perf_counter() - chunk_started
|
|
397
|
+
|
|
398
|
+
for meta in chunk_results:
|
|
399
|
+
results.append(meta)
|
|
400
|
+
if progress_cb:
|
|
401
|
+
progress_cb(meta)
|
|
402
|
+
|
|
403
|
+
if chunk_results:
|
|
404
|
+
mark_uids_fetched([m.uid for m in chunk_results])
|
|
405
|
+
save_mails_cache(chunk_results, batch_size=rl.db_batch_size)
|
|
406
|
+
|
|
407
|
+
if had_retry:
|
|
408
|
+
base = max(chunk_delay, rl.chunk_delay)
|
|
409
|
+
chunk_delay = min(0.5, (base * 1.5) + 0.01)
|
|
410
|
+
elif chunk_elapsed < 0.4 and chunk_delay > 0:
|
|
411
|
+
chunk_delay = max(0.0, (chunk_delay * 0.85) - 0.005)
|
|
412
|
+
|
|
413
|
+
if chunk_idx < total_chunks and chunk_delay > 0:
|
|
414
|
+
time.sleep(chunk_delay)
|
|
415
|
+
|
|
416
|
+
log.info(f"Successfully fetched {len(results)} message headers")
|
|
417
|
+
return results
|
|
418
|
+
|
|
419
|
+
def fetch_body_for_cached_mails(
|
|
420
|
+
self,
|
|
421
|
+
mails: list[MailMeta],
|
|
422
|
+
progress_cb: Optional[Callable[[MailMeta], None]] = None,
|
|
423
|
+
) -> list[MailMeta]:
|
|
424
|
+
assert self._conn, "Not connected"
|
|
425
|
+
from ..db.database import save_mails_cache
|
|
426
|
+
|
|
427
|
+
rl = self._rl
|
|
428
|
+
# Zaten body_preview'ı dolu olanları elediğimizden emin oluyoruz
|
|
429
|
+
uids = [m.uid for m in mails if not m.body_preview]
|
|
430
|
+
if not uids:
|
|
431
|
+
return mails
|
|
432
|
+
|
|
433
|
+
fetched_dict: dict[str, str] = {}
|
|
434
|
+
chunks = list(chunk_list(uids, rl.fetch_chunk_size))
|
|
435
|
+
total_chunks = len(chunks)
|
|
436
|
+
|
|
437
|
+
log.info(
|
|
438
|
+
f"Fetching bodies for {len(uids)} cached messages "
|
|
439
|
+
f"in {total_chunks} chunks"
|
|
440
|
+
)
|
|
441
|
+
|
|
442
|
+
for chunk_idx, chunk in enumerate(chunks, start=1):
|
|
443
|
+
def _fetch(c=chunk):
|
|
444
|
+
return _fetch_mails_bulk(
|
|
445
|
+
self._conn,
|
|
446
|
+
c,
|
|
447
|
+
fetch_body=True,
|
|
448
|
+
max_body_chars=self.cfg.ollama.max_body_chars,
|
|
449
|
+
)
|
|
450
|
+
|
|
451
|
+
try:
|
|
452
|
+
chunk_results = _with_retry(
|
|
453
|
+
_fetch, rl, label=f"body chunk {chunk_idx}/{total_chunks}",
|
|
454
|
+
on_error=lambda exc, attempt, lbl=(
|
|
455
|
+
f"body chunk {chunk_idx}/{total_chunks}"
|
|
456
|
+
): self._recover_connection(exc, lbl, attempt),
|
|
457
|
+
)
|
|
458
|
+
except Exception as exc:
|
|
459
|
+
log.error(f"Body chunk {chunk_idx}/{total_chunks} failed permanently: {exc}")
|
|
460
|
+
chunk_results = []
|
|
461
|
+
|
|
462
|
+
for meta in chunk_results:
|
|
463
|
+
fetched_dict[meta.uid] = meta.body_preview
|
|
464
|
+
|
|
465
|
+
if chunk_idx < total_chunks and rl.chunk_delay > 0:
|
|
466
|
+
time.sleep(rl.chunk_delay)
|
|
467
|
+
|
|
468
|
+
for mail in mails:
|
|
469
|
+
if mail.uid in fetched_dict:
|
|
470
|
+
mail.body_preview = fetched_dict[mail.uid]
|
|
471
|
+
if progress_cb:
|
|
472
|
+
progress_cb(mail)
|
|
473
|
+
|
|
474
|
+
save_mails_cache(mails, batch_size=rl.db_batch_size)
|
|
475
|
+
return mails
|
|
476
|
+
|
|
477
|
+
# ------------------------------------------------------------------
|
|
478
|
+
# Analysis
|
|
479
|
+
# ------------------------------------------------------------------
|
|
480
|
+
|
|
481
|
+
def analyze(
|
|
482
|
+
self,
|
|
483
|
+
mails: list[MailMeta],
|
|
484
|
+
progress_cb: Optional[Callable[[ScanResult], None]] = None,
|
|
485
|
+
cancel_event: Optional[threading.Event] = None,
|
|
486
|
+
) -> tuple[list[ScanResult], ScanStats]:
|
|
487
|
+
from .analyzers.fast import fast_analyze
|
|
488
|
+
from .analyzers.pro import pro_analyze
|
|
489
|
+
from ..utils.hardware import calculate_optimal_workers
|
|
490
|
+
|
|
491
|
+
stats = ScanStats()
|
|
492
|
+
stats_lock = Lock()
|
|
493
|
+
need_llm = self.cfg.mode == Mode.PRO
|
|
494
|
+
|
|
495
|
+
# ── Phase 1: Fast heuristic scan ──
|
|
496
|
+
fast_results: list[ScanResult] = []
|
|
497
|
+
for meta in mails:
|
|
498
|
+
if cancel_event and cancel_event.is_set():
|
|
499
|
+
break
|
|
500
|
+
res = fast_analyze(meta)
|
|
501
|
+
fast_results.append(res)
|
|
502
|
+
|
|
503
|
+
if not need_llm or (cancel_event and cancel_event.is_set()):
|
|
504
|
+
for res in fast_results:
|
|
505
|
+
self._record_stats(stats, stats_lock, res, progress_cb)
|
|
506
|
+
return fast_results, stats
|
|
507
|
+
|
|
508
|
+
# ── Phase 2: Batch fetch bodies only for SIL candidates ──
|
|
509
|
+
sil_candidates = [r for r in fast_results if r.decision == "SIL"]
|
|
510
|
+
tut_results = [r for r in fast_results if r.decision == "TUT"]
|
|
511
|
+
|
|
512
|
+
need_body_mails = [r.mail for r in sil_candidates if not r.mail.body_preview]
|
|
513
|
+
if need_body_mails:
|
|
514
|
+
log.info(f"analyze(): Fetching bodies for {len(need_body_mails)} candidates")
|
|
515
|
+
self.fetch_body_for_cached_mails(need_body_mails)
|
|
516
|
+
|
|
517
|
+
# ── Phase 3: Parallel LLM verification ──
|
|
518
|
+
max_workers = calculate_optimal_workers(
|
|
519
|
+
self.cfg.ollama.model,
|
|
520
|
+
self.cfg.mode.value,
|
|
521
|
+
manual_workers=self.cfg.max_workers,
|
|
522
|
+
backend=self.cfg.llm_backend
|
|
523
|
+
)
|
|
524
|
+
|
|
525
|
+
llm_results: list[ScanResult | None] = [None] * len(sil_candidates)
|
|
526
|
+
|
|
527
|
+
def _llm_worker(idx: int, candidate: ScanResult) -> tuple[int, ScanResult]:
|
|
528
|
+
if cancel_event and cancel_event.is_set():
|
|
529
|
+
return idx, ScanResult(mail=candidate.mail, decision="TUT", reason="cancelled")
|
|
530
|
+
|
|
531
|
+
llm_cfg = self.cfg.lm_studio if self.cfg.llm_backend == "lm_studio" else self.cfg.ollama
|
|
532
|
+
# Timeout mekanizması pro_analyze içinde HTTP client seviyesinde ele alınmalıdır.
|
|
533
|
+
res = pro_analyze(
|
|
534
|
+
candidate.mail,
|
|
535
|
+
llm_cfg,
|
|
536
|
+
backend=self.cfg.llm_backend,
|
|
537
|
+
fast_reason=candidate.reason,
|
|
538
|
+
cancel_event=cancel_event
|
|
539
|
+
)
|
|
540
|
+
return idx, res
|
|
541
|
+
|
|
542
|
+
# Model takılmalarını engellemek için agresif timeout takibi
|
|
543
|
+
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
544
|
+
future_to_idx = {
|
|
545
|
+
executor.submit(_llm_worker, i, c): i
|
|
546
|
+
for i, c in enumerate(sil_candidates)
|
|
547
|
+
}
|
|
548
|
+
# Eğer local model (VRAM dolması vs.) yanıt vermezse worker kilitlenmesini önle
|
|
549
|
+
timeout_limit = 60.0
|
|
550
|
+
|
|
551
|
+
for future in as_completed(future_to_idx):
|
|
552
|
+
idx = future_to_idx[future]
|
|
553
|
+
try:
|
|
554
|
+
# Timeout limitini burada da uyguluyoruz
|
|
555
|
+
_, res = future.result(timeout=timeout_limit)
|
|
556
|
+
llm_results[idx] = res
|
|
557
|
+
except TimeoutError:
|
|
558
|
+
log.error(f"LLM worker timeout ({timeout_limit}s) for candidate {idx}")
|
|
559
|
+
llm_results[idx] = ScanResult(
|
|
560
|
+
mail=sil_candidates[idx].mail,
|
|
561
|
+
decision="TUT",
|
|
562
|
+
reason="analyze-timeout"
|
|
563
|
+
)
|
|
564
|
+
except Exception as exc:
|
|
565
|
+
log.warning(f"LLM worker failed for candidate {idx}: {exc}")
|
|
566
|
+
llm_results[idx] = ScanResult(
|
|
567
|
+
mail=sil_candidates[idx].mail,
|
|
568
|
+
decision="TUT",
|
|
569
|
+
reason=f"analyze-error:{exc}"
|
|
570
|
+
)
|
|
571
|
+
|
|
572
|
+
final_results = tut_results + [r for r in llm_results if r is not None]
|
|
573
|
+
for res in final_results:
|
|
574
|
+
self._record_stats(stats, stats_lock, res, progress_cb)
|
|
575
|
+
|
|
576
|
+
return final_results, stats
|
|
577
|
+
|
|
578
|
+
def _record_stats(self, stats: ScanStats, lock: Lock, res: ScanResult, cb: Optional[Callable]) -> None:
|
|
579
|
+
with lock:
|
|
580
|
+
stats.total_scanned += 1
|
|
581
|
+
stats.total_size_bytes += res.mail.size_bytes
|
|
582
|
+
if res.decision == "SIL":
|
|
583
|
+
stats.marked_for_deletion += 1
|
|
584
|
+
stats.marked_size_bytes += res.mail.size_bytes
|
|
585
|
+
if cb:
|
|
586
|
+
cb(res)
|
|
587
|
+
|
|
588
|
+
# ------------------------------------------------------------------
|
|
589
|
+
# Deletion / trash
|
|
590
|
+
# ------------------------------------------------------------------
|
|
591
|
+
|
|
592
|
+
def delete_mails(
|
|
593
|
+
self,
|
|
594
|
+
uids: list[str],
|
|
595
|
+
progress_cb: Optional[Callable[[str], None]] = None,
|
|
596
|
+
) -> list[str]:
|
|
597
|
+
assert self._conn, "Not connected"
|
|
598
|
+
rl = self._rl
|
|
599
|
+
deleted: list[str] = []
|
|
600
|
+
chunks = list(chunk_list(uids, rl.delete_chunk_size))
|
|
601
|
+
total_chunks = len(chunks)
|
|
602
|
+
log.info(f"Deleting {len(uids)} messages in {total_chunks} chunks")
|
|
603
|
+
|
|
604
|
+
for chunk_idx, chunk in enumerate(chunks, start=1):
|
|
605
|
+
uid_str = ",".join(chunk)
|
|
606
|
+
|
|
607
|
+
def _store(u=uid_str):
|
|
608
|
+
status, _ = self._conn.uid("store", u, "+FLAGS", r"(\Deleted)")
|
|
609
|
+
if status != "OK":
|
|
610
|
+
raise RuntimeError(f"STORE returned {status}")
|
|
611
|
+
|
|
612
|
+
try:
|
|
613
|
+
_with_retry(
|
|
614
|
+
_store,
|
|
615
|
+
rl,
|
|
616
|
+
label=f"delete chunk {chunk_idx}/{total_chunks}",
|
|
617
|
+
on_error=lambda exc, attempt, lbl=(
|
|
618
|
+
f"delete chunk {chunk_idx}/{total_chunks}"
|
|
619
|
+
): self._recover_connection(exc, lbl, attempt),
|
|
620
|
+
)
|
|
621
|
+
deleted.extend(chunk)
|
|
622
|
+
if progress_cb:
|
|
623
|
+
for uid in chunk:
|
|
624
|
+
progress_cb(uid)
|
|
625
|
+
except Exception as e:
|
|
626
|
+
log.error(f"Delete chunk {chunk_idx}/{total_chunks} failed: {e}")
|
|
627
|
+
|
|
628
|
+
if chunk_idx < total_chunks and rl.chunk_delay > 0:
|
|
629
|
+
time.sleep(rl.chunk_delay)
|
|
630
|
+
|
|
631
|
+
log.info("Expunging deleted messages")
|
|
632
|
+
try:
|
|
633
|
+
_with_retry(
|
|
634
|
+
lambda: self._conn.expunge(),
|
|
635
|
+
rl,
|
|
636
|
+
label="delete expunge",
|
|
637
|
+
on_error=lambda exc, attempt: self._recover_connection(
|
|
638
|
+
exc, "delete expunge", attempt
|
|
639
|
+
),
|
|
640
|
+
)
|
|
641
|
+
except Exception as exc:
|
|
642
|
+
log.error(f"Delete expunge failed after retries: {exc}")
|
|
643
|
+
return []
|
|
644
|
+
|
|
645
|
+
log.info(f"Deleted {len(deleted)} messages successfully")
|
|
646
|
+
return deleted
|
|
647
|
+
|
|
648
|
+
def _resolve_trash_folders(self, hint: str) -> list[str]:
|
|
649
|
+
assert self._conn, "Not connected"
|
|
650
|
+
candidates: list[str] = []
|
|
651
|
+
if hint:
|
|
652
|
+
candidates.append(hint.strip('"'))
|
|
653
|
+
|
|
654
|
+
lines: list | None = None
|
|
655
|
+
for attempt in range(1, self._rl.max_retries + 1):
|
|
656
|
+
try:
|
|
657
|
+
status, listed_lines = self._conn.list('""', "*")
|
|
658
|
+
if status == "OK":
|
|
659
|
+
lines = listed_lines
|
|
660
|
+
else:
|
|
661
|
+
log.warning(f"Trash folder LIST returned status={status}")
|
|
662
|
+
break
|
|
663
|
+
except Exception as e:
|
|
664
|
+
if self._is_connection_error(e) and attempt < self._rl.max_retries:
|
|
665
|
+
self._force_reconnect("trash folder discovery", attempt)
|
|
666
|
+
continue
|
|
667
|
+
log.warning(f"Trash folder discovery failed: {e}")
|
|
668
|
+
break
|
|
669
|
+
|
|
670
|
+
if lines:
|
|
671
|
+
for line in lines:
|
|
672
|
+
if not line:
|
|
673
|
+
continue
|
|
674
|
+
text = (
|
|
675
|
+
line.decode(errors="replace") if isinstance(line, bytes) else str(line)
|
|
676
|
+
)
|
|
677
|
+
m = re.search(r'"([^"]+)"\s*$|(\S+)\s*$', text)
|
|
678
|
+
if not m:
|
|
679
|
+
continue
|
|
680
|
+
name = (m.group(1) or m.group(2)).strip('"')
|
|
681
|
+
lower_text = text.lower()
|
|
682
|
+
has_trash_flag = r"\trash" in lower_text
|
|
683
|
+
keywords = (
|
|
684
|
+
"trash",
|
|
685
|
+
"bin",
|
|
686
|
+
"deleted",
|
|
687
|
+
"çöp",
|
|
688
|
+
"silinmiş",
|
|
689
|
+
"papelera",
|
|
690
|
+
"corbeille",
|
|
691
|
+
"papierkorb",
|
|
692
|
+
)
|
|
693
|
+
lower_name = name.lower()
|
|
694
|
+
has_keyword = any(k in lower_name for k in keywords)
|
|
695
|
+
if has_trash_flag or has_keyword:
|
|
696
|
+
candidates.append(name)
|
|
697
|
+
|
|
698
|
+
if self.cfg.provider == Provider.GMAIL:
|
|
699
|
+
candidates.extend([
|
|
700
|
+
"[Gmail]/Trash", "[Google Mail]/Trash", "[Gmail]/Bin",
|
|
701
|
+
"[Google Mail]/Bin", "Trash",
|
|
702
|
+
])
|
|
703
|
+
|
|
704
|
+
deduped: list[str] = []
|
|
705
|
+
seen: set[str] = set()
|
|
706
|
+
for folder in candidates:
|
|
707
|
+
clean_folder = folder.strip().strip('"')
|
|
708
|
+
if not clean_folder:
|
|
709
|
+
continue
|
|
710
|
+
key = clean_folder.casefold()
|
|
711
|
+
if key in seen:
|
|
712
|
+
continue
|
|
713
|
+
seen.add(key)
|
|
714
|
+
deduped.append(clean_folder)
|
|
715
|
+
|
|
716
|
+
if not deduped:
|
|
717
|
+
deduped = ["Trash"]
|
|
718
|
+
|
|
719
|
+
if len(deduped) == 1 and deduped[0] == hint.strip('"'):
|
|
720
|
+
log.warning(f"No trash folder found via LIST; falling back to: {deduped[0]}")
|
|
721
|
+
else:
|
|
722
|
+
log.debug(f"Trash folder candidates: {deduped}")
|
|
723
|
+
|
|
724
|
+
return deduped
|
|
725
|
+
|
|
726
|
+
def move_to_trash(
|
|
727
|
+
self,
|
|
728
|
+
uids: list[str],
|
|
729
|
+
trash_folder: str = "Trash",
|
|
730
|
+
progress_cb: Optional[Callable[[str], None]] = None,
|
|
731
|
+
) -> list[str]:
|
|
732
|
+
assert self._conn, "Not connected"
|
|
733
|
+
rl = self._rl
|
|
734
|
+
folders = self._resolve_trash_folders(trash_folder)
|
|
735
|
+
moved: list[str] = []
|
|
736
|
+
chunks = list(chunk_list(uids, rl.delete_chunk_size))
|
|
737
|
+
total_chunks = len(chunks)
|
|
738
|
+
log.info(
|
|
739
|
+
f"Moving {len(uids)} messages to trash folders {folders} "
|
|
740
|
+
f"in {total_chunks} chunks"
|
|
741
|
+
)
|
|
742
|
+
|
|
743
|
+
for chunk_idx, chunk in enumerate(chunks, start=1):
|
|
744
|
+
uid_str = ",".join(chunk)
|
|
745
|
+
chunk_folders = list(folders)
|
|
746
|
+
|
|
747
|
+
# --- HATA DÜZELTİLDİ: Copy ve Store işlemleri ayrıldı ---
|
|
748
|
+
def _copy(u=uid_str):
|
|
749
|
+
last_status = "NO"
|
|
750
|
+
for folder in chunk_folders:
|
|
751
|
+
status, _ = self._conn.uid("copy", u, f'"{folder}"')
|
|
752
|
+
if status == "OK":
|
|
753
|
+
return # Başarılı kopyalama, Store işi diğer adıma bırakıldı.
|
|
754
|
+
last_status = status
|
|
755
|
+
log.debug(f"COPY returned {status} for trash folder '{folder}', trying next candidate")
|
|
756
|
+
raise RuntimeError(f"COPY returned {last_status}")
|
|
757
|
+
|
|
758
|
+
def _on_copy_error(exc: Exception, attempt: int, lbl: str) -> None:
|
|
759
|
+
nonlocal folders, chunk_folders
|
|
760
|
+
self._recover_connection(exc, lbl, attempt)
|
|
761
|
+
refreshed_folders = self._resolve_trash_folders(trash_folder)
|
|
762
|
+
if refreshed_folders != folders:
|
|
763
|
+
log.info(f"Refreshed trash folder candidates: {refreshed_folders}")
|
|
764
|
+
folders = refreshed_folders
|
|
765
|
+
chunk_folders = list(refreshed_folders)
|
|
766
|
+
|
|
767
|
+
def _store_deleted(u=uid_str):
|
|
768
|
+
store_status, _ = self._conn.uid("store", u, "+FLAGS", r"(\Deleted)")
|
|
769
|
+
if store_status != "OK":
|
|
770
|
+
raise RuntimeError(f"STORE returned {store_status}")
|
|
771
|
+
|
|
772
|
+
try:
|
|
773
|
+
# 1. Aşama: Güvenli Kopyalama
|
|
774
|
+
_with_retry(
|
|
775
|
+
_copy, rl, label=f"trash chunk copy {chunk_idx}/{total_chunks}",
|
|
776
|
+
on_error=lambda exc, attempt, lbl=(
|
|
777
|
+
f"trash chunk copy {chunk_idx}/{total_chunks}"
|
|
778
|
+
): _on_copy_error(exc, attempt, lbl),
|
|
779
|
+
)
|
|
780
|
+
|
|
781
|
+
# 2. Aşama: Etiketleme (Eğer koparsa kopyalama baştan yapılmaz, sadece bu denenir)
|
|
782
|
+
_with_retry(
|
|
783
|
+
_store_deleted, rl, label=f"trash chunk store {chunk_idx}/{total_chunks}",
|
|
784
|
+
on_error=lambda exc, attempt, lbl=(
|
|
785
|
+
f"trash chunk store {chunk_idx}/{total_chunks}"
|
|
786
|
+
): self._recover_connection(exc, lbl, attempt),
|
|
787
|
+
)
|
|
788
|
+
|
|
789
|
+
moved.extend(chunk)
|
|
790
|
+
if progress_cb:
|
|
791
|
+
for uid in chunk:
|
|
792
|
+
progress_cb(uid)
|
|
793
|
+
except Exception as e:
|
|
794
|
+
log.error(f"Trash chunk {chunk_idx}/{total_chunks} failed: {e}")
|
|
795
|
+
|
|
796
|
+
if chunk_idx < total_chunks and rl.chunk_delay > 0:
|
|
797
|
+
time.sleep(rl.chunk_delay)
|
|
798
|
+
|
|
799
|
+
try:
|
|
800
|
+
_with_retry(
|
|
801
|
+
lambda: self._conn.expunge(), rl, label="trash expunge",
|
|
802
|
+
on_error=lambda exc, attempt: self._recover_connection(exc, "trash expunge", attempt),
|
|
803
|
+
)
|
|
804
|
+
except Exception as exc:
|
|
805
|
+
log.error(f"Trash expunge failed after retries: {exc}")
|
|
806
|
+
return []
|
|
807
|
+
|
|
808
|
+
log.info(f"Moved {len(moved)} messages to trash successfully")
|
|
809
|
+
return moved
|
|
810
|
+
|
|
811
|
+
# ------------------------------------------------------------------
|
|
812
|
+
# High-level convenience entry-point
|
|
813
|
+
# ------------------------------------------------------------------
|
|
814
|
+
|
|
815
|
+
def run(
|
|
816
|
+
self,
|
|
817
|
+
fetch_progress_cb: Optional[Callable] = None,
|
|
818
|
+
analyze_progress_cb: Optional[Callable] = None,
|
|
819
|
+
delete_progress_cb: Optional[Callable] = None,
|
|
820
|
+
resume: bool = False,
|
|
821
|
+
) -> tuple[list[ScanResult], ScanStats]:
|
|
822
|
+
log.info("Starting Engine run")
|
|
823
|
+
try:
|
|
824
|
+
self.connect()
|
|
825
|
+
uids = self.list_uids()
|
|
826
|
+
if not uids:
|
|
827
|
+
log.info("No messages found.")
|
|
828
|
+
return [], ScanStats()
|
|
829
|
+
|
|
830
|
+
mails = self.fetch_headers_concurrent(
|
|
831
|
+
uids, progress_cb=fetch_progress_cb, resume=resume
|
|
832
|
+
)
|
|
833
|
+
results, stats = self.analyze(mails, progress_cb=analyze_progress_cb)
|
|
834
|
+
|
|
835
|
+
if not self.cfg.dry_run and (
|
|
836
|
+
delete_uids := [r.mail.uid for r in results if r.decision == "SIL"]
|
|
837
|
+
):
|
|
838
|
+
self.delete_mails(delete_uids, progress_cb=delete_progress_cb)
|
|
839
|
+
|
|
840
|
+
return results, stats
|
|
841
|
+
finally:
|
|
842
|
+
self.disconnect()
|