transmute-cli 0.2.3a0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- transmute/__init__.py +1 -0
- transmute/__main__.py +4 -0
- transmute/app.py +1481 -0
- transmute/commands.py +321 -0
- transmute/config.py +21 -0
- transmute/downloader.py +190 -0
- transmute/enrich.py +865 -0
- transmute/history.py +858 -0
- transmute/iphone.py +247 -0
- transmute/keys.py +152 -0
- transmute/layout.py +125 -0
- transmute/main.py +38 -0
- transmute/settings.py +114 -0
- transmute/style.py +32 -0
- transmute/sync.py +532 -0
- transmute/widgets.py +86 -0
- transmute_cli-0.2.3a0.dist-info/METADATA +335 -0
- transmute_cli-0.2.3a0.dist-info/RECORD +21 -0
- transmute_cli-0.2.3a0.dist-info/WHEEL +4 -0
- transmute_cli-0.2.3a0.dist-info/entry_points.txt +2 -0
- transmute_cli-0.2.3a0.dist-info/licenses/LICENSE +674 -0
transmute/history.py
ADDED
|
@@ -0,0 +1,858 @@
|
|
|
1
|
+
"""Durable, prompt-toolkit-independent activity history.
|
|
2
|
+
|
|
3
|
+
The store deliberately opens a fresh SQLite connection for every operation.
|
|
4
|
+
That keeps connections confined to the calling thread while SQLite's WAL mode
|
|
5
|
+
and busy timeout coordinate Transmute's worker threads and other processes.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import socket
|
|
13
|
+
import sqlite3
|
|
14
|
+
import threading
|
|
15
|
+
from collections.abc import Iterator
|
|
16
|
+
from contextlib import contextmanager
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from datetime import datetime, timezone
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from uuid import uuid4
|
|
21
|
+
|
|
22
|
+
from .config import STATE_DIR
|
|
23
|
+
from .downloader import Job
|
|
24
|
+
|
|
25
|
+
SCHEMA_VERSION = 1
|
|
26
|
+
BUSY_TIMEOUT_MS = 5_000
|
|
27
|
+
_ACTIVE_STATUSES = ("queued", "downloading", "converting")
|
|
28
|
+
_INTERRUPTED = "interrupted — retry when ready"
|
|
29
|
+
_INTERRUPTED_DETAIL = "Transmute stopped before this job completed."
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class HistoryStoreError(RuntimeError):
|
|
33
|
+
"""The activity database could not be safely opened or updated."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class StoredJob:
|
|
38
|
+
"""A persisted job and the UI state that accompanies it."""
|
|
39
|
+
|
|
40
|
+
job: Job
|
|
41
|
+
detail: str | None
|
|
42
|
+
needs_hint: bool
|
|
43
|
+
hint_attempts: int
|
|
44
|
+
hint_in_progress: bool
|
|
45
|
+
session_id: str
|
|
46
|
+
created_at: datetime
|
|
47
|
+
updated_at: datetime
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class ActivityStore:
|
|
51
|
+
"""A versioned SQLite ledger of sessions and download outcomes."""
|
|
52
|
+
|
|
53
|
+
def __init__(self, path: Path | str) -> None:
|
|
54
|
+
self.path = Path(path).expanduser()
|
|
55
|
+
self._job_claims: dict[str, tuple[str, str]] = {}
|
|
56
|
+
self._job_claims_lock = threading.Lock()
|
|
57
|
+
self._prepare_path()
|
|
58
|
+
self._initialize()
|
|
59
|
+
self._recover_interrupted()
|
|
60
|
+
|
|
61
|
+
def start_session(self) -> str:
|
|
62
|
+
"""Create and return a session owned by this process."""
|
|
63
|
+
session_id = uuid4().hex
|
|
64
|
+
now = _utc_now()
|
|
65
|
+
self._write(
|
|
66
|
+
"""
|
|
67
|
+
INSERT INTO sessions (
|
|
68
|
+
session_id, pid, hostname, started_at, finished_at
|
|
69
|
+
) VALUES (?, ?, ?, ?, NULL)
|
|
70
|
+
""",
|
|
71
|
+
(session_id, os.getpid(), socket.gethostname(), now),
|
|
72
|
+
"start a history session",
|
|
73
|
+
)
|
|
74
|
+
return session_id
|
|
75
|
+
|
|
76
|
+
def finish_session(self, session_id: str) -> None:
|
|
77
|
+
"""Finish a session and make any remaining work safely retryable."""
|
|
78
|
+
now = _utc_now()
|
|
79
|
+
with self._transaction("finish a history session") as connection:
|
|
80
|
+
connection.execute(
|
|
81
|
+
"""
|
|
82
|
+
UPDATE sessions
|
|
83
|
+
SET finished_at = COALESCE(finished_at, ?)
|
|
84
|
+
WHERE session_id = ?
|
|
85
|
+
""",
|
|
86
|
+
(now, session_id),
|
|
87
|
+
)
|
|
88
|
+
self._interrupt_jobs(connection, (session_id,), now)
|
|
89
|
+
self._release_hint_claims(connection, (session_id,), now)
|
|
90
|
+
with self._job_claims_lock:
|
|
91
|
+
self._job_claims = {
|
|
92
|
+
history_id: claim
|
|
93
|
+
for history_id, claim in self._job_claims.items()
|
|
94
|
+
if claim[0] != session_id
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
def queue_job(self, job: Job, session_id: str) -> bool:
|
|
98
|
+
"""Atomically claim one new/retry job for this session."""
|
|
99
|
+
return job.history_id in self.queue_jobs([job], session_id)
|
|
100
|
+
|
|
101
|
+
def queue_jobs(self, jobs: list[Job], session_id: str) -> set[str]:
|
|
102
|
+
"""Claim a batch in one transaction and return the claimed history IDs."""
|
|
103
|
+
if not jobs:
|
|
104
|
+
return set()
|
|
105
|
+
now = _utc_now()
|
|
106
|
+
claimed: dict[str, str] = {}
|
|
107
|
+
with self._transaction("queue history jobs") as connection:
|
|
108
|
+
self._assert_session(connection, session_id)
|
|
109
|
+
for job in jobs:
|
|
110
|
+
claim_token = uuid4().hex
|
|
111
|
+
values = self._job_values(
|
|
112
|
+
job,
|
|
113
|
+
session_id=session_id,
|
|
114
|
+
status="queued",
|
|
115
|
+
error=None,
|
|
116
|
+
error_detail=None,
|
|
117
|
+
retryable=True,
|
|
118
|
+
)
|
|
119
|
+
cursor = connection.execute(
|
|
120
|
+
"""
|
|
121
|
+
INSERT INTO jobs (
|
|
122
|
+
history_id, session_id, claim_token, url, status,
|
|
123
|
+
title, uploader, duration, description, tags_json, path,
|
|
124
|
+
error, error_detail, retryable, detail, needs_hint,
|
|
125
|
+
hint_attempts, hint_claim_token, hint_session_id,
|
|
126
|
+
created_at, updated_at
|
|
127
|
+
) VALUES (
|
|
128
|
+
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
|
129
|
+
NULL, 0, 0, NULL, NULL, ?, ?
|
|
130
|
+
)
|
|
131
|
+
ON CONFLICT(history_id) DO UPDATE SET
|
|
132
|
+
session_id = excluded.session_id,
|
|
133
|
+
claim_token = excluded.claim_token,
|
|
134
|
+
url = excluded.url,
|
|
135
|
+
status = excluded.status,
|
|
136
|
+
title = excluded.title,
|
|
137
|
+
uploader = excluded.uploader,
|
|
138
|
+
duration = excluded.duration,
|
|
139
|
+
description = excluded.description,
|
|
140
|
+
tags_json = excluded.tags_json,
|
|
141
|
+
path = excluded.path,
|
|
142
|
+
error = NULL,
|
|
143
|
+
error_detail = NULL,
|
|
144
|
+
retryable = 1,
|
|
145
|
+
detail = NULL,
|
|
146
|
+
needs_hint = 0,
|
|
147
|
+
hint_attempts = 0,
|
|
148
|
+
hint_claim_token = NULL,
|
|
149
|
+
hint_session_id = NULL,
|
|
150
|
+
updated_at = excluded.updated_at
|
|
151
|
+
WHERE jobs.status IN ('done', 'error')
|
|
152
|
+
""",
|
|
153
|
+
(
|
|
154
|
+
values[0],
|
|
155
|
+
values[1],
|
|
156
|
+
claim_token,
|
|
157
|
+
*values[2:],
|
|
158
|
+
now,
|
|
159
|
+
now,
|
|
160
|
+
),
|
|
161
|
+
)
|
|
162
|
+
if cursor.rowcount == 1:
|
|
163
|
+
claimed[job.history_id] = claim_token
|
|
164
|
+
|
|
165
|
+
with self._job_claims_lock:
|
|
166
|
+
self._job_claims.update(
|
|
167
|
+
{
|
|
168
|
+
history_id: (session_id, claim_token)
|
|
169
|
+
for history_id, claim_token in claimed.items()
|
|
170
|
+
}
|
|
171
|
+
)
|
|
172
|
+
return set(claimed)
|
|
173
|
+
|
|
174
|
+
def save_failure(self, job: Job) -> bool:
|
|
175
|
+
"""Save a failure only while this store still owns the job claim."""
|
|
176
|
+
claim = self._job_claim(job.history_id)
|
|
177
|
+
if claim is None:
|
|
178
|
+
return False
|
|
179
|
+
session_id, claim_token = claim
|
|
180
|
+
now = _utc_now()
|
|
181
|
+
with self._transaction("save a failed history job") as connection:
|
|
182
|
+
cursor = connection.execute(
|
|
183
|
+
"""
|
|
184
|
+
UPDATE jobs
|
|
185
|
+
SET url = ?,
|
|
186
|
+
status = 'error',
|
|
187
|
+
title = ?,
|
|
188
|
+
uploader = ?,
|
|
189
|
+
duration = ?,
|
|
190
|
+
description = ?,
|
|
191
|
+
tags_json = ?,
|
|
192
|
+
path = ?,
|
|
193
|
+
error = ?,
|
|
194
|
+
error_detail = ?,
|
|
195
|
+
retryable = ?,
|
|
196
|
+
detail = NULL,
|
|
197
|
+
needs_hint = 0,
|
|
198
|
+
hint_attempts = 0,
|
|
199
|
+
claim_token = NULL,
|
|
200
|
+
hint_claim_token = NULL,
|
|
201
|
+
hint_session_id = NULL,
|
|
202
|
+
updated_at = ?
|
|
203
|
+
WHERE history_id = ?
|
|
204
|
+
AND session_id = ?
|
|
205
|
+
AND claim_token = ?
|
|
206
|
+
AND status IN ('queued', 'downloading', 'converting')
|
|
207
|
+
""",
|
|
208
|
+
(
|
|
209
|
+
*self._job_metadata_values(job),
|
|
210
|
+
job.error,
|
|
211
|
+
job.error_detail,
|
|
212
|
+
int(job.retryable),
|
|
213
|
+
now,
|
|
214
|
+
job.history_id,
|
|
215
|
+
session_id,
|
|
216
|
+
claim_token,
|
|
217
|
+
),
|
|
218
|
+
)
|
|
219
|
+
self._release_job_claim(job.history_id, claim_token)
|
|
220
|
+
return cursor.rowcount == 1
|
|
221
|
+
|
|
222
|
+
def save_success(
|
|
223
|
+
self,
|
|
224
|
+
job: Job,
|
|
225
|
+
detail: str | None,
|
|
226
|
+
needs_hint: bool,
|
|
227
|
+
) -> bool:
|
|
228
|
+
"""Save success only while this store still owns the job claim."""
|
|
229
|
+
claim = self._job_claim(job.history_id)
|
|
230
|
+
if claim is None:
|
|
231
|
+
return False
|
|
232
|
+
session_id, claim_token = claim
|
|
233
|
+
now = _utc_now()
|
|
234
|
+
with self._transaction("save a successful history job") as connection:
|
|
235
|
+
cursor = connection.execute(
|
|
236
|
+
"""
|
|
237
|
+
UPDATE jobs
|
|
238
|
+
SET url = ?,
|
|
239
|
+
status = 'done',
|
|
240
|
+
title = ?,
|
|
241
|
+
uploader = ?,
|
|
242
|
+
duration = ?,
|
|
243
|
+
description = ?,
|
|
244
|
+
tags_json = ?,
|
|
245
|
+
path = ?,
|
|
246
|
+
error = NULL,
|
|
247
|
+
error_detail = NULL,
|
|
248
|
+
retryable = 1,
|
|
249
|
+
detail = ?,
|
|
250
|
+
needs_hint = ?,
|
|
251
|
+
hint_attempts = 0,
|
|
252
|
+
claim_token = NULL,
|
|
253
|
+
hint_claim_token = NULL,
|
|
254
|
+
hint_session_id = NULL,
|
|
255
|
+
updated_at = ?
|
|
256
|
+
WHERE history_id = ?
|
|
257
|
+
AND session_id = ?
|
|
258
|
+
AND claim_token = ?
|
|
259
|
+
AND status IN ('queued', 'downloading', 'converting')
|
|
260
|
+
""",
|
|
261
|
+
(
|
|
262
|
+
*self._job_metadata_values(job),
|
|
263
|
+
detail,
|
|
264
|
+
int(needs_hint),
|
|
265
|
+
now,
|
|
266
|
+
job.history_id,
|
|
267
|
+
session_id,
|
|
268
|
+
claim_token,
|
|
269
|
+
),
|
|
270
|
+
)
|
|
271
|
+
self._release_job_claim(job.history_id, claim_token)
|
|
272
|
+
return cursor.rowcount == 1
|
|
273
|
+
|
|
274
|
+
def claim_hint(self, job: Job, session_id: str) -> str | bool:
|
|
275
|
+
"""Claim one low-confidence re-hint without changing download status."""
|
|
276
|
+
now = _utc_now()
|
|
277
|
+
claim_token = uuid4().hex
|
|
278
|
+
with self._transaction("claim a history hint") as connection:
|
|
279
|
+
self._assert_session(connection, session_id)
|
|
280
|
+
cursor = connection.execute(
|
|
281
|
+
"""
|
|
282
|
+
UPDATE jobs
|
|
283
|
+
SET hint_claim_token = ?,
|
|
284
|
+
hint_session_id = ?,
|
|
285
|
+
updated_at = ?
|
|
286
|
+
WHERE history_id = ?
|
|
287
|
+
AND status = 'done'
|
|
288
|
+
AND needs_hint = 1
|
|
289
|
+
AND hint_claim_token IS NULL
|
|
290
|
+
""",
|
|
291
|
+
(claim_token, session_id, now, job.history_id),
|
|
292
|
+
)
|
|
293
|
+
return claim_token if cursor.rowcount == 1 else False
|
|
294
|
+
|
|
295
|
+
def save_hint_success(
|
|
296
|
+
self,
|
|
297
|
+
job: Job,
|
|
298
|
+
detail: str | None,
|
|
299
|
+
needs_hint: bool,
|
|
300
|
+
claim_token: str,
|
|
301
|
+
) -> bool:
|
|
302
|
+
"""Apply a re-hint only if its unique cross-process claim is current."""
|
|
303
|
+
now = _utc_now()
|
|
304
|
+
with self._transaction("save a successful history hint") as connection:
|
|
305
|
+
cursor = connection.execute(
|
|
306
|
+
"""
|
|
307
|
+
UPDATE jobs
|
|
308
|
+
SET url = ?,
|
|
309
|
+
title = ?,
|
|
310
|
+
uploader = ?,
|
|
311
|
+
duration = ?,
|
|
312
|
+
description = ?,
|
|
313
|
+
tags_json = ?,
|
|
314
|
+
path = ?,
|
|
315
|
+
detail = ?,
|
|
316
|
+
needs_hint = ?,
|
|
317
|
+
hint_attempts = hint_attempts + 1,
|
|
318
|
+
hint_claim_token = NULL,
|
|
319
|
+
hint_session_id = NULL,
|
|
320
|
+
updated_at = ?
|
|
321
|
+
WHERE history_id = ?
|
|
322
|
+
AND status = 'done'
|
|
323
|
+
AND hint_claim_token = ?
|
|
324
|
+
""",
|
|
325
|
+
(
|
|
326
|
+
*self._job_metadata_values(job),
|
|
327
|
+
detail,
|
|
328
|
+
int(needs_hint),
|
|
329
|
+
now,
|
|
330
|
+
job.history_id,
|
|
331
|
+
claim_token,
|
|
332
|
+
),
|
|
333
|
+
)
|
|
334
|
+
return cursor.rowcount == 1
|
|
335
|
+
|
|
336
|
+
def release_hint(self, history_id: str, claim_token: str) -> bool:
|
|
337
|
+
"""Release a failed or canceled re-hint if its claim is still current."""
|
|
338
|
+
now = _utc_now()
|
|
339
|
+
with self._transaction("release a history hint") as connection:
|
|
340
|
+
cursor = connection.execute(
|
|
341
|
+
"""
|
|
342
|
+
UPDATE jobs
|
|
343
|
+
SET hint_claim_token = NULL,
|
|
344
|
+
hint_session_id = NULL,
|
|
345
|
+
updated_at = ?
|
|
346
|
+
WHERE history_id = ?
|
|
347
|
+
AND hint_claim_token = ?
|
|
348
|
+
""",
|
|
349
|
+
(now, history_id, claim_token),
|
|
350
|
+
)
|
|
351
|
+
return cursor.rowcount == 1
|
|
352
|
+
|
|
353
|
+
def load_jobs(self, limit: int = 500) -> list[StoredJob]:
|
|
354
|
+
"""Load the most recently updated jobs, ordered oldest-to-newest."""
|
|
355
|
+
if not isinstance(limit, int) or isinstance(limit, bool) or limit < 0:
|
|
356
|
+
raise ValueError("limit must be a non-negative integer")
|
|
357
|
+
if limit == 0:
|
|
358
|
+
return []
|
|
359
|
+
|
|
360
|
+
try:
|
|
361
|
+
with self._connection() as connection:
|
|
362
|
+
rows = connection.execute(
|
|
363
|
+
"""
|
|
364
|
+
SELECT *
|
|
365
|
+
FROM (
|
|
366
|
+
SELECT *
|
|
367
|
+
FROM jobs
|
|
368
|
+
ORDER BY updated_at DESC, row_id DESC
|
|
369
|
+
LIMIT ?
|
|
370
|
+
)
|
|
371
|
+
ORDER BY updated_at ASC, row_id ASC
|
|
372
|
+
""",
|
|
373
|
+
(limit,),
|
|
374
|
+
).fetchall()
|
|
375
|
+
return [self._stored_job(row) for row in rows]
|
|
376
|
+
except HistoryStoreError:
|
|
377
|
+
raise
|
|
378
|
+
except (json.JSONDecodeError, TypeError, ValueError, sqlite3.DatabaseError) as exc:
|
|
379
|
+
raise HistoryStoreError(
|
|
380
|
+
f"Activity history at {self.path} contains malformed job data: {exc}"
|
|
381
|
+
) from exc
|
|
382
|
+
|
|
383
|
+
def clear(self) -> None:
|
|
384
|
+
"""Delete terminal history while retaining work that is still in flight."""
|
|
385
|
+
placeholders = ", ".join("?" for _ in _ACTIVE_STATUSES)
|
|
386
|
+
self._write(
|
|
387
|
+
f"DELETE FROM jobs WHERE status NOT IN ({placeholders})",
|
|
388
|
+
_ACTIVE_STATUSES,
|
|
389
|
+
"clear activity history",
|
|
390
|
+
)
|
|
391
|
+
|
|
392
|
+
def _prepare_path(self) -> None:
|
|
393
|
+
try:
|
|
394
|
+
parent_was_created = not self.path.parent.exists()
|
|
395
|
+
self.path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
396
|
+
if parent_was_created or (
|
|
397
|
+
self.path.parent == STATE_DIR and not self.path.parent.is_symlink()
|
|
398
|
+
):
|
|
399
|
+
self.path.parent.chmod(0o700)
|
|
400
|
+
try:
|
|
401
|
+
descriptor = os.open(
|
|
402
|
+
self.path,
|
|
403
|
+
os.O_CREAT | os.O_EXCL | os.O_WRONLY,
|
|
404
|
+
0o600,
|
|
405
|
+
)
|
|
406
|
+
except FileExistsError:
|
|
407
|
+
pass
|
|
408
|
+
else:
|
|
409
|
+
os.close(descriptor)
|
|
410
|
+
if not self.path.is_file():
|
|
411
|
+
raise HistoryStoreError(
|
|
412
|
+
f"Activity history path is not a file: {self.path}"
|
|
413
|
+
)
|
|
414
|
+
self.path.chmod(0o600)
|
|
415
|
+
except HistoryStoreError:
|
|
416
|
+
raise
|
|
417
|
+
except OSError as exc:
|
|
418
|
+
raise HistoryStoreError(
|
|
419
|
+
f"Could not secure activity history at {self.path}: {exc}"
|
|
420
|
+
) from exc
|
|
421
|
+
|
|
422
|
+
def _initialize(self) -> None:
|
|
423
|
+
try:
|
|
424
|
+
with self._connection() as connection:
|
|
425
|
+
check = connection.execute("PRAGMA quick_check").fetchone()
|
|
426
|
+
if check is None or check[0] != "ok":
|
|
427
|
+
problem = check[0] if check else "no integrity result"
|
|
428
|
+
raise HistoryStoreError(
|
|
429
|
+
f"Activity history at {self.path} is corrupt: {problem}"
|
|
430
|
+
)
|
|
431
|
+
|
|
432
|
+
version = connection.execute("PRAGMA user_version").fetchone()[0]
|
|
433
|
+
tables = {
|
|
434
|
+
row[0]
|
|
435
|
+
for row in connection.execute(
|
|
436
|
+
"""
|
|
437
|
+
SELECT name FROM sqlite_master
|
|
438
|
+
WHERE type = 'table' AND name NOT LIKE 'sqlite_%'
|
|
439
|
+
"""
|
|
440
|
+
)
|
|
441
|
+
}
|
|
442
|
+
if version == 0 and not tables:
|
|
443
|
+
# Recheck after taking the write lock: another process may
|
|
444
|
+
# have initialized the empty file after our first read.
|
|
445
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
446
|
+
locked_version = connection.execute(
|
|
447
|
+
"PRAGMA user_version"
|
|
448
|
+
).fetchone()[0]
|
|
449
|
+
locked_tables = {
|
|
450
|
+
row[0]
|
|
451
|
+
for row in connection.execute(
|
|
452
|
+
"""
|
|
453
|
+
SELECT name FROM sqlite_master
|
|
454
|
+
WHERE type = 'table' AND name NOT LIKE 'sqlite_%'
|
|
455
|
+
"""
|
|
456
|
+
)
|
|
457
|
+
}
|
|
458
|
+
if locked_version == 0 and not locked_tables:
|
|
459
|
+
self._create_schema(connection)
|
|
460
|
+
connection.commit()
|
|
461
|
+
version = connection.execute(
|
|
462
|
+
"PRAGMA user_version"
|
|
463
|
+
).fetchone()[0]
|
|
464
|
+
tables = {
|
|
465
|
+
row[0]
|
|
466
|
+
for row in connection.execute(
|
|
467
|
+
"""
|
|
468
|
+
SELECT name FROM sqlite_master
|
|
469
|
+
WHERE type = 'table' AND name NOT LIKE 'sqlite_%'
|
|
470
|
+
"""
|
|
471
|
+
)
|
|
472
|
+
}
|
|
473
|
+
if version == 0:
|
|
474
|
+
raise HistoryStoreError(
|
|
475
|
+
f"Activity history at {self.path} has an unversioned schema; "
|
|
476
|
+
"the existing data was left untouched"
|
|
477
|
+
)
|
|
478
|
+
elif version > SCHEMA_VERSION:
|
|
479
|
+
raise HistoryStoreError(
|
|
480
|
+
f"Activity history schema version {version} is newer than "
|
|
481
|
+
f"the supported version {SCHEMA_VERSION}; upgrade Transmute"
|
|
482
|
+
)
|
|
483
|
+
elif version < SCHEMA_VERSION:
|
|
484
|
+
raise HistoryStoreError(
|
|
485
|
+
f"Activity history schema version {version} is unsupported"
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
self._validate_schema(connection)
|
|
489
|
+
mode = connection.execute("PRAGMA journal_mode = WAL").fetchone()
|
|
490
|
+
if mode is None or str(mode[0]).lower() != "wal":
|
|
491
|
+
raise HistoryStoreError(
|
|
492
|
+
f"Could not enable WAL mode for activity history at {self.path}"
|
|
493
|
+
)
|
|
494
|
+
except HistoryStoreError:
|
|
495
|
+
raise
|
|
496
|
+
except sqlite3.DatabaseError as exc:
|
|
497
|
+
raise HistoryStoreError(
|
|
498
|
+
f"Could not open activity history at {self.path}: {exc}"
|
|
499
|
+
) from exc
|
|
500
|
+
finally:
|
|
501
|
+
try:
|
|
502
|
+
self.path.chmod(0o600)
|
|
503
|
+
except OSError:
|
|
504
|
+
pass
|
|
505
|
+
|
|
506
|
+
@staticmethod
|
|
507
|
+
def _create_schema(connection: sqlite3.Connection) -> None:
|
|
508
|
+
connection.execute(
|
|
509
|
+
"""
|
|
510
|
+
CREATE TABLE sessions (
|
|
511
|
+
session_id TEXT PRIMARY KEY,
|
|
512
|
+
pid INTEGER NOT NULL,
|
|
513
|
+
hostname TEXT NOT NULL,
|
|
514
|
+
started_at TEXT NOT NULL,
|
|
515
|
+
finished_at TEXT
|
|
516
|
+
)
|
|
517
|
+
"""
|
|
518
|
+
)
|
|
519
|
+
connection.execute(
|
|
520
|
+
"""
|
|
521
|
+
CREATE TABLE jobs (
|
|
522
|
+
row_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
523
|
+
history_id TEXT NOT NULL UNIQUE,
|
|
524
|
+
session_id TEXT NOT NULL REFERENCES sessions(session_id),
|
|
525
|
+
claim_token TEXT,
|
|
526
|
+
url TEXT NOT NULL,
|
|
527
|
+
status TEXT NOT NULL,
|
|
528
|
+
title TEXT,
|
|
529
|
+
uploader TEXT,
|
|
530
|
+
duration INTEGER,
|
|
531
|
+
description TEXT,
|
|
532
|
+
tags_json TEXT,
|
|
533
|
+
path TEXT,
|
|
534
|
+
error TEXT,
|
|
535
|
+
error_detail TEXT,
|
|
536
|
+
retryable INTEGER NOT NULL,
|
|
537
|
+
detail TEXT,
|
|
538
|
+
needs_hint INTEGER NOT NULL DEFAULT 0,
|
|
539
|
+
hint_attempts INTEGER NOT NULL DEFAULT 0,
|
|
540
|
+
hint_claim_token TEXT,
|
|
541
|
+
hint_session_id TEXT,
|
|
542
|
+
created_at TEXT NOT NULL,
|
|
543
|
+
updated_at TEXT NOT NULL
|
|
544
|
+
)
|
|
545
|
+
"""
|
|
546
|
+
)
|
|
547
|
+
connection.execute(
|
|
548
|
+
"CREATE INDEX jobs_updated_idx ON jobs(updated_at, row_id)"
|
|
549
|
+
)
|
|
550
|
+
connection.execute("CREATE INDEX jobs_session_idx ON jobs(session_id)")
|
|
551
|
+
connection.execute(
|
|
552
|
+
"CREATE INDEX jobs_hint_session_idx ON jobs(hint_session_id)"
|
|
553
|
+
)
|
|
554
|
+
connection.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
|
|
555
|
+
|
|
556
|
+
@staticmethod
|
|
557
|
+
def _validate_schema(connection: sqlite3.Connection) -> None:
|
|
558
|
+
required = {
|
|
559
|
+
"sessions": {
|
|
560
|
+
"session_id",
|
|
561
|
+
"pid",
|
|
562
|
+
"hostname",
|
|
563
|
+
"started_at",
|
|
564
|
+
"finished_at",
|
|
565
|
+
},
|
|
566
|
+
"jobs": {
|
|
567
|
+
"row_id",
|
|
568
|
+
"history_id",
|
|
569
|
+
"session_id",
|
|
570
|
+
"claim_token",
|
|
571
|
+
"url",
|
|
572
|
+
"status",
|
|
573
|
+
"title",
|
|
574
|
+
"uploader",
|
|
575
|
+
"duration",
|
|
576
|
+
"description",
|
|
577
|
+
"tags_json",
|
|
578
|
+
"path",
|
|
579
|
+
"error",
|
|
580
|
+
"error_detail",
|
|
581
|
+
"retryable",
|
|
582
|
+
"detail",
|
|
583
|
+
"needs_hint",
|
|
584
|
+
"hint_attempts",
|
|
585
|
+
"hint_claim_token",
|
|
586
|
+
"hint_session_id",
|
|
587
|
+
"created_at",
|
|
588
|
+
"updated_at",
|
|
589
|
+
},
|
|
590
|
+
}
|
|
591
|
+
for table, expected_columns in required.items():
|
|
592
|
+
rows = connection.execute(f"PRAGMA table_info({table})").fetchall()
|
|
593
|
+
columns = {row[1] for row in rows}
|
|
594
|
+
missing = expected_columns - columns
|
|
595
|
+
if missing:
|
|
596
|
+
names = ", ".join(sorted(missing))
|
|
597
|
+
raise HistoryStoreError(
|
|
598
|
+
f"Activity history schema is malformed: "
|
|
599
|
+
f"{table} is missing {names}"
|
|
600
|
+
)
|
|
601
|
+
|
|
602
|
+
def _recover_interrupted(self) -> None:
|
|
603
|
+
"""Close dead sessions and turn their unfinished rows into failures."""
|
|
604
|
+
try:
|
|
605
|
+
with self._connection() as connection:
|
|
606
|
+
sessions = connection.execute(
|
|
607
|
+
"""
|
|
608
|
+
SELECT session_id, pid, hostname
|
|
609
|
+
FROM sessions
|
|
610
|
+
WHERE finished_at IS NULL
|
|
611
|
+
"""
|
|
612
|
+
).fetchall()
|
|
613
|
+
except sqlite3.DatabaseError as exc:
|
|
614
|
+
raise HistoryStoreError(
|
|
615
|
+
f"Could not inspect activity history at {self.path}: {exc}"
|
|
616
|
+
) from exc
|
|
617
|
+
|
|
618
|
+
hostname = socket.gethostname()
|
|
619
|
+
stale = []
|
|
620
|
+
for row in sessions:
|
|
621
|
+
inactive_session = (
|
|
622
|
+
row["hostname"] != hostname or not _process_is_alive(row["pid"])
|
|
623
|
+
)
|
|
624
|
+
if inactive_session:
|
|
625
|
+
stale.append(row["session_id"])
|
|
626
|
+
|
|
627
|
+
if not stale:
|
|
628
|
+
return
|
|
629
|
+
now = _utc_now()
|
|
630
|
+
placeholders = ", ".join("?" for _ in stale)
|
|
631
|
+
with self._transaction("recover interrupted history jobs") as connection:
|
|
632
|
+
connection.execute(
|
|
633
|
+
f"""
|
|
634
|
+
UPDATE sessions
|
|
635
|
+
SET finished_at = COALESCE(finished_at, ?)
|
|
636
|
+
WHERE session_id IN ({placeholders})
|
|
637
|
+
""",
|
|
638
|
+
(now, *stale),
|
|
639
|
+
)
|
|
640
|
+
self._interrupt_jobs(connection, tuple(stale), now)
|
|
641
|
+
self._release_hint_claims(connection, tuple(stale), now)
|
|
642
|
+
|
|
643
|
+
@staticmethod
|
|
644
|
+
def _interrupt_jobs(
|
|
645
|
+
connection: sqlite3.Connection,
|
|
646
|
+
session_ids: tuple[str, ...],
|
|
647
|
+
now: str,
|
|
648
|
+
) -> None:
|
|
649
|
+
if not session_ids:
|
|
650
|
+
return
|
|
651
|
+
session_placeholders = ", ".join("?" for _ in session_ids)
|
|
652
|
+
status_placeholders = ", ".join("?" for _ in _ACTIVE_STATUSES)
|
|
653
|
+
connection.execute(
|
|
654
|
+
f"""
|
|
655
|
+
UPDATE jobs
|
|
656
|
+
SET status = 'error',
|
|
657
|
+
error = ?,
|
|
658
|
+
error_detail = ?,
|
|
659
|
+
retryable = 1,
|
|
660
|
+
detail = NULL,
|
|
661
|
+
needs_hint = 0,
|
|
662
|
+
claim_token = NULL,
|
|
663
|
+
updated_at = ?
|
|
664
|
+
WHERE session_id IN ({session_placeholders})
|
|
665
|
+
AND status IN ({status_placeholders})
|
|
666
|
+
""",
|
|
667
|
+
(
|
|
668
|
+
_INTERRUPTED,
|
|
669
|
+
_INTERRUPTED_DETAIL,
|
|
670
|
+
now,
|
|
671
|
+
*session_ids,
|
|
672
|
+
*_ACTIVE_STATUSES,
|
|
673
|
+
),
|
|
674
|
+
)
|
|
675
|
+
|
|
676
|
+
@staticmethod
|
|
677
|
+
def _release_hint_claims(
|
|
678
|
+
connection: sqlite3.Connection,
|
|
679
|
+
session_ids: tuple[str, ...],
|
|
680
|
+
now: str,
|
|
681
|
+
) -> None:
|
|
682
|
+
if not session_ids:
|
|
683
|
+
return
|
|
684
|
+
placeholders = ", ".join("?" for _ in session_ids)
|
|
685
|
+
connection.execute(
|
|
686
|
+
f"""
|
|
687
|
+
UPDATE jobs
|
|
688
|
+
SET hint_claim_token = NULL,
|
|
689
|
+
hint_session_id = NULL,
|
|
690
|
+
updated_at = ?
|
|
691
|
+
WHERE hint_session_id IN ({placeholders})
|
|
692
|
+
""",
|
|
693
|
+
(now, *session_ids),
|
|
694
|
+
)
|
|
695
|
+
|
|
696
|
+
@staticmethod
|
|
697
|
+
def _assert_session(connection: sqlite3.Connection, session_id: str) -> None:
|
|
698
|
+
row = connection.execute(
|
|
699
|
+
"SELECT finished_at FROM sessions WHERE session_id = ?",
|
|
700
|
+
(session_id,),
|
|
701
|
+
).fetchone()
|
|
702
|
+
if row is None:
|
|
703
|
+
raise HistoryStoreError(f"Unknown activity session: {session_id}")
|
|
704
|
+
if row["finished_at"] is not None:
|
|
705
|
+
raise HistoryStoreError(f"Activity session is already finished: {session_id}")
|
|
706
|
+
|
|
707
|
+
def _job_claim(self, history_id: str) -> tuple[str, str] | None:
|
|
708
|
+
with self._job_claims_lock:
|
|
709
|
+
return self._job_claims.get(history_id)
|
|
710
|
+
|
|
711
|
+
def _release_job_claim(self, history_id: str, claim_token: str) -> None:
|
|
712
|
+
with self._job_claims_lock:
|
|
713
|
+
claim = self._job_claims.get(history_id)
|
|
714
|
+
if claim is not None and claim[1] == claim_token:
|
|
715
|
+
del self._job_claims[history_id]
|
|
716
|
+
|
|
717
|
+
@staticmethod
|
|
718
|
+
def _job_metadata_values(job: Job) -> tuple[object, ...]:
|
|
719
|
+
tags_json = (
|
|
720
|
+
None
|
|
721
|
+
if job.tags is None
|
|
722
|
+
else json.dumps(job.tags, ensure_ascii=False, separators=(",", ":"))
|
|
723
|
+
)
|
|
724
|
+
return (
|
|
725
|
+
job.url,
|
|
726
|
+
job.title,
|
|
727
|
+
job.uploader,
|
|
728
|
+
job.duration,
|
|
729
|
+
job.description,
|
|
730
|
+
tags_json,
|
|
731
|
+
str(job.path) if job.path is not None else None,
|
|
732
|
+
)
|
|
733
|
+
|
|
734
|
+
@staticmethod
|
|
735
|
+
def _job_values(
|
|
736
|
+
job: Job,
|
|
737
|
+
*,
|
|
738
|
+
session_id: str,
|
|
739
|
+
status: str,
|
|
740
|
+
error: str | None,
|
|
741
|
+
error_detail: str | None,
|
|
742
|
+
retryable: bool,
|
|
743
|
+
) -> tuple[object, ...]:
|
|
744
|
+
metadata = ActivityStore._job_metadata_values(job)
|
|
745
|
+
return (
|
|
746
|
+
job.history_id,
|
|
747
|
+
session_id,
|
|
748
|
+
metadata[0],
|
|
749
|
+
status,
|
|
750
|
+
*metadata[1:],
|
|
751
|
+
error,
|
|
752
|
+
error_detail,
|
|
753
|
+
int(retryable),
|
|
754
|
+
)
|
|
755
|
+
|
|
756
|
+
@staticmethod
|
|
757
|
+
def _stored_job(row: sqlite3.Row) -> StoredJob:
|
|
758
|
+
raw_tags = row["tags_json"]
|
|
759
|
+
tags = None if raw_tags is None else json.loads(raw_tags)
|
|
760
|
+
if tags is not None and (
|
|
761
|
+
not isinstance(tags, list)
|
|
762
|
+
or any(not isinstance(tag, str) for tag in tags)
|
|
763
|
+
):
|
|
764
|
+
raise ValueError("tags_json is not a list of strings")
|
|
765
|
+
job = Job(
|
|
766
|
+
url=row["url"],
|
|
767
|
+
status=row["status"],
|
|
768
|
+
title=row["title"],
|
|
769
|
+
uploader=row["uploader"],
|
|
770
|
+
duration=row["duration"],
|
|
771
|
+
description=row["description"],
|
|
772
|
+
tags=tags,
|
|
773
|
+
path=Path(row["path"]) if row["path"] is not None else None,
|
|
774
|
+
error=row["error"],
|
|
775
|
+
error_detail=row["error_detail"],
|
|
776
|
+
retryable=bool(row["retryable"]),
|
|
777
|
+
history_id=row["history_id"],
|
|
778
|
+
)
|
|
779
|
+
return StoredJob(
|
|
780
|
+
job=job,
|
|
781
|
+
detail=row["detail"],
|
|
782
|
+
needs_hint=bool(row["needs_hint"]),
|
|
783
|
+
hint_attempts=row["hint_attempts"],
|
|
784
|
+
hint_in_progress=row["hint_claim_token"] is not None,
|
|
785
|
+
session_id=row["session_id"],
|
|
786
|
+
created_at=_parse_utc(row["created_at"]),
|
|
787
|
+
updated_at=_parse_utc(row["updated_at"]),
|
|
788
|
+
)
|
|
789
|
+
|
|
790
|
+
@contextmanager
|
|
791
|
+
def _connection(self) -> Iterator[sqlite3.Connection]:
|
|
792
|
+
connection: sqlite3.Connection | None = None
|
|
793
|
+
try:
|
|
794
|
+
connection = sqlite3.connect(
|
|
795
|
+
self.path,
|
|
796
|
+
timeout=BUSY_TIMEOUT_MS / 1_000,
|
|
797
|
+
)
|
|
798
|
+
connection.row_factory = sqlite3.Row
|
|
799
|
+
connection.execute(f"PRAGMA busy_timeout = {BUSY_TIMEOUT_MS}")
|
|
800
|
+
connection.execute("PRAGMA foreign_keys = ON")
|
|
801
|
+
connection.execute("PRAGMA synchronous = NORMAL")
|
|
802
|
+
yield connection
|
|
803
|
+
finally:
|
|
804
|
+
if connection is not None:
|
|
805
|
+
connection.close()
|
|
806
|
+
|
|
807
|
+
@contextmanager
|
|
808
|
+
def _transaction(self, action: str) -> Iterator[sqlite3.Connection]:
|
|
809
|
+
try:
|
|
810
|
+
with self._connection() as connection:
|
|
811
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
812
|
+
try:
|
|
813
|
+
yield connection
|
|
814
|
+
except Exception:
|
|
815
|
+
connection.rollback()
|
|
816
|
+
raise
|
|
817
|
+
else:
|
|
818
|
+
connection.commit()
|
|
819
|
+
except HistoryStoreError:
|
|
820
|
+
raise
|
|
821
|
+
except sqlite3.DatabaseError as exc:
|
|
822
|
+
raise HistoryStoreError(
|
|
823
|
+
f"Could not {action} in {self.path}: {exc}"
|
|
824
|
+
) from exc
|
|
825
|
+
|
|
826
|
+
def _write(
|
|
827
|
+
self,
|
|
828
|
+
sql: str,
|
|
829
|
+
parameters: tuple[object, ...],
|
|
830
|
+
action: str,
|
|
831
|
+
) -> None:
|
|
832
|
+
with self._transaction(action) as connection:
|
|
833
|
+
connection.execute(sql, parameters)
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
def _utc_now() -> str:
|
|
837
|
+
return datetime.now(timezone.utc).isoformat(timespec="microseconds")
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
def _parse_utc(value: str) -> datetime:
|
|
841
|
+
parsed = datetime.fromisoformat(value)
|
|
842
|
+
if parsed.tzinfo is None:
|
|
843
|
+
raise ValueError("timestamp is missing a UTC offset")
|
|
844
|
+
return parsed.astimezone(timezone.utc)
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
def _process_is_alive(pid: int) -> bool:
|
|
848
|
+
if pid <= 0:
|
|
849
|
+
return False
|
|
850
|
+
try:
|
|
851
|
+
os.kill(pid, 0)
|
|
852
|
+
except ProcessLookupError:
|
|
853
|
+
return False
|
|
854
|
+
except PermissionError:
|
|
855
|
+
return True
|
|
856
|
+
except OSError:
|
|
857
|
+
return False
|
|
858
|
+
return True
|