bunkr-api 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bunkr_api/__init__.py +5 -0
- bunkr_api/__main__.py +4 -0
- bunkr_api/api.py +206 -0
- bunkr_api/cli.py +541 -0
- bunkr_api/config.py +26 -0
- bunkr_api/core/__init__.py +0 -0
- bunkr_api/core/db.py +353 -0
- bunkr_api/core/scraper.py +310 -0
- bunkr_api/core/tokens.py +197 -0
- bunkr_api/inspector.py +534 -0
- bunkr_api/media/__init__.py +0 -0
- bunkr_api/media/downloader.py +437 -0
- bunkr_api/media/player.py +480 -0
- bunkr_api/utils/__init__.py +0 -0
- bunkr_api/utils/formatting.py +86 -0
- bunkr_api/utils/http.py +19 -0
- bunkr_api-0.1.0.dist-info/METADATA +697 -0
- bunkr_api-0.1.0.dist-info/RECORD +22 -0
- bunkr_api-0.1.0.dist-info/WHEEL +5 -0
- bunkr_api-0.1.0.dist-info/entry_points.txt +7 -0
- bunkr_api-0.1.0.dist-info/licenses/LICENSE +21 -0
- bunkr_api-0.1.0.dist-info/top_level.txt +1 -0
bunkr_api/core/db.py
ADDED
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
import sqlite3
|
|
2
|
+
import time
|
|
3
|
+
from contextlib import closing, contextmanager
|
|
4
|
+
|
|
5
|
+
# Internal package imports
|
|
6
|
+
from ..config import DB_PATH
|
|
7
|
+
from ..utils.formatting import extract_expiry_from_url
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class DatabaseManager:
|
|
11
|
+
def __init__(self, db_path: str = DB_PATH):
|
|
12
|
+
self.db_path = db_path
|
|
13
|
+
self._init_db()
|
|
14
|
+
|
|
15
|
+
def _get_connection(self) -> sqlite3.Connection:
|
|
16
|
+
conn = sqlite3.connect(self.db_path, timeout=5.0)
|
|
17
|
+
conn.row_factory = sqlite3.Row
|
|
18
|
+
conn.execute("PRAGMA foreign_keys = ON;")
|
|
19
|
+
return conn
|
|
20
|
+
|
|
21
|
+
@contextmanager
|
|
22
|
+
def connection(self):
|
|
23
|
+
conn = self._get_connection()
|
|
24
|
+
try:
|
|
25
|
+
with conn:
|
|
26
|
+
yield conn
|
|
27
|
+
finally:
|
|
28
|
+
conn.close()
|
|
29
|
+
|
|
30
|
+
def _init_db(self):
|
|
31
|
+
"""Initializes tables and handles schema migrations."""
|
|
32
|
+
with closing(self._get_connection()) as conn:
|
|
33
|
+
conn.execute("PRAGMA journal_mode = WAL;")
|
|
34
|
+
with conn:
|
|
35
|
+
conn.execute("""
|
|
36
|
+
CREATE TABLE IF NOT EXISTS albums (
|
|
37
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
38
|
+
album_id_str TEXT UNIQUE,
|
|
39
|
+
title TEXT NOT NULL,
|
|
40
|
+
search_term TEXT,
|
|
41
|
+
global_index INTEGER,
|
|
42
|
+
aggregate_size TEXT DEFAULT '0 MB',
|
|
43
|
+
file_count INTEGER DEFAULT 0,
|
|
44
|
+
is_staged INTEGER DEFAULT 0,
|
|
45
|
+
created_at INTEGER NOT NULL,
|
|
46
|
+
updated_at INTEGER NOT NULL
|
|
47
|
+
);
|
|
48
|
+
""")
|
|
49
|
+
conn.execute("""
|
|
50
|
+
CREATE TABLE IF NOT EXISTS assets (
|
|
51
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
52
|
+
album_id INTEGER NOT NULL,
|
|
53
|
+
track_number INTEGER,
|
|
54
|
+
true_file_id INTEGER,
|
|
55
|
+
title TEXT NOT NULL,
|
|
56
|
+
original_filename TEXT,
|
|
57
|
+
raw_size_bytes INTEGER DEFAULT 0,
|
|
58
|
+
source_url TEXT UNIQUE,
|
|
59
|
+
signed_cdn_url TEXT,
|
|
60
|
+
token_expiry_timestamp INTEGER,
|
|
61
|
+
is_staged INTEGER DEFAULT 0,
|
|
62
|
+
download_status TEXT CHECK(
|
|
63
|
+
download_status IN ('PENDING', 'DOWNLOADING', 'COMPLETED', 'FAILED')
|
|
64
|
+
) DEFAULT 'PENDING',
|
|
65
|
+
local_file_path TEXT,
|
|
66
|
+
error_message TEXT,
|
|
67
|
+
FOREIGN KEY (album_id) REFERENCES albums(id) ON DELETE CASCADE
|
|
68
|
+
);
|
|
69
|
+
""")
|
|
70
|
+
conn.execute(
|
|
71
|
+
"CREATE TABLE IF NOT EXISTS system_config (config_key TEXT PRIMARY KEY, config_value TEXT NOT NULL, description TEXT);"
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
# Performance Indexing
|
|
75
|
+
conn.execute(
|
|
76
|
+
"CREATE INDEX IF NOT EXISTS idx_assets_expiry ON assets(token_expiry_timestamp);"
|
|
77
|
+
)
|
|
78
|
+
conn.execute("CREATE INDEX IF NOT EXISTS idx_assets_album_id ON assets(album_id);")
|
|
79
|
+
|
|
80
|
+
# Seed Defaults
|
|
81
|
+
conn.execute("""
|
|
82
|
+
INSERT OR IGNORE INTO system_config (config_key, config_value, description)
|
|
83
|
+
VALUES
|
|
84
|
+
('max_workers', '4', 'Default concurrency worker thread limit'),
|
|
85
|
+
('token_buffer_seconds', '600', 'Force token refresh lookahead window'),
|
|
86
|
+
('minter_poll_interval_seconds', '60', 'Polling interval for the background minter')
|
|
87
|
+
""")
|
|
88
|
+
|
|
89
|
+
def get_config_val(self, key: str, default: str) -> str:
|
|
90
|
+
with closing(self._get_connection()) as conn:
|
|
91
|
+
row = conn.execute(
|
|
92
|
+
"SELECT config_value FROM system_config WHERE config_key = ?;", (key,)
|
|
93
|
+
).fetchone()
|
|
94
|
+
return row["config_value"] if row else default
|
|
95
|
+
|
|
96
|
+
def register_album_from_json(self, data: dict) -> tuple[int, int, int]:
|
|
97
|
+
"""
|
|
98
|
+
Registers/syncs an album and its files.
|
|
99
|
+
|
|
100
|
+
Identity is keyed on the album's stable Bunkr slug (album_slug), not on
|
|
101
|
+
title + search-result position — result position shifts between scrapes
|
|
102
|
+
(pagination/sort/new uploads), which previously caused re-scrapes of the
|
|
103
|
+
same album to register as a brand new album with zero attached files.
|
|
104
|
+
|
|
105
|
+
Legacy JSON imports that predate album_slug fall back to the old
|
|
106
|
+
title+index identity, since there's nothing more stable to key on.
|
|
107
|
+
|
|
108
|
+
Returns (album_id, new_file_count, updated_file_count) so callers can
|
|
109
|
+
give the user real feedback instead of a blind "success" message.
|
|
110
|
+
"""
|
|
111
|
+
album_meta = data.get("selected_album", {})
|
|
112
|
+
search_term = data.get("search_term", "N/A")
|
|
113
|
+
files_found = data.get("files_found", [])
|
|
114
|
+
|
|
115
|
+
album_title = album_meta.get("title", "Unknown Album")
|
|
116
|
+
global_idx = album_meta.get("album_index_number", 0)
|
|
117
|
+
album_slug = album_meta.get("album_slug")
|
|
118
|
+
|
|
119
|
+
if album_slug:
|
|
120
|
+
album_id_str = f"slug_{album_slug}".lower()
|
|
121
|
+
else:
|
|
122
|
+
# Legacy fallback (old JSON exports without a slug) — best effort only.
|
|
123
|
+
album_id_str = f"{album_title}_{global_idx}".lower().replace(" ", "_")
|
|
124
|
+
|
|
125
|
+
now = int(time.time())
|
|
126
|
+
|
|
127
|
+
with closing(self._get_connection()) as conn:
|
|
128
|
+
with conn:
|
|
129
|
+
cursor = conn.execute(
|
|
130
|
+
"""
|
|
131
|
+
INSERT INTO albums (album_id_str, title, search_term, global_index, aggregate_size, file_count, created_at, updated_at)
|
|
132
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
133
|
+
ON CONFLICT(album_id_str) DO UPDATE SET
|
|
134
|
+
updated_at = excluded.updated_at,
|
|
135
|
+
aggregate_size = excluded.aggregate_size,
|
|
136
|
+
file_count = excluded.file_count
|
|
137
|
+
RETURNING id;
|
|
138
|
+
""",
|
|
139
|
+
(
|
|
140
|
+
album_id_str,
|
|
141
|
+
album_title,
|
|
142
|
+
search_term,
|
|
143
|
+
global_idx,
|
|
144
|
+
album_meta.get("aggregate_size", 0),
|
|
145
|
+
len(files_found),
|
|
146
|
+
now,
|
|
147
|
+
now,
|
|
148
|
+
),
|
|
149
|
+
)
|
|
150
|
+
album_id = cursor.fetchone()["id"]
|
|
151
|
+
|
|
152
|
+
# Pre-compute rows and check which source_urls already exist so we
|
|
153
|
+
# can report real new-vs-updated counts (INSERT...ON CONFLICT alone
|
|
154
|
+
# can't tell you which branch it took).
|
|
155
|
+
prepared_rows = []
|
|
156
|
+
source_urls = []
|
|
157
|
+
for idx, file_rec in enumerate(files_found, start=1):
|
|
158
|
+
cdn_url = file_rec.get("signed_cdn_url")
|
|
159
|
+
expiry_ts = extract_expiry_from_url(cdn_url)
|
|
160
|
+
true_file_id = file_rec.get("true_file_id") or file_rec.get("slug_id")
|
|
161
|
+
source_url = file_rec.get("href", cdn_url)
|
|
162
|
+
prepared_rows.append(
|
|
163
|
+
(idx, file_rec, cdn_url, expiry_ts, true_file_id, source_url)
|
|
164
|
+
)
|
|
165
|
+
source_urls.append(source_url)
|
|
166
|
+
|
|
167
|
+
existing_urls = set()
|
|
168
|
+
if source_urls:
|
|
169
|
+
placeholders = ",".join("?" for _ in source_urls)
|
|
170
|
+
existing_urls = {
|
|
171
|
+
row["source_url"]
|
|
172
|
+
for row in conn.execute(
|
|
173
|
+
f"SELECT source_url FROM assets WHERE source_url IN ({placeholders});",
|
|
174
|
+
source_urls,
|
|
175
|
+
).fetchall()
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
new_count = 0
|
|
179
|
+
updated_count = 0
|
|
180
|
+
for idx, file_rec, cdn_url, expiry_ts, true_file_id, source_url in prepared_rows:
|
|
181
|
+
if source_url in existing_urls:
|
|
182
|
+
updated_count += 1
|
|
183
|
+
else:
|
|
184
|
+
new_count += 1
|
|
185
|
+
|
|
186
|
+
conn.execute(
|
|
187
|
+
"""
|
|
188
|
+
INSERT INTO assets (
|
|
189
|
+
album_id, track_number, true_file_id, title, original_filename,
|
|
190
|
+
raw_size_bytes, source_url, signed_cdn_url, token_expiry_timestamp
|
|
191
|
+
)
|
|
192
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
193
|
+
ON CONFLICT(source_url) DO UPDATE SET
|
|
194
|
+
album_id = excluded.album_id,
|
|
195
|
+
track_number = excluded.track_number,
|
|
196
|
+
true_file_id = excluded.true_file_id,
|
|
197
|
+
title = excluded.title,
|
|
198
|
+
original_filename = excluded.original_filename,
|
|
199
|
+
raw_size_bytes = excluded.raw_size_bytes,
|
|
200
|
+
signed_cdn_url = excluded.signed_cdn_url,
|
|
201
|
+
token_expiry_timestamp = excluded.token_expiry_timestamp
|
|
202
|
+
""",
|
|
203
|
+
(
|
|
204
|
+
album_id,
|
|
205
|
+
idx,
|
|
206
|
+
true_file_id,
|
|
207
|
+
file_rec.get("title", f"Track {idx}"),
|
|
208
|
+
file_rec.get("original", f"file_{idx}"),
|
|
209
|
+
file_rec.get("size", 0),
|
|
210
|
+
source_url,
|
|
211
|
+
cdn_url,
|
|
212
|
+
expiry_ts,
|
|
213
|
+
),
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
return album_id, new_count, updated_count
|
|
217
|
+
|
|
218
|
+
def get_all_albums(self) -> list[sqlite3.Row]:
|
|
219
|
+
"""Returns every album, plus a per-album total_assets/completed_assets
|
|
220
|
+
count so callers can show a '[COMPLETED]' badge without an extra
|
|
221
|
+
query per album. All original `albums` columns are still present.
|
|
222
|
+
"""
|
|
223
|
+
with closing(self._get_connection()) as conn:
|
|
224
|
+
return conn.execute("""
|
|
225
|
+
SELECT al.*,
|
|
226
|
+
COUNT(a.id) AS total_assets,
|
|
227
|
+
SUM(CASE WHEN a.download_status = 'COMPLETED' THEN 1 ELSE 0 END) AS completed_assets
|
|
228
|
+
FROM albums al
|
|
229
|
+
LEFT JOIN assets a ON a.album_id = al.id
|
|
230
|
+
GROUP BY al.id
|
|
231
|
+
ORDER BY al.updated_at DESC;
|
|
232
|
+
""").fetchall()
|
|
233
|
+
|
|
234
|
+
def get_album(self, album_id: int) -> sqlite3.Row | None:
|
|
235
|
+
"""Returns a single album's row, or None if no such album exists."""
|
|
236
|
+
with closing(self._get_connection()) as conn:
|
|
237
|
+
return conn.execute("SELECT * FROM albums WHERE id = ?;", (album_id,)).fetchone()
|
|
238
|
+
|
|
239
|
+
def delete_album(self, album_id: int) -> bool:
|
|
240
|
+
"""Deletes an album and all its assets (cascades via the assets
|
|
241
|
+
table's ON DELETE CASCADE foreign key — foreign_keys is enabled on
|
|
242
|
+
every connection in _get_connection()).
|
|
243
|
+
|
|
244
|
+
Returns True if a row was actually deleted, False if no album with
|
|
245
|
+
that id existed.
|
|
246
|
+
"""
|
|
247
|
+
with closing(self._get_connection()) as conn, conn:
|
|
248
|
+
cursor = conn.execute("DELETE FROM albums WHERE id = ?;", (album_id,))
|
|
249
|
+
return cursor.rowcount > 0
|
|
250
|
+
|
|
251
|
+
def get_staged_assets(self) -> list[sqlite3.Row]:
|
|
252
|
+
"""Returns every asset flagged as staged, joined with its parent
|
|
253
|
+
album's title.
|
|
254
|
+
|
|
255
|
+
Filters on the asset-level flag only. The album-level is_staged
|
|
256
|
+
flag is a display convenience (shows a '[STAGED]' badge in the
|
|
257
|
+
catalog for an album with any staged items) — it is deliberately
|
|
258
|
+
NOT used to gate which assets get downloaded here. Staging a whole
|
|
259
|
+
album sets both flags at once (see toggle_staging), but downloads
|
|
260
|
+
only ever clear the asset-level flag as each file finishes, so
|
|
261
|
+
joining on the album-level flag here would keep resurfacing
|
|
262
|
+
already-completed files for as long as any sibling asset in that
|
|
263
|
+
album remains staged.
|
|
264
|
+
"""
|
|
265
|
+
with closing(self._get_connection()) as conn:
|
|
266
|
+
return conn.execute("""
|
|
267
|
+
SELECT a.*, al.title AS album_title FROM assets a
|
|
268
|
+
LEFT JOIN albums al ON a.album_id = al.id
|
|
269
|
+
WHERE a.is_staged = 1
|
|
270
|
+
ORDER BY a.album_id, a.track_number ASC;
|
|
271
|
+
""").fetchall()
|
|
272
|
+
|
|
273
|
+
def sync_album_staged_flag(self, album_id: int):
|
|
274
|
+
"""Clears an album's is_staged badge once none of its assets are
|
|
275
|
+
still staged. Call this after an asset's is_staged flag is cleared
|
|
276
|
+
(e.g. on download completion) so the catalog's '[STAGED]' display
|
|
277
|
+
doesn't keep showing an album as staged after every file in it is
|
|
278
|
+
actually done.
|
|
279
|
+
"""
|
|
280
|
+
with closing(self._get_connection()) as conn, conn:
|
|
281
|
+
remaining = conn.execute(
|
|
282
|
+
"SELECT COUNT(*) AS c FROM assets WHERE album_id = ? AND is_staged = 1", (album_id,)
|
|
283
|
+
).fetchone()["c"]
|
|
284
|
+
if remaining == 0:
|
|
285
|
+
conn.execute("UPDATE albums SET is_staged = 0 WHERE id = ?;", (album_id,))
|
|
286
|
+
|
|
287
|
+
def get_failed_assets(self) -> list[sqlite3.Row]:
|
|
288
|
+
"""Returns every asset currently marked FAILED, joined with its
|
|
289
|
+
parent album's title.
|
|
290
|
+
"""
|
|
291
|
+
with closing(self._get_connection()) as conn:
|
|
292
|
+
return conn.execute("""
|
|
293
|
+
SELECT a.*, al.title AS album_title FROM assets a
|
|
294
|
+
LEFT JOIN albums al ON a.album_id = al.id
|
|
295
|
+
WHERE a.download_status = 'FAILED'
|
|
296
|
+
ORDER BY a.album_id, a.track_number ASC;
|
|
297
|
+
""").fetchall()
|
|
298
|
+
|
|
299
|
+
def get_album_assets(self, album_id: int) -> list[sqlite3.Row]:
|
|
300
|
+
with closing(self._get_connection()) as conn:
|
|
301
|
+
return conn.execute(
|
|
302
|
+
"SELECT * FROM assets WHERE album_id = ? ORDER BY track_number ASC;", (album_id,)
|
|
303
|
+
).fetchall()
|
|
304
|
+
|
|
305
|
+
def update_asset_url(self, asset_id: int, new_cdn_url: str):
|
|
306
|
+
expiry_ts = extract_expiry_from_url(new_cdn_url)
|
|
307
|
+
with closing(self._get_connection()) as conn, conn:
|
|
308
|
+
conn.execute(
|
|
309
|
+
"UPDATE assets SET signed_cdn_url = ?, token_expiry_timestamp = ? WHERE id = ?",
|
|
310
|
+
(new_cdn_url, expiry_ts, asset_id),
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
def update_download_status(
|
|
314
|
+
self, asset_id: int, status: str, local_path: str | None = None, error: str | None = None
|
|
315
|
+
):
|
|
316
|
+
with closing(self._get_connection()) as conn, conn:
|
|
317
|
+
conn.execute(
|
|
318
|
+
"UPDATE assets SET download_status = ?, local_file_path = ?, error_message = ? WHERE id = ?",
|
|
319
|
+
(status, local_path, error, asset_id),
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
def get_needs_refresh(self, album_id: int | None = None) -> list[sqlite3.Row]:
|
|
323
|
+
lookahead = int(self.get_config_val("token_buffer_seconds", "600"))
|
|
324
|
+
now_with_buffer = int(time.time()) + lookahead
|
|
325
|
+
with closing(self._get_connection()) as conn:
|
|
326
|
+
if album_id is not None:
|
|
327
|
+
return conn.execute(
|
|
328
|
+
"SELECT * FROM assets WHERE album_id = ? AND (token_expiry_timestamp IS NULL OR token_expiry_timestamp <= ?);",
|
|
329
|
+
(album_id, now_with_buffer),
|
|
330
|
+
).fetchall()
|
|
331
|
+
return conn.execute(
|
|
332
|
+
"SELECT * FROM assets WHERE token_expiry_timestamp IS NULL OR token_expiry_timestamp <= ?;",
|
|
333
|
+
(now_with_buffer,),
|
|
334
|
+
).fetchall()
|
|
335
|
+
|
|
336
|
+
def get_valid_url(self, asset_id: int) -> str:
|
|
337
|
+
with closing(self._get_connection()) as conn:
|
|
338
|
+
asset = conn.execute("SELECT * FROM assets WHERE id = ?;", (asset_id,)).fetchone()
|
|
339
|
+
if not asset:
|
|
340
|
+
return ""
|
|
341
|
+
lookahead = int(self.get_config_val("token_buffer_seconds", "600"))
|
|
342
|
+
if (
|
|
343
|
+
asset["signed_cdn_url"]
|
|
344
|
+
and asset["token_expiry_timestamp"]
|
|
345
|
+
and asset["token_expiry_timestamp"] > (time.time() + lookahead)
|
|
346
|
+
):
|
|
347
|
+
return asset["signed_cdn_url"]
|
|
348
|
+
from .tokens import mint_now
|
|
349
|
+
|
|
350
|
+
file_id = str(asset["true_file_id"]) if asset["true_file_id"] else asset["source_url"]
|
|
351
|
+
fresh_url = mint_now(file_id)
|
|
352
|
+
self.update_asset_url(asset_id, fresh_url)
|
|
353
|
+
return fresh_url
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import asyncio
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
import sys
|
|
6
|
+
import urllib.parse
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from bs4 import BeautifulSoup
|
|
10
|
+
from rich.console import Console
|
|
11
|
+
|
|
12
|
+
from ..config import (
|
|
13
|
+
DEFAULT_JSON_DIR,
|
|
14
|
+
DEFAULT_OUTPUT_DIR,
|
|
15
|
+
HEADERS,
|
|
16
|
+
SEARCH_MODES,
|
|
17
|
+
VALID_COUNTS,
|
|
18
|
+
)
|
|
19
|
+
from ..utils.formatting import slugify_filename
|
|
20
|
+
from ..utils.http import execute_request_with_retry_async
|
|
21
|
+
|
|
22
|
+
console = Console()
|
|
23
|
+
|
|
24
|
+
class ScraperEngine:
|
|
25
|
+
def __init__(self, db):
|
|
26
|
+
"""
|
|
27
|
+
Initializes the scraper engine with a shared database instance.
|
|
28
|
+
"""
|
|
29
|
+
self.db = db
|
|
30
|
+
|
|
31
|
+
def _safe_int(self, val):
|
|
32
|
+
"""Helper to prevent ValueError crashes during parsing of sizes or counts."""
|
|
33
|
+
if isinstance(val, int):
|
|
34
|
+
return val
|
|
35
|
+
if not val:
|
|
36
|
+
return 0
|
|
37
|
+
try:
|
|
38
|
+
clean_val = re.sub(r'[^\d]', '', str(val))
|
|
39
|
+
return int(clean_val) if clean_val else 0
|
|
40
|
+
except Exception:
|
|
41
|
+
return 0
|
|
42
|
+
|
|
43
|
+
def standardize_top_url(self, url: str) -> str:
|
|
44
|
+
"""Converts specific /v/ and /i/ redirect configurations to /f/."""
|
|
45
|
+
return re.sub(r'/(v|i)/', '/f/', url)
|
|
46
|
+
|
|
47
|
+
def parse_albums(self, html):
|
|
48
|
+
"""Extract album information and file counts from search results page."""
|
|
49
|
+
soup = BeautifulSoup(html, 'html.parser')
|
|
50
|
+
albums = []
|
|
51
|
+
|
|
52
|
+
for card in soup.find_all('a', href=True):
|
|
53
|
+
href = card.get('href', '')
|
|
54
|
+
|
|
55
|
+
if re.search(r'/a/[\w-]+', href):
|
|
56
|
+
title_tag = card.find('h3')
|
|
57
|
+
title = title_tag.get_text(strip=True) if title_tag else card.get_text(strip=True)
|
|
58
|
+
|
|
59
|
+
file_count = None
|
|
60
|
+
span_tags = card.find_all('span')
|
|
61
|
+
for span in span_tags:
|
|
62
|
+
span_text = span.get_text(strip=True)
|
|
63
|
+
if 'file' in span_text.lower():
|
|
64
|
+
file_count = span_text
|
|
65
|
+
break
|
|
66
|
+
|
|
67
|
+
if title:
|
|
68
|
+
albums.append({
|
|
69
|
+
'title': title,
|
|
70
|
+
'url': href if href.startswith('http') else 'https://bunkr.cr' + href,
|
|
71
|
+
'file_count': file_count if file_count else "0 files"
|
|
72
|
+
})
|
|
73
|
+
|
|
74
|
+
seen = set()
|
|
75
|
+
unique_albums = []
|
|
76
|
+
for album in albums:
|
|
77
|
+
if album['url'] not in seen:
|
|
78
|
+
unique_albums.append(album)
|
|
79
|
+
seen.add(album['url'])
|
|
80
|
+
|
|
81
|
+
return unique_albums
|
|
82
|
+
|
|
83
|
+
def parse_top_items(self, html, category):
|
|
84
|
+
"""Extract and isolate trending listings for the 'Trending' CLI feature."""
|
|
85
|
+
soup = BeautifulSoup(html, 'html.parser')
|
|
86
|
+
items = []
|
|
87
|
+
prefix_map = {"albums": "a", "videos": "v", "files": "f", "images": "i"}
|
|
88
|
+
target_prefix = prefix_map.get(category, "f")
|
|
89
|
+
|
|
90
|
+
for card in soup.find_all('a', href=True):
|
|
91
|
+
href = card.get('href', '')
|
|
92
|
+
if re.search(rf'/{target_prefix}/[\w-]+', href):
|
|
93
|
+
title_tag = card.find('h3') or card.find('p')
|
|
94
|
+
title = title_tag.get_text(strip=True) if title_tag else card.get_text(strip=True)
|
|
95
|
+
full_url = href if href.startswith('http') else 'https://bunkr.cr' + href
|
|
96
|
+
|
|
97
|
+
if category in ["videos", "images"]:
|
|
98
|
+
full_url = self.standardize_top_url(full_url)
|
|
99
|
+
|
|
100
|
+
file_count = "1 file"
|
|
101
|
+
for span in card.find_all('span'):
|
|
102
|
+
if 'file' in span.get_text(strip=True).lower():
|
|
103
|
+
file_count = span.get_text(strip=True)
|
|
104
|
+
break
|
|
105
|
+
if title:
|
|
106
|
+
items.append({'title': title, 'url': full_url, 'file_count': file_count})
|
|
107
|
+
seen = set()
|
|
108
|
+
return [x for x in items if not (x['url'] in seen or seen.add(x['url']))]
|
|
109
|
+
|
|
110
|
+
def extract_page_metadata(self, soup):
|
|
111
|
+
"""Parses total global pages from the HTML pagination section of a search results page."""
|
|
112
|
+
footer_div = soup.find('div', class_='text-xs text-[var(--text-soft)] mono')
|
|
113
|
+
if footer_div:
|
|
114
|
+
text = footer_div.get_text(strip=True)
|
|
115
|
+
match = re.search(r'Page\s+\d+\s+of\s+(\d+)', text, re.IGNORECASE)
|
|
116
|
+
if match:
|
|
117
|
+
return match.group(1)
|
|
118
|
+
|
|
119
|
+
top_span = soup.find('span', class_='text-[var(--text)]')
|
|
120
|
+
if top_span:
|
|
121
|
+
parent_text = top_span.parent.get_text(strip=True) if top_span.parent else ""
|
|
122
|
+
match = re.search(r'page\s+\d+\s+of\s+(\d+)', parent_text, re.IGNORECASE)
|
|
123
|
+
if match:
|
|
124
|
+
return match.group(1)
|
|
125
|
+
|
|
126
|
+
return "Unknown"
|
|
127
|
+
|
|
128
|
+
def parse_album_header_metadata(self, soup):
|
|
129
|
+
"""Extract full album size and total files from the visitors paragraph."""
|
|
130
|
+
album_size = None
|
|
131
|
+
total_files = None
|
|
132
|
+
|
|
133
|
+
size_el = soup.select_one(".visitors .font-semibold")
|
|
134
|
+
if size_el:
|
|
135
|
+
text = size_el.get_text(strip=True)
|
|
136
|
+
size_match = re.search(r'\((.*?)\)', text)
|
|
137
|
+
if size_match:
|
|
138
|
+
album_size = size_match.group(1)
|
|
139
|
+
|
|
140
|
+
files_match = re.search(r'\)\s*(.*)', text)
|
|
141
|
+
if files_match:
|
|
142
|
+
total_files = files_match.group(1).strip()
|
|
143
|
+
|
|
144
|
+
return album_size, total_files
|
|
145
|
+
|
|
146
|
+
def extract_advanced_album_files(self, html_content: str) -> list:
|
|
147
|
+
"""RESTORED: Exact regex and logic to handle 'slug' vs 'name' vs 'original'."""
|
|
148
|
+
match = re.search(r'window\.albumFiles\s*=\s*\[(.*?)\];', html_content, re.DOTALL)
|
|
149
|
+
if not match:
|
|
150
|
+
return []
|
|
151
|
+
|
|
152
|
+
array_content = match.group(1)
|
|
153
|
+
object_strings = re.findall(r'\{([^}]+)\}', array_content, re.DOTALL)
|
|
154
|
+
parsed_files = []
|
|
155
|
+
|
|
156
|
+
for obj_str in object_strings:
|
|
157
|
+
file_meta = {}
|
|
158
|
+
matches = re.findall(r'(\w+):\s*(?:"([^"]*)"|(\d+))', obj_str)
|
|
159
|
+
|
|
160
|
+
for key, str_val, num_val in matches:
|
|
161
|
+
if num_val:
|
|
162
|
+
file_meta[key] = int(num_val)
|
|
163
|
+
else:
|
|
164
|
+
file_meta[key] = str_val
|
|
165
|
+
|
|
166
|
+
if file_meta:
|
|
167
|
+
slug = file_meta.get('slug') or file_meta.get('name')
|
|
168
|
+
display_name = file_meta.get('original') or file_meta.get('name', 'Unknown Title')
|
|
169
|
+
|
|
170
|
+
parsed_files.append({
|
|
171
|
+
'true_file_id': file_meta.get('id'),
|
|
172
|
+
'slug_id': file_meta.get('id'),
|
|
173
|
+
'href': f"https://bunkr.cr/f/{urllib.parse.quote(str(slug))}" if slug else None,
|
|
174
|
+
'title': display_name,
|
|
175
|
+
'original': display_name,
|
|
176
|
+
'size': self._safe_int(file_meta.get('size', 0)),
|
|
177
|
+
**{k: v for k, v in file_meta.items() if k not in ['id', 'name', 'slug', 'original', 'size']}
|
|
178
|
+
})
|
|
179
|
+
|
|
180
|
+
return parsed_files
|
|
181
|
+
|
|
182
|
+
async def scrape_album(self, session, url, search_term, album_number_index=0, save_json=False, output_dir=None):
|
|
183
|
+
"""Deep resolution of an album with original informative console logging."""
|
|
184
|
+
parsed_url = urllib.parse.urlparse(url)
|
|
185
|
+
params = dict(urllib.parse.parse_qsl(parsed_url.query))
|
|
186
|
+
params['advanced'] = '1'
|
|
187
|
+
optimized_url = urllib.parse.urlunparse(
|
|
188
|
+
parsed_url._replace(query=urllib.parse.urlencode(params))
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
console.print("\n[bold yellow][*][/bold yellow] Navigating optimized album view ")
|
|
192
|
+
|
|
193
|
+
res = await execute_request_with_retry_async(session, optimized_url, headers=HEADERS)
|
|
194
|
+
soup = BeautifulSoup(res.text, 'html.parser')
|
|
195
|
+
|
|
196
|
+
album_title = (
|
|
197
|
+
re.sub(r'\s*[\|\-]\s*Bunkr\s*$', '', soup.title.string, flags=re.IGNORECASE)
|
|
198
|
+
if soup.title else "Unknown Album"
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
console.print(f"\n[bold green][+][/bold green] Selected: #[bold yellow]{album_number_index}[/bold yellow] - [bold white]{album_title}[/bold white]")
|
|
202
|
+
|
|
203
|
+
album_size, total_files = self.parse_album_header_metadata(soup)
|
|
204
|
+
final_files = self.extract_advanced_album_files(res.text)
|
|
205
|
+
|
|
206
|
+
if album_size and total_files:
|
|
207
|
+
console.print(f"[bold green][+][/bold green] Album Info -> Aggregate Size: [bold yellow]{album_size}[/bold yellow] | Count: [bold yellow]{total_files}[/bold yellow]")
|
|
208
|
+
|
|
209
|
+
console.print(f"[bold green][+][/bold green] Deep resolution complete. Pulled [bold cyan]{len(final_files)}[/bold cyan] accurate item profiles instantly.")
|
|
210
|
+
|
|
211
|
+
for i, f_rec in enumerate(final_files[:10], start=1):
|
|
212
|
+
raw_size = f_rec.get('size', 0)
|
|
213
|
+
size_mb = round(raw_size / (1024*1024), 2)
|
|
214
|
+
size_str = f" ({size_mb} MB)" if raw_size else ""
|
|
215
|
+
console.print(f" {i}. {f_rec['title']}{size_str} [[italic green]True[/italic green] ID: {f_rec['true_file_id']}]")
|
|
216
|
+
|
|
217
|
+
# Stable identity for this album, independent of its position in any
|
|
218
|
+
# given search results page (which shifts between scrapes and previously
|
|
219
|
+
# caused re-scrapes to register as phantom duplicate albums).
|
|
220
|
+
slug_match = re.search(r'/a/([\w-]+)', url)
|
|
221
|
+
album_slug = slug_match.group(1) if slug_match else None
|
|
222
|
+
|
|
223
|
+
data = {
|
|
224
|
+
'search_term': search_term or "N/A",
|
|
225
|
+
'selected_album': {
|
|
226
|
+
'title': album_title.strip(),
|
|
227
|
+
'album_index_number': album_number_index,
|
|
228
|
+
'album_slug': album_slug,
|
|
229
|
+
'album_url': url,
|
|
230
|
+
'aggregate_size': album_size if album_size else "0 MB",
|
|
231
|
+
'clean_file_count': total_files if total_files else f"{len(final_files)} files"
|
|
232
|
+
},
|
|
233
|
+
'files_found': final_files
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
if save_json:
|
|
237
|
+
# Determine correct output directory
|
|
238
|
+
save_path = Path(output_dir) if output_dir else DEFAULT_OUTPUT_DIR
|
|
239
|
+
save_path.mkdir(parents=True, exist_ok=True)
|
|
240
|
+
|
|
241
|
+
output_filename = save_path / f"{slugify_filename(album_number_index, album_title)}.json"
|
|
242
|
+
try:
|
|
243
|
+
payload = json.dumps(data, indent=2, ensure_ascii=False)
|
|
244
|
+
await asyncio.to_thread(output_filename.write_text, payload, encoding="utf-8")
|
|
245
|
+
console.print(f"[bold green][+][/bold green] Enriched results saved out to [bold white]{output_filename}[/bold white]")
|
|
246
|
+
except (OSError, TypeError, ValueError) as e:
|
|
247
|
+
console.print(f"[red][!] JSON Backup Failed: {e}[/red]")
|
|
248
|
+
|
|
249
|
+
console.print("[bold yellow][*][/bold yellow] Syncing records with Database Manager...")
|
|
250
|
+
album_id, new_count, updated_count = self.db.register_album_from_json(data)
|
|
251
|
+
|
|
252
|
+
if new_count and updated_count:
|
|
253
|
+
console.print(
|
|
254
|
+
f"[bold green][+][/bold green] Synced Album ID [bold yellow]#{album_id}[/bold yellow] — "
|
|
255
|
+
f"[bold cyan]{new_count}[/bold cyan] new file(s), [bold cyan]{updated_count}[/bold cyan] refreshed."
|
|
256
|
+
)
|
|
257
|
+
elif new_count:
|
|
258
|
+
console.print(
|
|
259
|
+
f"[bold green][+][/bold green] Registered Album ID [bold yellow]#{album_id}[/bold yellow] with "
|
|
260
|
+
f"[bold cyan]{new_count}[/bold cyan] file(s)."
|
|
261
|
+
)
|
|
262
|
+
else:
|
|
263
|
+
console.print(
|
|
264
|
+
f"[bold green][+][/bold green] Album ID [bold yellow]#{album_id}[/bold yellow] already up to date "
|
|
265
|
+
f"— [bold cyan]{updated_count}[/bold cyan] file(s) refreshed, no new files found."
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
return album_id
|
|
269
|
+
|
|
270
|
+
def main():
|
|
271
|
+
"""Standalone CLI entry point for bunkr-scrape command."""
|
|
272
|
+
parser = argparse.ArgumentParser(
|
|
273
|
+
prog="bunkr-scrape",
|
|
274
|
+
description="Bunkr Standalone Scraper CLI"
|
|
275
|
+
)
|
|
276
|
+
parser.add_argument("search", nargs="?", default=None, help="The search query")
|
|
277
|
+
parser.add_argument("-m", "--mode", choices=list(SEARCH_MODES.keys()), help="Search mode")
|
|
278
|
+
parser.add_argument("-p", "--per", type=int, choices=VALID_COUNTS, help="No. of results per page")
|
|
279
|
+
parser.add_argument("-s", "--sort", choices=["latest", "oldest", "mostfiles"], help="Sort by")
|
|
280
|
+
parser.add_argument("-t", "--top", nargs="?", const="prompt", help="Trending category")
|
|
281
|
+
parser.add_argument("--save-json", action="store_true", help="Save backup JSON")
|
|
282
|
+
parser.add_argument("-o", "--output", type=str, default=str(DEFAULT_JSON_DIR), help="Output directory for a JSON metadata")
|
|
283
|
+
|
|
284
|
+
args = parser.parse_args()
|
|
285
|
+
|
|
286
|
+
url_mode = SEARCH_MODES.get(args.mode) if args.mode else None
|
|
287
|
+
|
|
288
|
+
from ..cli import run_scrape_interactive, run_top_engine_interactive
|
|
289
|
+
|
|
290
|
+
async def _run():
|
|
291
|
+
if args.top:
|
|
292
|
+
await run_top_engine_interactive(
|
|
293
|
+
category_seed=args.top if args.top != "prompt" else None,
|
|
294
|
+
save_json_seed=args.save_json,
|
|
295
|
+
output_dir_seed=args.output
|
|
296
|
+
)
|
|
297
|
+
else:
|
|
298
|
+
await run_scrape_interactive(
|
|
299
|
+
search_seed=args.search,
|
|
300
|
+
mode_seed=url_mode,
|
|
301
|
+
per_seed=args.per,
|
|
302
|
+
sort_seed=args.sort,
|
|
303
|
+
save_json_seed=args.save_json,
|
|
304
|
+
output_dir_seed=args.output
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
try:
|
|
308
|
+
asyncio.run(_run())
|
|
309
|
+
except KeyboardInterrupt:
|
|
310
|
+
sys.exit(0)
|