playlist-admin 2.6.80 → 2.6.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/index.js +99 -94
- package/package.json +4 -6
- package/rag/README.md +48 -0
- package/rag/__pycache__/_resolve.cpython-311.pyc +0 -0
- package/rag/__pycache__/build_db.cpython-311.pyc +0 -0
- package/rag/__pycache__/query.cpython-311.pyc +0 -0
- package/rag/__pycache__/srt_to_text.cpython-311.pyc +0 -0
- package/rag/_resolve.py +35 -0
- package/rag/build_db.py +213 -0
- package/rag/query.py +140 -0
- package/rag/srt_to_text.py +50 -0
- package/COMMANDS.md +0 -263
- package/assets/zhcdict.json +0 -26406
- package/cli/lib/audio_converter.js +0 -42
- package/cli/lib/cleanup.js +0 -61
- package/cli/lib/config.js +0 -166
- package/cli/lib/library_index.js +0 -370
- package/cli/lib/lufs.js +0 -178
- package/cli/lib/metadata.js +0 -31
- package/cli/lib/pipeline.js +0 -493
- package/cli/lib/playlist_parser.js +0 -85
- package/cli/lib/podcast.js +0 -572
- package/cli/lib/scraper.js +0 -267
- package/cli/lib/snapshot.js +0 -42
- package/cli/lib/spotube.js +0 -452
- package/cli/lib/util.js +0 -139
- package/cli/lib/zhcn.js +0 -68
|
@@ -1,85 +0,0 @@
|
|
|
1
|
-
import fs from 'node:fs';
|
|
2
|
-
import { stemOf } from './util.js';
|
|
3
|
-
|
|
4
|
-
// Port of lib/services/playlist_parser.dart
|
|
5
|
-
function decodePath(s) {
|
|
6
|
-
try { return decodeURIComponent(s); } catch { return s; }
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
export class PlaylistParser {
|
|
10
|
-
static isInternalPlaylist(name) {
|
|
11
|
-
const n = name.toLowerCase();
|
|
12
|
-
if (n.includes('_removed songs')) return true;
|
|
13
|
-
return ['_unsorted', 'single tracks', '_unsorted_songs'].some((m) => n.includes(m));
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
static parseTrackNames(filePath) {
|
|
17
|
-
const songs = [];
|
|
18
|
-
if (!fs.existsSync(filePath)) return songs;
|
|
19
|
-
let content;
|
|
20
|
-
try { content = fs.readFileSync(filePath, 'utf-8'); }
|
|
21
|
-
catch { return songs; }
|
|
22
|
-
|
|
23
|
-
const lines = content.split('\n').map((l) => l.trim()).filter((l) => l.length > 0);
|
|
24
|
-
if (lines.length === 0) return songs;
|
|
25
|
-
const isM3u = lines.some((l) => l.includes('#EXTM3U'));
|
|
26
|
-
|
|
27
|
-
let i = 0;
|
|
28
|
-
while (i < lines.length) {
|
|
29
|
-
const line = lines[i];
|
|
30
|
-
if (isM3u && line.startsWith('#EXTINF:')) {
|
|
31
|
-
let j = i + 1;
|
|
32
|
-
while (j < lines.length && (lines[j].startsWith('#') || lines[j].length === 0)) j++;
|
|
33
|
-
if (j < lines.length) {
|
|
34
|
-
const pathLine = lines[j];
|
|
35
|
-
const name = pathLine.includes('\\') || pathLine.includes('/')
|
|
36
|
-
? pathLine.replace(/[\\/]/g, '\\').split('\\').pop()
|
|
37
|
-
: pathLine;
|
|
38
|
-
songs.push(decodePath(name).replace(/\.[^.]+$/, ''));
|
|
39
|
-
i = j + 1;
|
|
40
|
-
} else {
|
|
41
|
-
i++;
|
|
42
|
-
}
|
|
43
|
-
} else {
|
|
44
|
-
if (!line.startsWith('#')) {
|
|
45
|
-
songs.push(line);
|
|
46
|
-
}
|
|
47
|
-
i++;
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
return songs;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
static parseTrackEntries(filePath) {
|
|
54
|
-
const entries = [];
|
|
55
|
-
if (!fs.existsSync(filePath)) return entries;
|
|
56
|
-
let content;
|
|
57
|
-
try { content = fs.readFileSync(filePath, 'utf-8'); }
|
|
58
|
-
catch { return entries; }
|
|
59
|
-
|
|
60
|
-
const lines = content.split('\n').map((l) => l.trim()).filter((l) => l.length > 0);
|
|
61
|
-
if (lines.length === 0) return entries;
|
|
62
|
-
const isM3u = lines.some((l) => l.includes('#EXTM3U'));
|
|
63
|
-
|
|
64
|
-
let i = 0;
|
|
65
|
-
while (i < lines.length) {
|
|
66
|
-
const line = lines[i];
|
|
67
|
-
if (isM3u && line.startsWith('#EXTINF:')) {
|
|
68
|
-
let j = i + 1;
|
|
69
|
-
while (j < lines.length && (lines[j].startsWith('#') || lines[j].length === 0)) j++;
|
|
70
|
-
if (j < lines.length) {
|
|
71
|
-
entries.push(decodePath(lines[j]));
|
|
72
|
-
i = j + 1;
|
|
73
|
-
} else {
|
|
74
|
-
i++;
|
|
75
|
-
}
|
|
76
|
-
} else {
|
|
77
|
-
if (!line.startsWith('#')) {
|
|
78
|
-
entries.push(line);
|
|
79
|
-
}
|
|
80
|
-
i++;
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
return entries;
|
|
84
|
-
}
|
|
85
|
-
}
|
package/cli/lib/podcast.js
DELETED
|
@@ -1,572 +0,0 @@
|
|
|
1
|
-
import fs from 'node:fs';
|
|
2
|
-
import path from 'node:path';
|
|
3
|
-
import os from 'node:os';
|
|
4
|
-
import { spawn } from 'node:child_process';
|
|
5
|
-
import { run, runStream, sleep, normalizeFileName, stemOf } from './util.js';
|
|
6
|
-
|
|
7
|
-
const CACHE_FILE = 'podcast_processed_cache.json';
|
|
8
|
-
const SRT_LANGS = ['zh-TW', 'zh-Hant', 'zh', 'zh-Hans', 'en'];
|
|
9
|
-
|
|
10
|
-
// ---------- RSS ----------
|
|
11
|
-
function parseRss(xml) {
|
|
12
|
-
const items = [];
|
|
13
|
-
const itemRe = /<item>([\s\S]*?)<\/item>/g;
|
|
14
|
-
let m;
|
|
15
|
-
while ((m = itemRe.exec(xml)) != null) {
|
|
16
|
-
const block = m[1];
|
|
17
|
-
const title = (block.match(/<title>([\s\S]*?)<\/title>/) || [])[1] || '';
|
|
18
|
-
const pubDate = (block.match(/<pubDate>([\s\S]*?)<\/pubDate>/) || [])[1] || '';
|
|
19
|
-
const enclosure = block.match(/<enclosure[^>]*\/?>/);
|
|
20
|
-
let audioUrl = '';
|
|
21
|
-
let audioType = '';
|
|
22
|
-
if (enclosure) {
|
|
23
|
-
const u = enclosure[0].match(/url="\s*([^"]+)"/);
|
|
24
|
-
const t = enclosure[0].match(/type="([^"]+)"/);
|
|
25
|
-
if (u) audioUrl = decodeXml(u[1]);
|
|
26
|
-
if (t) audioType = t[1];
|
|
27
|
-
}
|
|
28
|
-
const duration = (block.match(/<itunes:duration>([\s\S]*?)<\/itunes:duration>/) || [])[1] || '';
|
|
29
|
-
const description = (block.match(/<description>([\s\S]*?)<\/description>/) || [])[1] || '';
|
|
30
|
-
items.push({
|
|
31
|
-
title: decodeXml(title),
|
|
32
|
-
pub_date: decodeXml(pubDate),
|
|
33
|
-
audio_url: audioUrl,
|
|
34
|
-
audio_type: audioType,
|
|
35
|
-
duration: decodeXml(duration),
|
|
36
|
-
description: decodeXml(description),
|
|
37
|
-
});
|
|
38
|
-
}
|
|
39
|
-
const titleMatch = xml.match(/<channel>[\s\S]*?<title>([\s\S]*?)<\/title>/);
|
|
40
|
-
return {
|
|
41
|
-
title: titleMatch ? decodeXml(titleMatch[1]) : '',
|
|
42
|
-
episodes: items,
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
function decodeXml(s) {
|
|
47
|
-
return s
|
|
48
|
-
.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
|
|
49
|
-
.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
|
50
|
-
.replace(/"/g, '"').replace(/'/g, "'").replace(/'/g, "'")
|
|
51
|
-
.trim();
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
function guessExtension(url) {
|
|
55
|
-
try {
|
|
56
|
-
const p = new URL(url).pathname.toLowerCase();
|
|
57
|
-
for (const ext of ['mp3', 'm4a', 'wav', 'ogg', 'flac', 'aac']) {
|
|
58
|
-
if (p.endsWith(`.${ext}`)) return ext;
|
|
59
|
-
}
|
|
60
|
-
} catch {}
|
|
61
|
-
return 'mp3';
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
export class PodcastService {
|
|
65
|
-
constructor(basePath) {
|
|
66
|
-
this.basePath = basePath;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
podcastDir(podcastName) {
|
|
70
|
-
const sub = podcastName ? podcastName.replace(/[<>:"/\\|?*]/g, '_') : '';
|
|
71
|
-
const dir = sub
|
|
72
|
-
? path.join(this.basePath, 'podcast_downloads', sub)
|
|
73
|
-
: path.join(this.basePath, 'podcast_downloads');
|
|
74
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
75
|
-
return dir;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
async fetchEpisodes(rssUrl) {
|
|
79
|
-
const resp = await fetch(rssUrl, {
|
|
80
|
-
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' },
|
|
81
|
-
signal: AbortSignal.timeout(30000),
|
|
82
|
-
});
|
|
83
|
-
if (!resp.ok) throw new Error(`RSS HTTP ${resp.status}`);
|
|
84
|
-
const xml = await resp.text();
|
|
85
|
-
return parseRss(xml);
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
async downloadEpisode(rssUrl, index, podcastName) {
|
|
89
|
-
const result = await this.fetchEpisodes(rssUrl);
|
|
90
|
-
const ep = result.episodes[index];
|
|
91
|
-
if (!ep || !ep.audio_url) throw new Error('No audio URL found');
|
|
92
|
-
const name = normalizeFileName(ep.title || `episode_${index}`);
|
|
93
|
-
const ext = guessExtension(ep.audio_url);
|
|
94
|
-
const outDir = this.podcastDir(podcastName);
|
|
95
|
-
const outputPath = path.join(outDir, `${name}.${ext}`);
|
|
96
|
-
if (fs.existsSync(outputPath)) return false;
|
|
97
|
-
|
|
98
|
-
const resp = await fetch(ep.audio_url, {
|
|
99
|
-
signal: AbortSignal.timeout(600000),
|
|
100
|
-
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' },
|
|
101
|
-
});
|
|
102
|
-
if (!resp.ok) throw new Error(`Download HTTP ${resp.status}`);
|
|
103
|
-
const total = parseInt(resp.headers.get('content-length') || '0', 10);
|
|
104
|
-
let downloaded = 0;
|
|
105
|
-
let lastPct = -1;
|
|
106
|
-
const ws = fs.createWriteStream(outputPath + '.part');
|
|
107
|
-
const reader = resp.body.getReader();
|
|
108
|
-
while (true) {
|
|
109
|
-
const { done, value } = await reader.read();
|
|
110
|
-
if (done) break;
|
|
111
|
-
ws.write(value);
|
|
112
|
-
downloaded += value.length;
|
|
113
|
-
if (total > 0) {
|
|
114
|
-
const pct = Math.trunc(downloaded / total * 100);
|
|
115
|
-
if (pct !== lastPct) {
|
|
116
|
-
lastPct = pct;
|
|
117
|
-
process.stdout.write(`\r ⬇️ ${pct}%`);
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
ws.end();
|
|
122
|
-
if (total > 0 && lastPct !== 100) process.stdout.write('\r ⬇️ 100%');
|
|
123
|
-
process.stdout.write('\n');
|
|
124
|
-
await new Promise((r) => ws.on('finish', r));
|
|
125
|
-
fs.renameSync(outputPath + '.part', outputPath);
|
|
126
|
-
return true;
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
async downloadSubtitles(episodeTitle, podcastName, onLog) {
|
|
130
|
-
const query = `${episodeTitle} ${podcastName}`;
|
|
131
|
-
const outDir = this.podcastDir(podcastName);
|
|
132
|
-
const safeName = normalizeFileName(episodeTitle);
|
|
133
|
-
const outputPath = path.join(outDir, `${safeName}.mp3`);
|
|
134
|
-
|
|
135
|
-
if (fs.existsSync(outputPath.replace(/\.mp3$/, '.srt'))) {
|
|
136
|
-
onLog('⏭️ 已有字幕,跳過');
|
|
137
|
-
return 'found';
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
onLog(`🔍 搜尋 YouTube: ${query}`);
|
|
141
|
-
let videoId = null;
|
|
142
|
-
try {
|
|
143
|
-
const searchUrl = 'https://www.youtube.com/results?' + new URLSearchParams({ search_query: query });
|
|
144
|
-
const resp = await fetch(searchUrl, {
|
|
145
|
-
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' },
|
|
146
|
-
signal: AbortSignal.timeout(15000),
|
|
147
|
-
});
|
|
148
|
-
const html = await resp.text();
|
|
149
|
-
const vids = [...html.matchAll(/watch\?v=([a-zA-Z0-9_-]{11})/g)].map((m) => m[1]);
|
|
150
|
-
const unique = [...new Set(vids)];
|
|
151
|
-
if (unique.length === 0) {
|
|
152
|
-
onLog('❌ 找不到符合的 YouTube 影片');
|
|
153
|
-
return 'not_found';
|
|
154
|
-
}
|
|
155
|
-
videoId = unique[0];
|
|
156
|
-
onLog(` ✅ 找到影片: https://youtube.com/watch?v=${videoId}`);
|
|
157
|
-
} catch (e) {
|
|
158
|
-
onLog(`❌ 搜尋失敗: ${e.message}`);
|
|
159
|
-
return 'failed';
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
// Download subtitles via yt-dlp
|
|
163
|
-
const srtPath = outputPath.replace(/\.mp3$/, '.srt');
|
|
164
|
-
fs.mkdirSync(path.dirname(srtPath), { recursive: true });
|
|
165
|
-
const outtmpl = srtPath.replace(/\.srt$/, '.%(ext)s');
|
|
166
|
-
const ytdlp = process.env.YTDLP_PATH || 'yt-dlp';
|
|
167
|
-
const args = [
|
|
168
|
-
'--quiet', '--no-warnings',
|
|
169
|
-
'--skip-download',
|
|
170
|
-
'--write-subs', '--write-auto-subs',
|
|
171
|
-
'--sub-langs', SRT_LANGS.join(','),
|
|
172
|
-
'--sub-format', 'srt',
|
|
173
|
-
'--output', outtmpl,
|
|
174
|
-
'--windows-filenames',
|
|
175
|
-
'--sleep-requests', '3',
|
|
176
|
-
`https://www.youtube.com/watch?v=${videoId}`,
|
|
177
|
-
];
|
|
178
|
-
const cookieFile = path.join(os.homedir(), 'Desktop', 'thumb', '大拇哥實驗室', 'cookies.txt');
|
|
179
|
-
if (fs.existsSync(cookieFile)) args.push('--cookies', cookieFile);
|
|
180
|
-
|
|
181
|
-
const result = await run(ytdlp, args, { timeout: 180000 });
|
|
182
|
-
if (result.code !== 0) {
|
|
183
|
-
onLog(`❌ 下載字幕失敗(yt-dlp exit ${result.code})`);
|
|
184
|
-
return 'failed';
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
// Find the SRT (may have language suffix)
|
|
188
|
-
let srtFound = null;
|
|
189
|
-
try {
|
|
190
|
-
const base = srtPath.replace(/\.srt$/, '');
|
|
191
|
-
for (const f of fs.readdirSync(path.dirname(srtPath))) {
|
|
192
|
-
if (f.startsWith(path.basename(base)) && f.toLowerCase().endsWith('.srt')) {
|
|
193
|
-
srtFound = path.join(path.dirname(srtPath), f);
|
|
194
|
-
break;
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
} catch {}
|
|
198
|
-
if (srtFound && fs.existsSync(srtFound)) {
|
|
199
|
-
if (srtFound !== srtPath) fs.renameSync(srtFound, srtPath);
|
|
200
|
-
onLog(` ✅ 字幕已儲存: ${srtPath}`);
|
|
201
|
-
return 'found';
|
|
202
|
-
}
|
|
203
|
-
onLog('❌ 下載字幕失敗(無可用字幕)');
|
|
204
|
-
return 'not_found';
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
async searchPodcasts(query) {
|
|
208
|
-
const url = `https://itunes.apple.com/search?term=${encodeURIComponent(query)}&media=podcast&limit=20`;
|
|
209
|
-
const resp = await fetch(url, {
|
|
210
|
-
headers: { 'User-Agent': 'PlaylistAdministrator/2.0' },
|
|
211
|
-
signal: AbortSignal.timeout(30000),
|
|
212
|
-
});
|
|
213
|
-
if (resp.status !== 200) throw new Error(`搜尋失敗 (${resp.status})`);
|
|
214
|
-
const data = await resp.json();
|
|
215
|
-
return (data.results || [])
|
|
216
|
-
.map((r) => ({
|
|
217
|
-
name: r.collectionName || '',
|
|
218
|
-
feedUrl: r.feedUrl || '',
|
|
219
|
-
artist: r.artistName || '',
|
|
220
|
-
artworkUrl: r.artworkUrl100 || '',
|
|
221
|
-
episodeCount: r.trackCount || 0,
|
|
222
|
-
}))
|
|
223
|
-
.filter((r) => r.feedUrl);
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
// ---------- Groq transcription (chunking + fetch) ----------
|
|
228
|
-
async function transcribeAudio(audioPath, keys, model, language, onLog) {
|
|
229
|
-
const ffmpeg = process.env.FFMPEG_PATH || 'ffmpeg';
|
|
230
|
-
let fileToSend = audioPath;
|
|
231
|
-
let tempFile = null;
|
|
232
|
-
const size = fs.existsSync(audioPath) ? fs.statSync(audioPath).size : 0;
|
|
233
|
-
|
|
234
|
-
// Convert to flac for small files
|
|
235
|
-
if (size > 0 && size <= 20 * 1024 * 1024 && !audioPath.toLowerCase().endsWith('.flac')) {
|
|
236
|
-
const tmp = path.join(os.tmpdir(), `pa_conv_${Date.now()}_${Math.random().toString(36).slice(2)}.flac`);
|
|
237
|
-
const r = await run(ffmpeg, ['-y', '-i', audioPath, '-ar', '16000', '-ac', '1', '-map', '0:a', '-c:a', 'flac', '-compression_level', '0', tmp], { timeout: 180000 });
|
|
238
|
-
if (r.code === 0 && fs.existsSync(tmp) && fs.statSync(tmp).size > 0) {
|
|
239
|
-
fileToSend = tmp;
|
|
240
|
-
tempFile = tmp;
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
// Chunk >20MB into 5-min pieces
|
|
245
|
-
let chunkFiles = [fileToSend];
|
|
246
|
-
if (size > 20 * 1024 * 1024) {
|
|
247
|
-
try {
|
|
248
|
-
onLog('Splitting audio...');
|
|
249
|
-
const dur = await run('ffprobe', ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', fileToSend], { timeout: 30000 });
|
|
250
|
-
const duration = parseFloat((dur.stdout || '').trim());
|
|
251
|
-
if (duration > 0) {
|
|
252
|
-
const base = path.basename(fileToSend).replace(/\.[^.]+$/, '');
|
|
253
|
-
const chunks = [];
|
|
254
|
-
const chunkSec = 300;
|
|
255
|
-
for (let s = 0; s < Math.floor(duration); s += chunkSec) {
|
|
256
|
-
const cp = path.join(os.tmpdir(), `chunk_${base}_${chunks.length}.flac`);
|
|
257
|
-
const r = await run(ffmpeg, ['-y', '-i', fileToSend, '-ss', String(s), '-t', String(chunkSec), '-ar', '16000', '-ac', '1', '-c:a', 'flac', cp], { timeout: 300000 });
|
|
258
|
-
void r;
|
|
259
|
-
if (fs.existsSync(cp) && fs.statSync(cp).size > 0) chunks.push(cp);
|
|
260
|
-
}
|
|
261
|
-
if (chunks.length > 0) chunkFiles = chunks;
|
|
262
|
-
}
|
|
263
|
-
} catch {}
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
const results = [];
|
|
267
|
-
for (let ci = 0; ci < chunkFiles.length; ci++) {
|
|
268
|
-
const cf = chunkFiles[ci];
|
|
269
|
-
if (fs.existsSync(cf) && fs.statSync(cf).size > 24 * 1024 * 1024) {
|
|
270
|
-
onLog(`Chunk ${ci + 1} too large, skipping`);
|
|
271
|
-
continue;
|
|
272
|
-
}
|
|
273
|
-
if (chunkFiles.length > 1) onLog(`Chunk ${ci + 1}/${chunkFiles.length}...`);
|
|
274
|
-
const keyList = keys.split(',').map((k) => k.trim()).filter(Boolean);
|
|
275
|
-
for (let attempt = 0; attempt < keyList.length + 2; attempt++) {
|
|
276
|
-
const tryKey = keyList[attempt % keyList.length];
|
|
277
|
-
try {
|
|
278
|
-
const fd = new FormData();
|
|
279
|
-
fd.append('model', model);
|
|
280
|
-
fd.append('response_format', 'verbose_json');
|
|
281
|
-
fd.append('temperature', '0.0');
|
|
282
|
-
if (language) fd.append('language', language);
|
|
283
|
-
const buf = fs.readFileSync(cf);
|
|
284
|
-
fd.append('file', new Blob([buf], { type: contentType(cf) }), path.basename(cf));
|
|
285
|
-
const resp = await fetch('https://api.groq.com/openai/v1/audio/transcriptions', {
|
|
286
|
-
method: 'POST',
|
|
287
|
-
headers: { Authorization: `Bearer ${tryKey}` },
|
|
288
|
-
body: fd,
|
|
289
|
-
signal: AbortSignal.timeout(600000),
|
|
290
|
-
});
|
|
291
|
-
if (resp.status === 200) {
|
|
292
|
-
const j = await resp.json();
|
|
293
|
-
results.push(j.text || '');
|
|
294
|
-
break;
|
|
295
|
-
}
|
|
296
|
-
if ([429, 500, 502, 503].includes(resp.status)) {
|
|
297
|
-
onLog(`HTTP ${resp.status}, retry ${attempt + 1}/${keyList.length + 2}`);
|
|
298
|
-
await sleep(5000 + attempt * 3000);
|
|
299
|
-
continue;
|
|
300
|
-
}
|
|
301
|
-
const body = await resp.text();
|
|
302
|
-
let msg = body;
|
|
303
|
-
try { msg = JSON.parse(body).error?.message || body; } catch {}
|
|
304
|
-
throw new Error(`[${resp.status}] ${msg}`);
|
|
305
|
-
} catch (e) {
|
|
306
|
-
if (e.message?.startsWith('[')) throw e;
|
|
307
|
-
if (attempt < keyList.length + 1) {
|
|
308
|
-
onLog(`Retry ${attempt + 1}: ${String(e.message || e).slice(0, 60)}`);
|
|
309
|
-
await sleep(3000 + attempt * 2000);
|
|
310
|
-
continue;
|
|
311
|
-
}
|
|
312
|
-
throw e;
|
|
313
|
-
}
|
|
314
|
-
}
|
|
315
|
-
}
|
|
316
|
-
|
|
317
|
-
if (tempFile) { try { fs.unlinkSync(tempFile); } catch {} }
|
|
318
|
-
for (const c of chunkFiles) {
|
|
319
|
-
if (c !== audioPath && c !== tempFile) { try { fs.unlinkSync(c); } catch {} }
|
|
320
|
-
}
|
|
321
|
-
return results.join('\n\n---\n\n');
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
function contentType(fpath) {
|
|
325
|
-
const ext = path.extname(fpath).toLowerCase();
|
|
326
|
-
return {
|
|
327
|
-
'.flac': 'audio/flac', '.mp3': 'audio/mpeg', '.wav': 'audio/wav',
|
|
328
|
-
'.ogg': 'audio/ogg', '.m4a': 'audio/mp4', '.mp4': 'audio/mp4',
|
|
329
|
-
}[ext] || 'audio/flac';
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
// ---------- Pipeline ----------
|
|
333
|
-
export class PodcastPipeline {
|
|
334
|
-
constructor({ config, onLog, isCancelled = () => false }) {
|
|
335
|
-
this.config = config;
|
|
336
|
-
this.onLog = onLog;
|
|
337
|
-
this.isCancelled = isCancelled;
|
|
338
|
-
this.svc = new PodcastService(config.basePath);
|
|
339
|
-
this.cachePath = path.join(config.basePath, CACHE_FILE);
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
_loadCache() {
|
|
343
|
-
try {
|
|
344
|
-
if (!fs.existsSync(this.cachePath)) return {};
|
|
345
|
-
return JSON.parse(fs.readFileSync(this.cachePath, 'utf-8'));
|
|
346
|
-
} catch { return {}; }
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
_saveCache(cache) {
|
|
350
|
-
try { fs.writeFileSync(this.cachePath, JSON.stringify(cache), 'utf-8'); } catch {}
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
_findSrt(podDir, name) {
|
|
354
|
-
const plain = path.join(podDir, `${name}.srt`);
|
|
355
|
-
if (fs.existsSync(plain)) return plain;
|
|
356
|
-
if (!fs.existsSync(podDir)) return null;
|
|
357
|
-
try {
|
|
358
|
-
for (const f of fs.readdirSync(podDir)) {
|
|
359
|
-
if (f.startsWith(`${name}.`) && f.toLowerCase().endsWith('.srt')) {
|
|
360
|
-
return path.join(podDir, f);
|
|
361
|
-
}
|
|
362
|
-
}
|
|
363
|
-
} catch {}
|
|
364
|
-
return null;
|
|
365
|
-
}
|
|
366
|
-
|
|
367
|
-
_srtToTxt(srtPath, txtPath) {
|
|
368
|
-
try {
|
|
369
|
-
if (!fs.existsSync(srtPath)) return;
|
|
370
|
-
const content = fs.readFileSync(srtPath, 'utf-8');
|
|
371
|
-
const textLines = [];
|
|
372
|
-
let last = '';
|
|
373
|
-
for (const line of content.split('\n')) {
|
|
374
|
-
const t = line.trim();
|
|
375
|
-
if (!t) continue;
|
|
376
|
-
if (/^\d+$/.test(t)) continue;
|
|
377
|
-
if (t.includes('-->')) continue;
|
|
378
|
-
const clean = t.replace(/<[^>]+>/g, '');
|
|
379
|
-
if (!clean || clean === last) continue;
|
|
380
|
-
textLines.push(clean);
|
|
381
|
-
last = clean;
|
|
382
|
-
}
|
|
383
|
-
fs.writeFileSync(txtPath, textLines.join('\n'), 'utf-8');
|
|
384
|
-
} catch {}
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
async run() {
|
|
388
|
-
const config = this.config;
|
|
389
|
-
const subs = config.podcastSubscriptions || {};
|
|
390
|
-
if (Object.keys(subs).length === 0) {
|
|
391
|
-
this.onLog('沒有訂閱任何 Podcast');
|
|
392
|
-
return;
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
const stamp = new Date().toISOString().substring(0, 19).replace('T', ' ');
|
|
396
|
-
this.onLog(`${stamp} Playlist Administrator (Node CLI)`);
|
|
397
|
-
const hasGroq = !!(config.groqApiKey || '').trim();
|
|
398
|
-
|
|
399
|
-
this.onLog(`Podcast 訂閱數: ${Object.keys(subs).length}`);
|
|
400
|
-
const entries = Object.entries(subs);
|
|
401
|
-
|
|
402
|
-
for (let si = 0; si < entries.length; si++) {
|
|
403
|
-
if (this.isCancelled()) break;
|
|
404
|
-
const [podcastName, rssUrl] = entries[si];
|
|
405
|
-
const ts = new Date().toISOString().substring(0, 19).replace('T', ' ');
|
|
406
|
-
this.onLog(`\n${ts} --- [${si + 1}/${entries.length}] ${podcastName} ---`);
|
|
407
|
-
try {
|
|
408
|
-
await this._processPodcast(podcastName, rssUrl, hasGroq);
|
|
409
|
-
} catch (e) {
|
|
410
|
-
this.onLog(` ❌ ${podcastName} 失敗: ${e.message || e}`);
|
|
411
|
-
}
|
|
412
|
-
}
|
|
413
|
-
this.onLog(this.isCancelled() ? 'Podcast Pipeline 已取消' : 'Podcast Pipeline 完成');
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
async _processPodcast(podcastName, rssUrl, hasGroq) {
|
|
417
|
-
const cache = this._loadCache();
|
|
418
|
-
const podDir = this.svc.podcastDir(podcastName);
|
|
419
|
-
const ext = 'mp3';
|
|
420
|
-
|
|
421
|
-
this.onLog(' 讀取 RSS Feed...');
|
|
422
|
-
const result = await this.svc.fetchEpisodes(rssUrl);
|
|
423
|
-
const episodes = result.episodes;
|
|
424
|
-
this.onLog(` Feed: ${result.title} (${episodes.length} 集)`);
|
|
425
|
-
|
|
426
|
-
const tasks = [];
|
|
427
|
-
let alreadyHave = 0;
|
|
428
|
-
for (let i = 0; i < episodes.length; i++) {
|
|
429
|
-
if (this.isCancelled()) break;
|
|
430
|
-
const ep = episodes[i];
|
|
431
|
-
const key = `${podcastName}|${ep.title}`;
|
|
432
|
-
const name = normalizeFileName(ep.title);
|
|
433
|
-
const srtPath = this._findSrt(podDir, name);
|
|
434
|
-
const txtPath = path.join(podDir, `${name}.txt`);
|
|
435
|
-
const hasSrt = srtPath != null;
|
|
436
|
-
const hasTxt = fs.existsSync(txtPath);
|
|
437
|
-
if (hasSrt || hasTxt) {
|
|
438
|
-
if (hasSrt && !hasTxt) this._srtToTxt(srtPath, txtPath);
|
|
439
|
-
cache[key] = {
|
|
440
|
-
srt: hasSrt,
|
|
441
|
-
txt: hasTxt || fs.existsSync(txtPath),
|
|
442
|
-
yt_status: hasSrt ? 'found' : (cache[key]?.yt_status || ''),
|
|
443
|
-
status: 'ok',
|
|
444
|
-
};
|
|
445
|
-
alreadyHave++;
|
|
446
|
-
continue;
|
|
447
|
-
}
|
|
448
|
-
if (!cache[key] || (cache[key].srt !== true && cache[key].txt !== true)) {
|
|
449
|
-
tasks.push({ index: i, episode: ep, key });
|
|
450
|
-
}
|
|
451
|
-
}
|
|
452
|
-
this._saveCache(cache);
|
|
453
|
-
|
|
454
|
-
if (tasks.length === 0) {
|
|
455
|
-
this.onLog(` 無新集數 (${alreadyHave} 集已處理過)`);
|
|
456
|
-
return;
|
|
457
|
-
}
|
|
458
|
-
this.onLog(` 需處理: ${tasks.length} 集 (×4 並行)`);
|
|
459
|
-
const total = tasks.length;
|
|
460
|
-
const groqQueue = [];
|
|
461
|
-
let groqActive = 0;
|
|
462
|
-
const groqLimit = Math.max(1, Math.min(8, this.config.groqConcurrency || 3));
|
|
463
|
-
|
|
464
|
-
const tryGroq = () => {
|
|
465
|
-
while (groqActive < groqLimit && groqQueue.length > 0 && !this.isCancelled()) {
|
|
466
|
-
const t = groqQueue.shift();
|
|
467
|
-
groqActive++;
|
|
468
|
-
this._runGroq(t, podcastName, podDir, ext, cache).finally(() => {
|
|
469
|
-
groqActive--;
|
|
470
|
-
this._saveCache(cache);
|
|
471
|
-
tryGroq();
|
|
472
|
-
});
|
|
473
|
-
}
|
|
474
|
-
};
|
|
475
|
-
|
|
476
|
-
for (let i = 0; i < total; i += 4) {
|
|
477
|
-
if (this.isCancelled()) break;
|
|
478
|
-
const batch = tasks.slice(i, i + 4);
|
|
479
|
-
await Promise.all(batch.map(async (t) => {
|
|
480
|
-
const needGroq = await this._processOne(t, podcastName, rssUrl, podDir, ext, cache);
|
|
481
|
-
this._saveCache(cache);
|
|
482
|
-
if (needGroq && hasGroq) {
|
|
483
|
-
groqQueue.push(t);
|
|
484
|
-
tryGroq();
|
|
485
|
-
}
|
|
486
|
-
}));
|
|
487
|
-
const done = Math.min(i + batch.length, total);
|
|
488
|
-
this.onLog(` 進度: ${done}/${total}`);
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
while (groqActive > 0 && !this.isCancelled()) {
|
|
492
|
-
await sleep(1000);
|
|
493
|
-
}
|
|
494
|
-
|
|
495
|
-
const values = Object.values(cache);
|
|
496
|
-
const srtCount = values.filter((v) => v.srt === true).length;
|
|
497
|
-
const txtCount = values.filter((v) => v.txt === true).length;
|
|
498
|
-
const errCount = values.filter((v) => v.status === 'error').length;
|
|
499
|
-
this.onLog(` ${podcastName} 完成: 總處理 ${values.length} 集 (SRT ${srtCount}, 逐字稿 ${txtCount}, 錯誤 ${errCount})`);
|
|
500
|
-
}
|
|
501
|
-
|
|
502
|
-
async _processOne(t, podcastName, rssUrl, podDir, ext, cache) {
|
|
503
|
-
const name = normalizeFileName(t.episode.title);
|
|
504
|
-
const audioPath = path.join(podDir, `${name}.${ext}`);
|
|
505
|
-
const txtPath = path.join(podDir, `${name}.txt`);
|
|
506
|
-
|
|
507
|
-
if (!fs.existsSync(audioPath)) {
|
|
508
|
-
this.onLog(` [${t.index}] ⬇️ ${name}`);
|
|
509
|
-
try {
|
|
510
|
-
await this.svc.downloadEpisode(rssUrl, t.index, podcastName);
|
|
511
|
-
} catch {
|
|
512
|
-
this.onLog(' ❌ 下載失敗');
|
|
513
|
-
cache[t.key] = { srt: false, txt: false, yt_status: '', status: 'error' };
|
|
514
|
-
return false;
|
|
515
|
-
}
|
|
516
|
-
}
|
|
517
|
-
|
|
518
|
-
const srtPath = this._findSrt(podDir, name);
|
|
519
|
-
if (srtPath != null || fs.existsSync(txtPath)) {
|
|
520
|
-
if (srtPath != null && !fs.existsSync(txtPath)) {
|
|
521
|
-
this._srtToTxt(srtPath, txtPath);
|
|
522
|
-
cache[t.key] = { srt: true, txt: fs.existsSync(txtPath), yt_status: 'found', status: 'ok' };
|
|
523
|
-
}
|
|
524
|
-
return false;
|
|
525
|
-
}
|
|
526
|
-
|
|
527
|
-
const prevYt = cache[t.key]?.yt_status;
|
|
528
|
-
if (prevYt === 'not_found') {
|
|
529
|
-
this.onLog(` ⏭️ ${name} (YT 上次已搜過)`);
|
|
530
|
-
cache[t.key] = { srt: false, txt: false, yt_status: 'not_found', status: 'no_sub' };
|
|
531
|
-
return true;
|
|
532
|
-
}
|
|
533
|
-
|
|
534
|
-
this.onLog(` 🔍 ${name}`);
|
|
535
|
-
await sleep((t.index % 4) * 300);
|
|
536
|
-
const subResult = await this.svc.downloadSubtitles(t.episode.title, podcastName, (msg) => this.onLog(` ${msg}`));
|
|
537
|
-
const srtAfter = this._findSrt(podDir, name);
|
|
538
|
-
if (subResult === 'found' && srtAfter != null) {
|
|
539
|
-
this._srtToTxt(srtAfter, txtPath);
|
|
540
|
-
cache[t.key] = { srt: true, txt: true, yt_status: 'found', status: 'ok' };
|
|
541
|
-
return false;
|
|
542
|
-
}
|
|
543
|
-
if (subResult === 'not_found') {
|
|
544
|
-
cache[t.key] = { srt: false, txt: false, yt_status: 'not_found', status: 'no_sub' };
|
|
545
|
-
} else {
|
|
546
|
-
cache[t.key] = { srt: false, txt: false, yt_status: '', status: 'no_sub' };
|
|
547
|
-
}
|
|
548
|
-
return true;
|
|
549
|
-
}
|
|
550
|
-
|
|
551
|
-
async _runGroq(t, podcastName, podDir, ext, cache) {
|
|
552
|
-
if (this.isCancelled()) return;
|
|
553
|
-
const name = normalizeFileName(t.episode.title);
|
|
554
|
-
const audioPath = path.join(podDir, `${name}.${ext}`);
|
|
555
|
-
const txtPath = path.join(podDir, `${name}.txt`);
|
|
556
|
-
if (!fs.existsSync(audioPath)) return;
|
|
557
|
-
if (this._findSrt(podDir, name) != null || fs.existsSync(txtPath)) return;
|
|
558
|
-
this.onLog(` 🎤 ${name}`);
|
|
559
|
-
try {
|
|
560
|
-
const text = await transcribeAudio(
|
|
561
|
-
audioPath, this.config.groqApiKey, 'whisper-large-v3-turbo', 'zh',
|
|
562
|
-
(msg) => this.onLog(` ${msg}`),
|
|
563
|
-
);
|
|
564
|
-
fs.writeFileSync(txtPath, text, 'utf-8');
|
|
565
|
-
cache[t.key] = { srt: false, txt: true, yt_status: 'not_found', status: 'ok' };
|
|
566
|
-
this.onLog(` ✅ (${text.length} 字)`);
|
|
567
|
-
} catch (e) {
|
|
568
|
-
this.onLog(` ❌ ${e.message || e}`);
|
|
569
|
-
cache[t.key] = { srt: false, txt: false, yt_status: '', status: 'error' };
|
|
570
|
-
}
|
|
571
|
-
}
|
|
572
|
-
}
|