tphim 2.3.2 → 2.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tphim",
-  "version": "2.3.2",
+  "version": "2.5.0",
   "description": "TPHIM - Ultimate Video Pipeline: Download, Transcode HLS, AI Subtitles (with skip option), Resume Upload, and Cloud Upload.",
   "main": "index.js",
   "type": "module",

package/pipeline.mjs

@@ -1,15 +1,15 @@
 #!/usr/bin/env node
-// pipeline.mjs v2.2.2
+// pipeline.mjs v2.4.0
 // Usage: node pipeline.mjs "URL_hoac_duong_dan_mp4" "slug" "Tên Phim" [vi|en|both|skip]
 // 
-// v2.2.2 FEATURES:
-// - Track and Display Total Execution Time (xxhxxmxxs)
-// - Dual Preview Sprites (High-Res + Low-Res) with FFmpeg Complex Filter
-// - Optimized CPU usage with single-pass processing
-// - Low-Res sprites optimized for mobile/preview (80x45 tiles, lighter file)
+// v2.4.0 FEATURES:
+// - Batch HLS Transcoding (2 qualities at a time)
+// - Incremental Uploads (Upload immediately after each batch)
+// - Fast-Track Storyboards (Capture & upload after first batch)
+// - Native HLS Upload Module integration
 
 import { execSync, spawn } from 'child_process';
-import { existsSync, mkdirSync, writeFileSync, rmSync, unlinkSync, readFileSync } from 'fs';
+import { existsSync, mkdirSync, writeFileSync, rmSync, unlinkSync, readFileSync, appendFileSync } from 'fs';
 import os from 'os';
 import path, { dirname } from 'path';
 import { fileURLToPath } from 'url';
@@ -257,15 +257,26 @@ function formatDuration(ms) {
     return parts.join('');
 }
 
+function updateMasterPlaylist(masterPath, processedQualities) {
+    let content = "#EXTM3U\n#EXT-X-VERSION:3\n";
+    // Sort high to low for better player compatibility
+    [...processedQualities].sort((a, b) => b.h - a.h).forEach(q => {
+        const bw = parseInt(q.bv) * 1000 + (parseInt(q.ba) || 128) * 1000;
+        content += `#EXT-X-STREAM-INF:BANDWIDTH=${bw},RESOLUTION=${q.w}x${q.h}\n`;
+        content += `${q.label}/index.m3u8\n`;
+    });
+    writeFileSync(masterPath, content);
+}
+
 export async function run() {
     // Nếu chạy trực tiếp từ CLI
-    let [, , input, slug, title = slug, langArg = 'vi'] = process.argv;
+    let [, , input, slug, title = slug, langArg = 'vi', skip3gp = 'false'] = process.argv;
 
     // Hỗ trợ truyền args vào function trong tương lai nếu cần
-    return await executePipeline({ input, slug, title, langArg });
+    return await executePipeline({ input, slug, title, langArg, skip3gp: skip3gp === 'true' });
 }
 
-export async function executePipeline({ input, slug, title, langArg = 'vi' }) {
+export async function executePipeline({ input, slug, title, langArg = 'vi', skip3gp = false }) {
     // --- ENV SHIELD ---
     const required = ['TEBI_ENDPOINT', 'TEBI_ACCESS_KEY_ID', 'TEBI_SECRET_ACCESS_KEY', 'TEBI_BUCKET'];
     const missing = required.filter(k => !process.env[k]);
@@ -274,11 +285,7 @@ export async function executePipeline({ input, slug, title, langArg = 'vi' }) {
     }
 
     if (!input || !slug) {
-        console.log('\nCách dùng:');
-        console.log('  node pipeline.mjs "URL_video"         "slug" "Tên Phim" vi');
-        console.log('  node pipeline.mjs "D:\\film.mp4"       "slug" "Tên Phim" vi');
-        console.log('  node pipeline.mjs "URL_video"         "slug" "Tên Phim" both');
-        console.log('  node pipeline.mjs "URL_video"         "slug" "Tên Phim" skip');
+        console.log('  node pipeline.mjs "URL_video"         "slug" "Tên Phim" vi    true (skip 3GP)');
         return;
     }
     console.log('\n╔══════════════════════════════════════════════════════════════════╗');
@@ -408,10 +415,6 @@ export async function executePipeline({ input, slug, title, langArg = 'vi' }) {
             throw new Error(
                 `❌ Download thất bại: ${input}\n` +
                 `   Đã thử: direct + ${Math.min(5, PROXY_LIST.length)} proxies\n` +
-                `   Nguyên nhân phổ biến:\n` +
-                `   • Link đã hết hạn hoặc bị xóa (HTTP 404)\n` +
-                `   • Server CDN chặn tất cả IP\n` +
-                `   • Link cần cookie/token đặc biệt\n` +
                 `   → Hãy thử mở link trong trình duyệt để kiểm tra.`
             );
         }
@@ -419,58 +422,24 @@ export async function executePipeline({ input, slug, title, langArg = 'vi' }) {
         console.log(`\n⏭  [1/5] File đã có sẵn: ${mp4Path}`);
     }
 
-    // --- STEP 2: PREVIEW SPRITE & STORYBOARD (Fast-Track) ---
-    const spriteLowPath = path.join('hls', slug, 'preview_low.jpg');
-    const storyboardCheckPath = path.join('hls', slug, 'L1_M0.jpg');
-    const outDir = path.join('hls', slug);
-
-    // Detect VPS environment and optimize
+    // --- BATCH PROCESSING LOGIC ---
     const vpsOpt = detectVPSOptimization();
-    console.log(`   💻 System: ${IS_LINUX ? 'Linux' : 'Windows'} | CPU Cores: ${vpsOpt.cpuCores}`);
-    if (vpsOpt.isVPS && vpsOpt.isUbuntu) {
-        console.log(`   🚀 Ubuntu VPS Mode: Enabled (preset: ${vpsOpt.preset}, tune: ${vpsOpt.tune})`);
-    }
-
-    console.log('\n📸 [2/5] Capturing Storyboards & Sprites (Turbo Mode)...');
-    
-    // 1. New Storyboard format (L1, L2, L3)
-    if (!existsSync(storyboardCheckPath)) {
-        try {
-            await generateStoryboards(mp4Path, outDir);
-        } catch (e) {
-            console.log('   ⚠ Storyboard generation failed:', e.message);
-        }
-    } else {
-        console.log(`   ⏭  Storyboard levels already exist`);
-    }
+    const masterName = `txa-${slug}.m3u8`;
+    const masterPath = path.join('hls', slug, masterName);
+    const logPath = path.join('hls', slug, 'pipeline.log');
+    const outDir = path.join('hls', slug);
 
-    // 2. Legacy Dual Sprite format
-    if (!existsSync(spritePath) || !existsSync(spriteLowPath)) {
-        try {
-            await generateDualPreviewSprites(mp4Path, outDir);
-        } catch (e) {
-            console.log('   ⚠ Dual sprite generation failed, falling back to single sprite...');
-            const spriteCmd = `"${FFMPEG_BIN}" -i "${mp4Path}" -vf "fps=1/10,scale=160:90,tile=10x10" -q:v 3 "${spritePath}"`;
-            try {
-                execSync(spriteCmd, { stdio: 'inherit', ...SHELL_OPTS });
-                console.log('   ✅ Fallback Preview SpriteSheet ready.');
-            } catch (fallbackError) {
-                console.log('   ⚠ All sprite generation methods failed.');
-            }
-        }
-    } else {
-        console.log(`   ⏭  Dual Preview Sprites already exist`);
-    }
+    const log = (msg) => {
+        console.log(msg);
+        appendFileSync(logPath, msg + '\n');
+    };
 
-    // ─── BƯỚC 3: Transcode HLS ────────────────────────────────────
-    const masterPath = path.join('hls', slug, masterName);
-    const nokia3gpPath = path.join('hls', slug, `${slug}.3gp`);
+    writeFileSync(logPath, `🎬 Log for ${title} - ${new Date().toLocaleString()}\n\n`);
 
-    // Danh sách toàn bộ chất lượng hỗ trợ
     const ALL_QUALITIES = [
         { label: '1080p', w: 1920, h: 1080, bv: '4500k', ba: '192k' },
         { label: '720p', w: 1280, h: 720, bv: '1500k', ba: '128k' },
-        { label: '480p', w: 854, h: 480, bv: '1000k', ba: '128k' }, // Tăng bitrate cho 480p chút
+        { label: '480p', w: 854, h: 480, bv: '1000k', ba: '128k' },
         { label: '360p', w: 640, h: 360, bv: '600k', ba: '96k' },
         { label: '240p', w: 426, h: 240, bv: '400k', ba: '64k' },
         { label: '144p', w: 256, h: 144, bv: '200k', ba: '48k' }
@@ -482,33 +451,35 @@ export async function executePipeline({ input, slug, title, langArg = 'vi' }) {
         .slice(0, count)
         .sort((a, b) => b.h - a.h);
 
-    if (!existsSync(masterPath)) {
-        console.log(`\n📦 [3/5] Transcoding HLS (${selectedQualities.length} qualities randomized)...`);
-        console.log(`   ↳ Selected: ${selectedQualities.map(q => q.label).join(', ')}`);
+    const batches = [];
+    for (let i = 0; i < selectedQualities.length; i += 2) {
+        batches.push(selectedQualities.slice(i, i + 2));
+    }
 
-        const outDir = path.join('hls', slug);
+    let processedQualities = [];
+    const timestamp = Date.now();
+    const encodedPrefix = Buffer.from(`txa_${timestamp}`).toString('base64').replace(/[^a-zA-Z0-9]/g, '').substring(0, 8);
 
-        // [TURBO HLS] Build dynamic filter chain for sequential scaling
-        // Scaling from previously scaled version (e.g. 1080 -> 720) is much faster
-        let filterParts = [`[0:v]split=${selectedQualities.length}`];
-        let splitLabels = selectedQualities.map((_, i) => `[v_raw${i}]`);
+    for (let bIndex = 0; bIndex < batches.length; bIndex++) {
+        const batch = batches[bIndex];
+        log(`\n📦 [Batch ${bIndex + 1}/${batches.length}] Processing: ${batch.map(q => q.label).join(', ')}`);
+
+        let filterParts = [`[0:v]split=${batch.length}`];
+        let splitLabels = batch.map((_, i) => `[v_raw${i}]`);
         filterParts[0] += splitLabels.join('');
 
         let outputs = '';
         let maps = '';
         let varStreamMapArr = [];
 
-        selectedQualities.forEach((q, i) => {
+        batch.forEach((q, i) => {
             filterParts.push(`[v_raw${i}]scale=${q.w}:${q.h},format=yuv420p[v${i}]`);
             maps += `-map "[v${i}]" -map 0:a `;
-            outputs += `-c:v:${i} libx264 -preset ultrafast -tune fastdecode -b:v:${i} ${q.bv} -c:a:${i} aac -b:a:${i} ${q.ba} -movflags +faststart `;
+            outputs += `-c:v:${i} libx264 -preset ${vpsOpt.preset} -tune ${vpsOpt.tune} -b:v:${i} ${q.bv} -c:a:${i} aac -b:a:${i} ${q.ba} -movflags +faststart `;
             varStreamMapArr.push(`v:${i},a:${i}`);
         });
 
-        const timestamp = Date.now();
-        const encodedPrefix = Buffer.from(`txa_${timestamp}`).toString('base64').replace(/[^a-zA-Z0-9]/g, '').substring(0, 8);
-        const threadCount = vpsOpt.isVPS ? Math.min(vpsOpt.cpuCores * 2, 16) : 0; // Oversubscribe for maximum saturation
-
+        const threadCount = vpsOpt.isVPS ? Math.min(vpsOpt.cpuCores * 2, 16) : 0;
         const ffmpegCmd = [
             `"${FFMPEG_BIN}"`, '-i', `"${mp4Path}"`,
             `-threads ${threadCount}`,
@@ -516,90 +487,78 @@ export async function executePipeline({ input, slug, title, langArg = 'vi' }) {
             maps,
             outputs,
             '-var_stream_map', `"${varStreamMapArr.join(' ')}"`,
-            '-master_pl_name', `"${masterName}"`,
-            '-f hls -hls_time 6 -hls_list_size 0 -hls_segment_type mpegts', // 6s segments are faster to write than 4s
+            '-f hls -hls_time 6 -hls_list_size 0 -hls_segment_type mpegts',
             '-hls_segment_filename', `"${outDir}/%v/${encodedPrefix}_%04d.ts"`,
             `"${outDir}/%v/index.m3u8"`
         ].join(' ');
 
+        log(`   🛠 Running FFmpeg Transcoding...`);
         execSync(ffmpegCmd, { stdio: 'inherit', ...SHELL_OPTS });
-        console.log('   HLS Master generated.');
 
-        // --- BƯỚC 2.1: XUẤT 3GP CHO MÁY CỔ (NOKIA S40/S60) ---
-        console.log('   [2.1/5] Exporting 3GP legacy format...');
-        console.log('   📱 [2.1/5] Exporting 3GP legacy format...');
-        const nokiaCmd = `"${FFMPEG_BIN}" -i "${mp4Path}" -s 320x240 -vcodec mpeg4 -acodec aac -ar 16000 -ac 1 -b:v 250k -b:a 32k -preset ${vpsOpt.preset} -f 3gp "${nokia3gpPath}"`;
-        try {
-            execSync(nokiaCmd, { stdio: 'ignore', ...SHELL_OPTS });
-            console.log('   ✅ Legacy 3GP ready.');
-        } catch (e) {
-            console.log('   ⚠ 3GP generation failed.');
+        // Rename folders from index (0, 1) to labels
+        batch.forEach((q, i) => {
+            const oldFolder = path.join(outDir, i.toString());
+            const newFolder = path.join(outDir, q.label);
+            if (existsSync(oldFolder)) {
+                if (existsSync(newFolder)) rmSync(newFolder, { recursive: true, force: true });
+                execSync(IS_WIN ? `move "${oldFolder}" "${newFolder}"` : `mv "${oldFolder}" "${newFolder}"`, SHELL_OPTS);
+            }
+        });
+
+        processedQualities.push(...batch);
+        updateMasterPlaylist(masterPath, processedQualities);
+
+        // Upload Batch
+        log(`   🚀 Uploading Batch ${bIndex + 1} to Tebi...`);
+        await uploadHLSFolder(slug);
+
+        // After Batch 1, capture and upload storyboards
+        if (bIndex === 0) {
+            log(`\n📸 [Fast-Track] First Batch Uploaded. Generating Storyboards...`);
+            await generateStoryboards(mp4Path, outDir);
+            await generateDualPreviewSprites(mp4Path, outDir);
+            log(`   🚀 Uploading Preview Assets...`);
+            await uploadHLSFolder(slug);
         }
-    } else {
-        console.log(`\n⏭  [2/5] HLS đã có sẵn`);
     }
 
+    // --- STEP 3: LEGACY & DOCS ---
+    const nokia3gpPath = path.join('hls', slug, `${slug}.3gp`);
+    if (!skip3gp && !existsSync(nokia3gpPath)) {
+        log('\n📱 Exporting legacy 3GP format...');
+        const nokiaCmd = `"${FFMPEG_BIN}" -i "${mp4Path}" -s 320x240 -vcodec mpeg4 -acodec aac -ar 16000 -ac 1 -b:v 250k -b:a 32k -preset ${vpsOpt.preset} -f 3gp "${nokia3gpPath}"`;
+        try { execSync(nokiaCmd, { stdio: 'ignore', ...SHELL_OPTS }); } catch (e) { log('   ⚠ 3GP failed.'); }
+    } else if (skip3gp) {
+        log('\n⏭  Skipping legacy 3GP export (skip3gp=true)');
+    }
 
-    // ─── BƯỚC 3: Tạo README ───────────────────────────────────────
-    console.log('\n📄 [3/5] Generating README.txt...');
+    log('\n📄 Generating README and Subtitles...');
     generateReadme(slug, mp4Path, { title });
 
-    // ─── BƯỚC 4: Tạo Subtitle (Python faster-whisper) ─────────────
     if (langArg !== 'skip') {
-        console.log('\n📝 [4/5] Generating subtitles (faster-whisper offline)...');
         const langs = langArg === 'both' ? ['vi', 'en'] : [langArg];
-
         for (const lang of langs) {
             const vttPath = path.join('hls', slug, `${lang}.vtt`);
             if (!existsSync(vttPath)) {
-                // Detection logic for python command (python vs python3)
                 let pythonCmd = 'python';
+                try { execSync('python --version', { stdio: 'ignore' }); } catch (e) { pythonCmd = 'python3'; }
                 try {
-                    execSync('python --version', { stdio: 'ignore' });
-                } catch (e) {
-                    pythonCmd = 'python3';
-                }
-
-                execSync(
-                    `${pythonCmd} ${path.join(__dirname, 'scripts', 'gen_subtitle.py')} "${mp4Path}" "${slug}" ${lang} "${FFMPEG_BIN}"`,
-                    { stdio: 'inherit', cwd: process.cwd() }
-                );
-            } else {
-                console.log(`   ⏭  Subtitle [${lang}] đã có sẵn`);
+                    execSync(`${pythonCmd} ${path.join(__dirname, 'scripts', 'gen_subtitle.py')} "${mp4Path}" "${slug}" ${lang} "${FFMPEG_BIN}"`, { stdio: 'inherit', cwd: process.cwd() });
+                } catch (e) { log(`   ⚠ Subtitle [${lang}] failed.`); }
             }
         }
-    } else {
-        console.log('\n⏭  [4/5] Skipping subtitle generation (selected "skip" option)');
     }
 
-    // ─── BƯỚC 5: Upload Tebi.io ───────────────────────────────────
-    console.log('\n🚀 [5/5] Deploying to Quantum Cloud (Tebi.io)...');
-
-    // Optimize upload with parallel processing
-    const uploadCmd = `node ${path.join(__dirname, 'scripts', 'upload-tebi.mjs')} "${slug}"`;
+    // Final Sync Upload
+    log('\n🚀 Final sync upload...');
+    await uploadHLSFolder(slug);
 
-    // Run upload with higher process priority and optimized settings
-    try {
-        execSync(uploadCmd, {
-            stdio: 'inherit',
-            cwd: process.cwd(),
-            maxBuffer: 1024 * 1024, // Increase buffer size
-            env: {
-                ...process.env,
-                NODE_OPTIONS: '--max-old-space-size=4096', // Optimize Node.js memory
-                UV_THREADPOOL_SIZE: '16' // Increase libuv thread pool
-            }
-        });
-    } catch (uploadError) {
-        console.error('   ⚠ Upload failed:', uploadError.message);
-        throw uploadError;
-    }
-
-    // --- FINAL STEP: GENERATE MASTER README.md ---
-    const finalReadmePath = path.join('hls', slug, 'README.md');
     const bucketName = process.env.TEBI_BUCKET || "txa-vod";
     const cloudUrl = `https://s3.tebi.io/${bucketName}/hls/${slug}/${masterName}`;
+    const finalReadmePath = path.join('hls', slug, 'README.md');
     const nokiaUrl = `https://s3.tebi.io/${bucketName}/hls/${slug}/${slug}.3gp`;
+    const spritePath = path.join('hls', slug, 'preview.jpg');
+    const spriteLowPath = path.join('hls', slug, 'preview_low.jpg');
 
     const report = `
 # 🎬 TXA PIPELINE REPORT: ${title}
@@ -609,7 +568,7 @@ Generated on: ${new Date().toLocaleString()}
 - [x] **Total Execution Time:** ${formatDuration(Date.now() - startTime)}
 - [x] **Master Playlist:** \`${masterName}\` (AUTO Quality)
 - [x] **Video Qualities:** ${selectedQualities.map(q => q.label).join(', ')} (Randomized)
-- [${existsSync(nokia3gpPath) ? 'x' : ' '}] **Legacy Format:** \`${slug}.3gp\` (Nokia S40/S60)
+- [${skip3gp ? 'skip' : (existsSync(nokia3gpPath) ? 'x' : ' ')}] **Legacy Format:** \`${slug}.3gp\` (Nokia S40/S60)
 - [${existsSync(spritePath) ? 'x' : ' '}] **High-Res Preview SpriteSheet:** \`preview.jpg\` (160x90 tiles, 10x10 grid)
 - [${existsSync(spriteLowPath) ? 'x' : ' '}] **Low-Res Preview SpriteSheet:** \`preview_low.jpg\` (80x45 tiles, 10x10 grid)
 - [${existsSync(path.join('hls', slug, 'vi.vtt')) ? 'x' : ' '}] **Vietnamese Subtitles:** AI Generated
@@ -619,7 +578,7 @@ Generated on: ${new Date().toLocaleString()}
 > **[WATCH NOW (HLS)](${cloudUrl})**
 
 ## 📱 Legacy Mobile Link (Nokia/Java)
-> **[DOWNLOAD 3GP](${nokiaUrl})**
+> ${skip3gp ? '*Disabled by user*' : `**[DOWNLOAD 3GP](${nokiaUrl})**`}
 
 ## 🖼️ Preview Sprites Information
 - **High-Res Version:** \`preview.jpg\` - Full quality for desktop/fast connections
@@ -654,7 +613,12 @@ Generated on: ${new Date().toLocaleString()}
 }
 
 // Tự khởi chạy nếu được gọi trực tiếp qua CLI
-if (import.meta.url.endsWith(process.argv[1].replace(/\\/g, '/')) || process.argv[1].endsWith('pipeline.mjs')) {
+const isCLI = process.argv[1] && (
+    import.meta.url.endsWith(process.argv[1].replace(/\\/g, '/')) || 
+    process.argv[1].endsWith('pipeline.mjs')
+);
+
+if (isCLI) {
     run().catch(err => {
         console.error('\n❌ LỖI:', err.message);
         process.exit(1);