roxify 1.13.5 → 1.13.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.toml +1 -1
- package/README.md +30 -31
- package/dist/cli.js +36 -194
- package/dist/rox-macos-universal +0 -0
- package/dist/roxify_native +0 -0
- package/dist/roxify_native-macos-arm64 +0 -0
- package/dist/roxify_native-macos-x64 +0 -0
- package/dist/roxify_native.exe +0 -0
- package/dist/utils/decoder.d.ts +1 -21
- package/dist/utils/decoder.js +40 -1243
- package/dist/utils/encoder.d.ts +1 -13
- package/dist/utils/encoder.js +35 -565
- package/dist/utils/rust-cli-wrapper.d.ts +2 -2
- package/dist/utils/rust-cli-wrapper.js +8 -2
- package/native/io_advice.rs +1 -1
- package/native/io_ntfs_optimized.rs +99 -0
- package/native/main.rs +293 -33
- package/native/streaming_decode.rs +5 -1
- package/native/streaming_encode.rs +14 -4
- package/package.json +1 -1
- package/roxify_native-aarch64-apple-darwin.node +0 -0
- package/roxify_native-aarch64-pc-windows-msvc.node +0 -0
- package/roxify_native-aarch64-unknown-linux-gnu.node +0 -0
- package/roxify_native-i686-pc-windows-msvc.node +0 -0
- package/roxify_native-i686-unknown-linux-gnu.node +0 -0
- package/roxify_native-x86_64-apple-darwin.node +0 -0
- package/roxify_native-x86_64-pc-windows-msvc.node +0 -0
- package/roxify_native-x86_64-unknown-linux-gnu.node +0 -0
|
@@ -233,7 +233,7 @@ function spawnRustCLI(args, options) {
|
|
|
233
233
|
runSpawn(cliPath);
|
|
234
234
|
});
|
|
235
235
|
}
|
|
236
|
-
export async function encodeWithRustCLI(inputPath, outputPath, compressionLevel = 3, passphrase, encryptType = 'aes', name, onProgress) {
|
|
236
|
+
export async function encodeWithRustCLI(inputPath, outputPath, compressionLevel = 3, passphrase, encryptType = 'aes', name, ramBudgetMb, onProgress) {
|
|
237
237
|
const cliPath = findRustBinary();
|
|
238
238
|
if (!cliPath)
|
|
239
239
|
throw new Error('Rust CLI binary not found');
|
|
@@ -250,10 +250,13 @@ export async function encodeWithRustCLI(inputPath, outputPath, compressionLevel
|
|
|
250
250
|
args.push('--passphrase', passphrase);
|
|
251
251
|
args.push('--encrypt', encryptType);
|
|
252
252
|
}
|
|
253
|
+
if (typeof ramBudgetMb === 'number' && Number.isFinite(ramBudgetMb)) {
|
|
254
|
+
args.push('--ram-budget-mb', String(Math.max(1, Math.floor(ramBudgetMb))));
|
|
255
|
+
}
|
|
253
256
|
args.push(inputPath, outputPath);
|
|
254
257
|
await spawnRustCLI(args, { onProgress });
|
|
255
258
|
}
|
|
256
|
-
export async function decodeWithRustCLI(inputPath, outputPath, passphrase, files, dict, onProgress) {
|
|
259
|
+
export async function decodeWithRustCLI(inputPath, outputPath, passphrase, files, dict, ramBudgetMb, onProgress) {
|
|
257
260
|
const args = ['decompress', inputPath, outputPath];
|
|
258
261
|
if (passphrase)
|
|
259
262
|
args.push('--passphrase', passphrase);
|
|
@@ -261,6 +264,9 @@ export async function decodeWithRustCLI(inputPath, outputPath, passphrase, files
|
|
|
261
264
|
args.push('--files', JSON.stringify(files));
|
|
262
265
|
if (dict)
|
|
263
266
|
args.push('--dict', dict);
|
|
267
|
+
if (typeof ramBudgetMb === 'number' && Number.isFinite(ramBudgetMb)) {
|
|
268
|
+
args.push('--ram-budget-mb', String(Math.max(1, Math.floor(ramBudgetMb))));
|
|
269
|
+
}
|
|
264
270
|
await spawnRustCLI(args, { onProgress });
|
|
265
271
|
}
|
|
266
272
|
export async function listWithRustCLI(inputPath) {
|
package/native/io_advice.rs
CHANGED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
use std::fs::{File, OpenOptions};
|
|
2
|
+
use std::io::{BufWriter, Write};
|
|
3
|
+
use std::path::Path;
|
|
4
|
+
|
|
5
|
+
/// Buffer size optimized for NTFS (larger = fewer syscalls)
|
|
6
|
+
#[cfg(windows)]
|
|
7
|
+
const NTFS_WRITE_BUFFER: usize = 4 * 1024 * 1024; // 4MB for Windows/NTFS
|
|
8
|
+
#[cfg(not(windows))]
|
|
9
|
+
const NTFS_WRITE_BUFFER: usize = 64 * 1024; // 64KB for Unix
|
|
10
|
+
|
|
11
|
+
/// Optimized file writer with large buffer for NTFS
|
|
12
|
+
pub struct OptimizedFileWriter {
|
|
13
|
+
writer: BufWriter<File>,
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
impl OptimizedFileWriter {
|
|
17
|
+
pub fn create(path: &Path) -> std::io::Result<Self> {
|
|
18
|
+
let file = OpenOptions::new()
|
|
19
|
+
.write(true)
|
|
20
|
+
.create(true)
|
|
21
|
+
.truncate(true)
|
|
22
|
+
.open(path)?;
|
|
23
|
+
|
|
24
|
+
Ok(Self {
|
|
25
|
+
writer: BufWriter::with_capacity(NTFS_WRITE_BUFFER, file),
|
|
26
|
+
})
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
pub fn write_all(&mut self, buf: &[u8]) -> std::io::Result<()> {
|
|
30
|
+
self.writer.write_all(buf)
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
pub fn flush(&mut self) -> std::io::Result<()> {
|
|
34
|
+
self.writer.flush()
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/// Write file with optimized buffering for target filesystem
|
|
39
|
+
pub fn write_file_optimized(path: &Path, content: &[u8]) -> std::io::Result<()> {
|
|
40
|
+
let mut writer = OptimizedFileWriter::create(path)?;
|
|
41
|
+
writer.write_all(content)?;
|
|
42
|
+
writer.flush()?;
|
|
43
|
+
Ok(())
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/// Batch write multiple files - keeps files open for better NTFS performance
|
|
47
|
+
pub fn write_files_batch(
|
|
48
|
+
base_dir: &Path,
|
|
49
|
+
files: &[(String, &[u8])],
|
|
50
|
+
) -> Result<Vec<String>, String> {
|
|
51
|
+
let mut written = Vec::with_capacity(files.len());
|
|
52
|
+
|
|
53
|
+
for (rel_path, content) in files {
|
|
54
|
+
let safe_path = sanitize_path(rel_path);
|
|
55
|
+
let dest = base_dir.join(&safe_path);
|
|
56
|
+
|
|
57
|
+
if let Some(parent) = dest.parent() {
|
|
58
|
+
std::fs::create_dir_all(parent)
|
|
59
|
+
.map_err(|e| format!("Cannot create parent dir {:?}: {}", parent, e))?;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
write_file_optimized(&dest, content)
|
|
63
|
+
.map_err(|e| format!("Cannot write {:?}: {}", dest, e))?;
|
|
64
|
+
|
|
65
|
+
written.push(safe_path.to_string_lossy().to_string());
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
Ok(written)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
fn sanitize_path(path: &str) -> std::path::PathBuf {
|
|
72
|
+
let mut safe = std::path::PathBuf::new();
|
|
73
|
+
for comp in std::path::Path::new(path).components() {
|
|
74
|
+
if let std::path::Component::Normal(osstr) = comp {
|
|
75
|
+
safe.push(osstr);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
safe
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/// Pre-allocate file space on NTFS to reduce fragmentation
|
|
82
|
+
#[cfg(windows)]
|
|
83
|
+
pub fn preallocate_file(path: &Path, size: u64) -> std::io::Result<()> {
|
|
84
|
+
use std::os::windows::fs::FileExt;
|
|
85
|
+
|
|
86
|
+
let file = OpenOptions::new()
|
|
87
|
+
.write(true)
|
|
88
|
+
.create(true)
|
|
89
|
+
.open(path)?;
|
|
90
|
+
|
|
91
|
+
// Pre-allocate space
|
|
92
|
+
file.set_len(size)?;
|
|
93
|
+
Ok(())
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
#[cfg(not(windows))]
|
|
97
|
+
pub fn preallocate_file(_path: &Path, _size: u64) -> std::io::Result<()> {
|
|
98
|
+
Ok(())
|
|
99
|
+
}
|
package/native/main.rs
CHANGED
|
@@ -24,6 +24,34 @@ mod progress;
|
|
|
24
24
|
use crate::encoder::ImageFormat;
|
|
25
25
|
use std::path::PathBuf;
|
|
26
26
|
|
|
27
|
+
const MB_U64: u64 = 1024 * 1024;
|
|
28
|
+
const GB_U64: u64 = 1024 * MB_U64;
|
|
29
|
+
|
|
30
|
+
// Adaptive RAM Budget Configuration
|
|
31
|
+
const MIN_RAM_BUDGET_MB: u64 = 512;
|
|
32
|
+
const DEFAULT_RAM_BUDGET_MB: u64 = 2048;
|
|
33
|
+
const RESERVED_RAM_MB: u64 = 1024;
|
|
34
|
+
const MIN_WRITE_BUFFER_BYTES: usize = 16 * 1024 * 1024;
|
|
35
|
+
const MAX_WRITE_BUFFER_BYTES: usize = 256 * 1024 * 1024;
|
|
36
|
+
|
|
37
|
+
// Performance Targets (under 10 seconds)
|
|
38
|
+
const TARGET_ENCODE_TIME_SECS: u64 = 10;
|
|
39
|
+
const TARGET_DECODE_TIME_SECS: u64 = 10;
|
|
40
|
+
const FAST_COMPRESSION_THRESHOLD_MB: u64 = 100;
|
|
41
|
+
const STREAMING_THRESHOLD_MB: u64 = 500;
|
|
42
|
+
|
|
43
|
+
// Adaptive Compression Levels
|
|
44
|
+
const COMPRESSION_ULTRA_FAST: i32 = 1; // < 100MB files, lots of RAM
|
|
45
|
+
const COMPRESSION_FAST: i32 = 2; // < 500MB files
|
|
46
|
+
const COMPRESSION_BALANCED: i32 = 3; // Default, 500MB-2GB
|
|
47
|
+
const COMPRESSION_SMALL_FILES: i32 = 5; // Many small files
|
|
48
|
+
|
|
49
|
+
// RAM Usage Tiers
|
|
50
|
+
const RAM_TIER_ULTRA: u64 = 16384; // 16GB+ - Aggressive optimization
|
|
51
|
+
const RAM_TIER_HIGH: u64 = 8192; // 8GB+ - Fast mode
|
|
52
|
+
const RAM_TIER_MEDIUM: u64 = 4096; // 4GB+ - Balanced
|
|
53
|
+
const RAM_TIER_LOW: u64 = 2048; // 2GB+ - Conservative
|
|
54
|
+
|
|
27
55
|
#[derive(Parser)]
|
|
28
56
|
#[command(author, version)]
|
|
29
57
|
struct Cli {
|
|
@@ -47,6 +75,8 @@ enum Commands {
|
|
|
47
75
|
/// optional zstd dictionary file for payload compression
|
|
48
76
|
#[arg(long, value_name = "FILE")]
|
|
49
77
|
dict: Option<PathBuf>,
|
|
78
|
+
#[arg(long, value_name = "MB")]
|
|
79
|
+
ram_budget_mb: Option<u64>,
|
|
50
80
|
},
|
|
51
81
|
|
|
52
82
|
List {
|
|
@@ -100,6 +130,8 @@ enum Commands {
|
|
|
100
130
|
/// optional dictionary file used during decompression
|
|
101
131
|
#[arg(long, value_name = "FILE")]
|
|
102
132
|
dict: Option<PathBuf>,
|
|
133
|
+
#[arg(long, value_name = "MB")]
|
|
134
|
+
ram_budget_mb: Option<u64>,
|
|
103
135
|
},
|
|
104
136
|
Crc32 {
|
|
105
137
|
input: PathBuf,
|
|
@@ -118,10 +150,192 @@ fn read_all(path: &PathBuf) -> anyhow::Result<Vec<u8>> {
|
|
|
118
150
|
Ok(buf)
|
|
119
151
|
}
|
|
120
152
|
|
|
153
|
+
pub fn parse_linux_mem_available_mb() -> Option<u64> {
|
|
154
|
+
let meminfo = std::fs::read_to_string("/proc/meminfo").ok()?;
|
|
155
|
+
let line = meminfo.lines().find(|l| l.starts_with("MemAvailable:"))?;
|
|
156
|
+
let kb = line
|
|
157
|
+
.split_whitespace()
|
|
158
|
+
.nth(1)
|
|
159
|
+
.and_then(|v| v.parse::<u64>().ok())?;
|
|
160
|
+
Some(kb / 1024)
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
fn parse_total_ram_mb() -> Option<u64> {
|
|
164
|
+
let meminfo = std::fs::read_to_string("/proc/meminfo").ok()?;
|
|
165
|
+
let line = meminfo.lines().find(|l| l.starts_with("MemTotal:"))?;
|
|
166
|
+
let kb = line
|
|
167
|
+
.split_whitespace()
|
|
168
|
+
.nth(1)
|
|
169
|
+
.and_then(|v| v.parse::<u64>().ok())?;
|
|
170
|
+
Some(kb / 1024)
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
fn get_cpu_cores() -> usize {
|
|
174
|
+
std::thread::available_parallelism()
|
|
175
|
+
.map(|p| p.get())
|
|
176
|
+
.unwrap_or(4)
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/// Determine RAM tier based on available memory
|
|
180
|
+
fn get_ram_tier(available_mb: u64) -> &'static str {
|
|
181
|
+
match available_mb {
|
|
182
|
+
x if x >= RAM_TIER_ULTRA => "ultra",
|
|
183
|
+
x if x >= RAM_TIER_HIGH => "high",
|
|
184
|
+
x if x >= RAM_TIER_MEDIUM => "medium",
|
|
185
|
+
_ => "low",
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/// Calculate adaptive RAM budget with aggressive optimization for Pyxelze
|
|
190
|
+
fn auto_ram_budget_mb() -> u64 {
|
|
191
|
+
let total_mb = parse_total_ram_mb().unwrap_or(4096);
|
|
192
|
+
let available_mb = parse_linux_mem_available_mb().unwrap_or(total_mb / 2);
|
|
193
|
+
let cpu_cores = get_cpu_cores();
|
|
194
|
+
|
|
195
|
+
// Base calculation: use up to 85% of available RAM for ultra-fast mode
|
|
196
|
+
let base_budget = available_mb.saturating_mul(85) / 100;
|
|
197
|
+
|
|
198
|
+
// Adjust based on RAM tier and CPU cores
|
|
199
|
+
let tier_multiplier = match get_ram_tier(total_mb) {
|
|
200
|
+
"ultra" if cpu_cores >= 8 => 90, // 16GB+ with 8+ cores: 90%
|
|
201
|
+
"ultra" => 85, // 16GB+ with fewer cores
|
|
202
|
+
"high" if cpu_cores >= 6 => 80, // 8GB+ with 6+ cores
|
|
203
|
+
"high" => 75, // 8GB+
|
|
204
|
+
"medium" => 70, // 4GB+
|
|
205
|
+
_ => 65, // 2GB+
|
|
206
|
+
};
|
|
207
|
+
|
|
208
|
+
let budget = available_mb.saturating_mul(tier_multiplier) / 100;
|
|
209
|
+
let budget = budget.max(MIN_RAM_BUDGET_MB);
|
|
210
|
+
|
|
211
|
+
// Cap at reasonable maximum to prevent OOM
|
|
212
|
+
let max_budget = (total_mb / 2).max(8192);
|
|
213
|
+
budget.min(max_budget)
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/// Auto-select optimal compression level based on file size and available RAM
|
|
217
|
+
fn auto_compression_level(file_size_mb: u64, ram_budget_mb: u64) -> i32 {
|
|
218
|
+
// Ultra-fast for small files with lots of RAM
|
|
219
|
+
if file_size_mb < FAST_COMPRESSION_THRESHOLD_MB && ram_budget_mb >= RAM_TIER_HIGH {
|
|
220
|
+
return COMPRESSION_ULTRA_FAST;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// Fast mode for medium files
|
|
224
|
+
if file_size_mb < STREAMING_THRESHOLD_MB {
|
|
225
|
+
return COMPRESSION_FAST;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// Balanced for large files
|
|
229
|
+
if ram_budget_mb >= RAM_TIER_MEDIUM {
|
|
230
|
+
return COMPRESSION_BALANCED;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// Conservative for low RAM
|
|
234
|
+
COMPRESSION_SMALL_FILES
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/// Determine if streaming mode should be used
|
|
238
|
+
fn should_use_streaming(file_size_mb: u64, ram_budget_mb: u64) -> bool {
|
|
239
|
+
// Force streaming for very large files
|
|
240
|
+
if file_size_mb >= 2048 { // 2GB+
|
|
241
|
+
return true;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// Use streaming when file is larger than 60% of RAM budget
|
|
245
|
+
let threshold = ram_budget_mb.saturating_mul(60) / 100;
|
|
246
|
+
file_size_mb >= threshold
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/// Get optimal thread count for zstd based on RAM and CPU
|
|
250
|
+
fn optimal_zstd_threads(file_size_mb: u64, ram_budget_mb: u64) -> i32 {
|
|
251
|
+
let cpu_cores = get_cpu_cores();
|
|
252
|
+
let ram_tier = get_ram_tier(parse_total_ram_mb().unwrap_or(4096));
|
|
253
|
+
|
|
254
|
+
match ram_tier {
|
|
255
|
+
"ultra" => cpu_cores.min(16) as i32,
|
|
256
|
+
"high" => cpu_cores.min(8) as i32,
|
|
257
|
+
_ if file_size_mb > 1000 => cpu_cores.min(4) as i32,
|
|
258
|
+
_ => cpu_cores.min(2) as i32,
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
fn resolve_ram_budget_mb(cli_value: Option<u64>) -> u64 {
|
|
263
|
+
if let Some(v) = cli_value {
|
|
264
|
+
return v.max(MIN_RAM_BUDGET_MB);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
if let Ok(v) = std::env::var("ROX_RAM_BUDGET_MB") {
|
|
268
|
+
if let Ok(parsed) = v.trim().parse::<u64>() {
|
|
269
|
+
return parsed.max(MIN_RAM_BUDGET_MB);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
auto_ram_budget_mb()
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
fn effective_ram_budget_mb() -> u64 {
|
|
277
|
+
if let Ok(v) = std::env::var("ROX_RAM_BUDGET_MB_EFFECTIVE") {
|
|
278
|
+
if let Ok(parsed) = v.trim().parse::<u64>() {
|
|
279
|
+
return parsed.max(MIN_RAM_BUDGET_MB);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
resolve_ram_budget_mb(None)
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
fn ram_budget_bytes(ram_budget_mb: u64) -> u64 {
|
|
286
|
+
ram_budget_mb.saturating_mul(MB_U64)
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
fn streaming_preference_threshold_bytes(ram_budget_mb: u64) -> u64 {
|
|
290
|
+
ram_budget_bytes(ram_budget_mb)
|
|
291
|
+
.saturating_mul(60)
|
|
292
|
+
.saturating_div(100)
|
|
293
|
+
.max(64 * MB_U64)
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
fn normalize_png_archive_bytes(png_data: &[u8], passphrase: Option<&str>) -> anyhow::Result<Vec<u8>> {
|
|
297
|
+
let payload = png_utils::extract_payload_from_png(png_data).map_err(|e| anyhow::anyhow!(e))?;
|
|
298
|
+
if payload.is_empty() {
|
|
299
|
+
return Err(anyhow::anyhow!("Empty payload"));
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
if payload[0] == 0x00u8 {
|
|
303
|
+
Ok(payload[1..].to_vec())
|
|
304
|
+
} else {
|
|
305
|
+
let decrypted = crate::crypto::try_decrypt(&payload, passphrase)
|
|
306
|
+
.map_err(|e| anyhow::anyhow!("Encrypted payload: {}", e))?;
|
|
307
|
+
Ok(decrypted)
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
fn unpack_archive_bytes(
|
|
312
|
+
normalized: Vec<u8>,
|
|
313
|
+
out_dir: &std::path::Path,
|
|
314
|
+
files_slice: Option<&[String]>,
|
|
315
|
+
progress: Option<&(dyn Fn(u64, u64, &str) + Send)>,
|
|
316
|
+
) -> anyhow::Result<Vec<String>> {
|
|
317
|
+
let mut reader: Box<dyn std::io::Read> = if normalized.starts_with(b"ROX1") {
|
|
318
|
+
Box::new(std::io::Cursor::new(normalized[4..].to_vec()))
|
|
319
|
+
} else {
|
|
320
|
+
let mut dec = zstd::stream::Decoder::new(std::io::Cursor::new(normalized))
|
|
321
|
+
.map_err(|e| anyhow::anyhow!("zstd decoder init: {}", e))?;
|
|
322
|
+
dec.window_log_max(31)
|
|
323
|
+
.map_err(|e| anyhow::anyhow!("zstd window_log_max: {}", e))?;
|
|
324
|
+
Box::new(dec)
|
|
325
|
+
};
|
|
326
|
+
|
|
327
|
+
std::fs::create_dir_all(out_dir)
|
|
328
|
+
.map_err(|e| anyhow::anyhow!("Cannot create output directory {:?}: {}", out_dir, e))?;
|
|
329
|
+
packer::unpack_stream_to_dir(&mut reader, out_dir, files_slice, progress, 0)
|
|
330
|
+
.map_err(|e| anyhow::anyhow!(e))
|
|
331
|
+
}
|
|
332
|
+
|
|
121
333
|
fn write_all(path: &PathBuf, data: &[u8]) -> anyhow::Result<()> {
|
|
122
334
|
let f = File::create(path)?;
|
|
123
|
-
let
|
|
124
|
-
|
|
335
|
+
let budget_bytes = ram_budget_bytes(effective_ram_budget_mb());
|
|
336
|
+
let dynamic_cap = (budget_bytes / 24)
|
|
337
|
+
.clamp(MIN_WRITE_BUFFER_BYTES as u64, MAX_WRITE_BUFFER_BYTES as u64) as usize;
|
|
338
|
+
let buf_size = dynamic_cap.min(data.len().max(8192));
|
|
125
339
|
let mut writer = std::io::BufWriter::with_capacity(buf_size, f);
|
|
126
340
|
writer.write_all(data)?;
|
|
127
341
|
writer.flush()?;
|
|
@@ -166,7 +380,9 @@ fn parse_requested_files(files: &str) -> anyhow::Result<Vec<String>> {
|
|
|
166
380
|
println!("wrote {} bytes dictionary to {:?}", dict.len(), output);
|
|
167
381
|
return Ok(());
|
|
168
382
|
}
|
|
169
|
-
Commands::Encode { input, output, level, passphrase, encrypt, name, dict } => {
|
|
383
|
+
Commands::Encode { input, output, level, passphrase, encrypt, name, dict, ram_budget_mb } => {
|
|
384
|
+
let ram_budget_mb = resolve_ram_budget_mb(ram_budget_mb);
|
|
385
|
+
std::env::set_var("ROX_RAM_BUDGET_MB_EFFECTIVE", ram_budget_mb.to_string());
|
|
170
386
|
let is_dir = input.is_dir();
|
|
171
387
|
|
|
172
388
|
let file_name = name.as_deref()
|
|
@@ -205,7 +421,7 @@ fn parse_requested_files(files: &str) -> anyhow::Result<Vec<String>> {
|
|
|
205
421
|
None => None,
|
|
206
422
|
};
|
|
207
423
|
|
|
208
|
-
let use_streaming = payload.len()
|
|
424
|
+
let use_streaming = (payload.len() as u64) > streaming_preference_threshold_bytes(ram_budget_mb);
|
|
209
425
|
eprintln!("PROGRESS:50:100:encoding");
|
|
210
426
|
|
|
211
427
|
if use_streaming {
|
|
@@ -359,7 +575,9 @@ fn parse_requested_files(files: &str) -> anyhow::Result<Vec<String>> {
|
|
|
359
575
|
let dest = output.unwrap_or_else(|| PathBuf::from("out.zst"));
|
|
360
576
|
write_all(&dest, &out)?;
|
|
361
577
|
}
|
|
362
|
-
Commands::Decompress { input, output, files, passphrase, dict } => {
|
|
578
|
+
Commands::Decompress { input, output, files, passphrase, dict, ram_budget_mb } => {
|
|
579
|
+
let ram_budget_mb = resolve_ram_budget_mb(ram_budget_mb);
|
|
580
|
+
std::env::set_var("ROX_RAM_BUDGET_MB_EFFECTIVE", ram_budget_mb.to_string());
|
|
363
581
|
let file_size = std::fs::metadata(&input).map(|m| m.len()).unwrap_or(0);
|
|
364
582
|
let is_png_file = input.extension().map(|e| e == "png").unwrap_or(false)
|
|
365
583
|
|| (file_size >= 8 && {
|
|
@@ -373,45 +591,87 @@ fn parse_requested_files(files: &str) -> anyhow::Result<Vec<String>> {
|
|
|
373
591
|
None => None,
|
|
374
592
|
};
|
|
375
593
|
|
|
376
|
-
if is_png_file &&
|
|
377
|
-
let out_dir =
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
594
|
+
if is_png_file && dict.is_none() {
|
|
595
|
+
let out_dir = if requested_files.is_some() {
|
|
596
|
+
output.clone().unwrap_or_else(|| PathBuf::from("."))
|
|
597
|
+
} else {
|
|
598
|
+
output.clone().unwrap_or_else(|| PathBuf::from("out.raw"))
|
|
599
|
+
};
|
|
600
|
+
|
|
601
|
+
let should_try_streaming = file_size >= streaming_preference_threshold_bytes(ram_budget_mb);
|
|
602
|
+
|
|
603
|
+
if should_try_streaming {
|
|
604
|
+
let streaming_result = if let Some(ref selected) = requested_files {
|
|
605
|
+
streaming_decode::streaming_decode_selected_to_dir_encrypted_with_progress(
|
|
606
|
+
&input,
|
|
607
|
+
&out_dir,
|
|
608
|
+
Some(selected.as_slice()),
|
|
609
|
+
passphrase.as_deref(),
|
|
610
|
+
Some(Box::new(|current, total, step| {
|
|
611
|
+
eprintln!("PROGRESS:{}:{}:{}", current, total, step);
|
|
612
|
+
})),
|
|
613
|
+
)
|
|
614
|
+
} else {
|
|
615
|
+
streaming_decode::streaming_decode_to_dir_encrypted_with_progress(
|
|
616
|
+
&input,
|
|
617
|
+
&out_dir,
|
|
618
|
+
passphrase.as_deref(),
|
|
619
|
+
Some(Box::new(|current, total, step| {
|
|
620
|
+
eprintln!("PROGRESS:{}:{}:{}", current, total, step);
|
|
621
|
+
})),
|
|
622
|
+
)
|
|
623
|
+
};
|
|
624
|
+
|
|
625
|
+
match streaming_result {
|
|
626
|
+
Ok(written) => {
|
|
627
|
+
eprintln!("PROGRESS:100:100:done");
|
|
628
|
+
println!("Unpacked {} files", written.len());
|
|
629
|
+
return Ok(());
|
|
630
|
+
}
|
|
631
|
+
Err(e) => {
|
|
632
|
+
eprintln!("PROGRESS:12:100:streaming_fallback");
|
|
633
|
+
eprintln!("Streaming decode failed, falling back to reconstruction: {}", e);
|
|
634
|
+
}
|
|
393
635
|
}
|
|
394
636
|
}
|
|
395
|
-
}
|
|
396
637
|
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
638
|
+
if file_size > ram_budget_bytes(ram_budget_mb) {
|
|
639
|
+
return Err(anyhow::anyhow!(
|
|
640
|
+
"PNG fallback requires in-memory reconstruction ({} MB file > {} MB RAM budget). Increase --ram-budget-mb.",
|
|
641
|
+
file_size / MB_U64,
|
|
642
|
+
ram_budget_mb
|
|
643
|
+
));
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
let buf = read_all(&input)?;
|
|
647
|
+
|
|
648
|
+
eprintln!("PROGRESS:20:100:decompressing");
|
|
649
|
+
let progress_cb = |current: u64, total: u64, step: &str| {
|
|
650
|
+
eprintln!("PROGRESS:{}:{}:{}", current, total, step);
|
|
651
|
+
};
|
|
652
|
+
|
|
653
|
+
let normalized = normalize_png_archive_bytes(&buf, passphrase.as_deref())?;
|
|
654
|
+
let written = unpack_archive_bytes(
|
|
655
|
+
normalized,
|
|
402
656
|
&out_dir,
|
|
403
657
|
requested_files.as_deref(),
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
eprintln!("PROGRESS:{}:{}:{}", current, total, step);
|
|
407
|
-
})),
|
|
408
|
-
).map_err(|e| anyhow::anyhow!(e))?;
|
|
658
|
+
Some(&progress_cb),
|
|
659
|
+
)?;
|
|
409
660
|
eprintln!("PROGRESS:100:100:done");
|
|
410
661
|
println!("Unpacked {} files", written.len());
|
|
411
662
|
return Ok(());
|
|
412
663
|
}
|
|
413
664
|
|
|
665
|
+
if file_size > ram_budget_bytes(ram_budget_mb) {
|
|
666
|
+
return Err(anyhow::anyhow!(
|
|
667
|
+
"Input is {} MB but RAM budget is {} MB. Increase --ram-budget-mb for non-streaming decode paths.",
|
|
668
|
+
file_size / MB_U64,
|
|
669
|
+
ram_budget_mb
|
|
670
|
+
));
|
|
671
|
+
}
|
|
672
|
+
|
|
414
673
|
let buf = read_all(&input)?;
|
|
674
|
+
|
|
415
675
|
eprintln!("PROGRESS:20:100:decompressing");
|
|
416
676
|
let dict_bytes: Option<Vec<u8>> = match dict {
|
|
417
677
|
Some(path) => Some(read_all(&path)?),
|
|
@@ -519,8 +519,12 @@ fn tar_unpack_from_reader_with_progress<R: Read>(
|
|
|
519
519
|
}
|
|
520
520
|
}
|
|
521
521
|
|
|
522
|
+
// NTFS optimization: larger buffer reduces syscalls (16MB max, 256KB min)
|
|
523
|
+
let buffer_size = (entry_size as usize)
|
|
524
|
+
.min(16 * 1024 * 1024) // 16MB max buffer
|
|
525
|
+
.max(256 * 1024); // 256KB min buffer for NTFS
|
|
522
526
|
let mut f = std::io::BufWriter::with_capacity(
|
|
523
|
-
|
|
527
|
+
buffer_size,
|
|
524
528
|
std::fs::File::create(&dest).map_err(|e| format!("create {:?}: {}", dest, e))?,
|
|
525
529
|
);
|
|
526
530
|
std::io::copy(&mut entry, &mut f).map_err(|e| format!("write {:?}: {}", dest, e))?;
|
|
@@ -316,14 +316,24 @@ fn estimate_zst_capacity(total_bytes: u64) -> usize {
|
|
|
316
316
|
|
|
317
317
|
fn select_zstd_threads(total_bytes: u64) -> u32 {
|
|
318
318
|
let max_threads = num_cpus::get().max(1) as u32;
|
|
319
|
-
|
|
319
|
+
let ram_mb = crate::parse_linux_mem_available_mb().unwrap_or(4096);
|
|
320
|
+
|
|
321
|
+
// Aggressive multi-threading for Pyxelze speed target (<10s)
|
|
322
|
+
if total_bytes <= 16 * MB {
|
|
323
|
+
// Small files: single thread to avoid overhead
|
|
320
324
|
1
|
|
321
|
-
} else if total_bytes <=
|
|
325
|
+
} else if total_bytes <= 64 * MB {
|
|
326
|
+
// Small-medium files: 2 threads
|
|
322
327
|
max_threads.min(2)
|
|
323
|
-
} else if total_bytes <=
|
|
328
|
+
} else if total_bytes <= 256 * MB || ram_mb >= 8192 {
|
|
329
|
+
// Medium files or high RAM: up to 4 threads
|
|
324
330
|
max_threads.min(4)
|
|
325
|
-
} else {
|
|
331
|
+
} else if total_bytes <= 1024 * MB || ram_mb >= 4096 {
|
|
332
|
+
// Large files or medium RAM: up to 8 threads
|
|
326
333
|
max_threads.min(8)
|
|
334
|
+
} else {
|
|
335
|
+
// Very large files: use all available cores up to 16
|
|
336
|
+
max_threads.min(16)
|
|
327
337
|
}
|
|
328
338
|
}
|
|
329
339
|
|
package/package.json
CHANGED
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|