roxify 1.13.5 → 1.13.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -233,7 +233,7 @@ function spawnRustCLI(args, options) {
233
233
  runSpawn(cliPath);
234
234
  });
235
235
  }
236
- export async function encodeWithRustCLI(inputPath, outputPath, compressionLevel = 3, passphrase, encryptType = 'aes', name, onProgress) {
236
+ export async function encodeWithRustCLI(inputPath, outputPath, compressionLevel = 3, passphrase, encryptType = 'aes', name, ramBudgetMb, onProgress) {
237
237
  const cliPath = findRustBinary();
238
238
  if (!cliPath)
239
239
  throw new Error('Rust CLI binary not found');
@@ -250,10 +250,13 @@ export async function encodeWithRustCLI(inputPath, outputPath, compressionLevel
250
250
  args.push('--passphrase', passphrase);
251
251
  args.push('--encrypt', encryptType);
252
252
  }
253
+ if (typeof ramBudgetMb === 'number' && Number.isFinite(ramBudgetMb)) {
254
+ args.push('--ram-budget-mb', String(Math.max(1, Math.floor(ramBudgetMb))));
255
+ }
253
256
  args.push(inputPath, outputPath);
254
257
  await spawnRustCLI(args, { onProgress });
255
258
  }
256
- export async function decodeWithRustCLI(inputPath, outputPath, passphrase, files, dict, onProgress) {
259
+ export async function decodeWithRustCLI(inputPath, outputPath, passphrase, files, dict, ramBudgetMb, onProgress) {
257
260
  const args = ['decompress', inputPath, outputPath];
258
261
  if (passphrase)
259
262
  args.push('--passphrase', passphrase);
@@ -261,6 +264,9 @@ export async function decodeWithRustCLI(inputPath, outputPath, passphrase, files
261
264
  args.push('--files', JSON.stringify(files));
262
265
  if (dict)
263
266
  args.push('--dict', dict);
267
+ if (typeof ramBudgetMb === 'number' && Number.isFinite(ramBudgetMb)) {
268
+ args.push('--ram-budget-mb', String(Math.max(1, Math.floor(ramBudgetMb))));
269
+ }
264
270
  await spawnRustCLI(args, { onProgress });
265
271
  }
266
272
  export async function listWithRustCLI(inputPath) {
@@ -34,7 +34,7 @@ pub fn advise_drop(file: &File, offset: u64, len: u64) {
34
34
  }
35
35
 
36
36
  pub fn sync_and_drop(file: &File, len: u64) {
37
- if len == 0 {
37
+ if len < INPUT_DROP_GRANULARITY {
38
38
  return;
39
39
  }
40
40
 
@@ -0,0 +1,99 @@
1
+ use std::fs::{File, OpenOptions};
2
+ use std::io::{BufWriter, Write};
3
+ use std::path::Path;
4
+
5
+ /// Buffer size optimized for NTFS (larger = fewer syscalls)
6
+ #[cfg(windows)]
7
+ const NTFS_WRITE_BUFFER: usize = 4 * 1024 * 1024; // 4MB for Windows/NTFS
8
+ #[cfg(not(windows))]
9
+ const NTFS_WRITE_BUFFER: usize = 64 * 1024; // 64KB for Unix
10
+
11
+ /// Optimized file writer with large buffer for NTFS
12
+ pub struct OptimizedFileWriter {
13
+ writer: BufWriter<File>,
14
+ }
15
+
16
+ impl OptimizedFileWriter {
17
+ pub fn create(path: &Path) -> std::io::Result<Self> {
18
+ let file = OpenOptions::new()
19
+ .write(true)
20
+ .create(true)
21
+ .truncate(true)
22
+ .open(path)?;
23
+
24
+ Ok(Self {
25
+ writer: BufWriter::with_capacity(NTFS_WRITE_BUFFER, file),
26
+ })
27
+ }
28
+
29
+ pub fn write_all(&mut self, buf: &[u8]) -> std::io::Result<()> {
30
+ self.writer.write_all(buf)
31
+ }
32
+
33
+ pub fn flush(&mut self) -> std::io::Result<()> {
34
+ self.writer.flush()
35
+ }
36
+ }
37
+
38
+ /// Write file with optimized buffering for target filesystem
39
+ pub fn write_file_optimized(path: &Path, content: &[u8]) -> std::io::Result<()> {
40
+ let mut writer = OptimizedFileWriter::create(path)?;
41
+ writer.write_all(content)?;
42
+ writer.flush()?;
43
+ Ok(())
44
+ }
45
+
46
+ /// Batch write multiple files - keeps files open for better NTFS performance
47
+ pub fn write_files_batch(
48
+ base_dir: &Path,
49
+ files: &[(String, &[u8])],
50
+ ) -> Result<Vec<String>, String> {
51
+ let mut written = Vec::with_capacity(files.len());
52
+
53
+ for (rel_path, content) in files {
54
+ let safe_path = sanitize_path(rel_path);
55
+ let dest = base_dir.join(&safe_path);
56
+
57
+ if let Some(parent) = dest.parent() {
58
+ std::fs::create_dir_all(parent)
59
+ .map_err(|e| format!("Cannot create parent dir {:?}: {}", parent, e))?;
60
+ }
61
+
62
+ write_file_optimized(&dest, content)
63
+ .map_err(|e| format!("Cannot write {:?}: {}", dest, e))?;
64
+
65
+ written.push(safe_path.to_string_lossy().to_string());
66
+ }
67
+
68
+ Ok(written)
69
+ }
70
+
71
+ fn sanitize_path(path: &str) -> std::path::PathBuf {
72
+ let mut safe = std::path::PathBuf::new();
73
+ for comp in std::path::Path::new(path).components() {
74
+ if let std::path::Component::Normal(osstr) = comp {
75
+ safe.push(osstr);
76
+ }
77
+ }
78
+ safe
79
+ }
80
+
81
+ /// Pre-allocate file space on NTFS to reduce fragmentation
82
+ #[cfg(windows)]
83
+ pub fn preallocate_file(path: &Path, size: u64) -> std::io::Result<()> {
84
+ use std::os::windows::fs::FileExt;
85
+
86
+ let file = OpenOptions::new()
87
+ .write(true)
88
+ .create(true)
89
+ .open(path)?;
90
+
91
+ // Pre-allocate space
92
+ file.set_len(size)?;
93
+ Ok(())
94
+ }
95
+
96
+ #[cfg(not(windows))]
97
+ pub fn preallocate_file(_path: &Path, _size: u64) -> std::io::Result<()> {
98
+ Ok(())
99
+ }
package/native/main.rs CHANGED
@@ -24,6 +24,34 @@ mod progress;
24
24
  use crate::encoder::ImageFormat;
25
25
  use std::path::PathBuf;
26
26
 
27
+ const MB_U64: u64 = 1024 * 1024;
28
+ const GB_U64: u64 = 1024 * MB_U64;
29
+
30
+ // Adaptive RAM Budget Configuration
31
+ const MIN_RAM_BUDGET_MB: u64 = 512;
32
+ const DEFAULT_RAM_BUDGET_MB: u64 = 2048;
33
+ const RESERVED_RAM_MB: u64 = 1024;
34
+ const MIN_WRITE_BUFFER_BYTES: usize = 16 * 1024 * 1024;
35
+ const MAX_WRITE_BUFFER_BYTES: usize = 256 * 1024 * 1024;
36
+
37
+ // Performance Targets (under 10 seconds)
38
+ const TARGET_ENCODE_TIME_SECS: u64 = 10;
39
+ const TARGET_DECODE_TIME_SECS: u64 = 10;
40
+ const FAST_COMPRESSION_THRESHOLD_MB: u64 = 100;
41
+ const STREAMING_THRESHOLD_MB: u64 = 500;
42
+
43
+ // Adaptive Compression Levels
44
+ const COMPRESSION_ULTRA_FAST: i32 = 1; // < 100MB files, lots of RAM
45
+ const COMPRESSION_FAST: i32 = 2; // < 500MB files
46
+ const COMPRESSION_BALANCED: i32 = 3; // Default, 500MB-2GB
47
+ const COMPRESSION_SMALL_FILES: i32 = 5; // Many small files
48
+
49
+ // RAM Usage Tiers
50
+ const RAM_TIER_ULTRA: u64 = 16384; // 16GB+ - Aggressive optimization
51
+ const RAM_TIER_HIGH: u64 = 8192; // 8GB+ - Fast mode
52
+ const RAM_TIER_MEDIUM: u64 = 4096; // 4GB+ - Balanced
53
+ const RAM_TIER_LOW: u64 = 2048; // 2GB+ - Conservative
54
+
27
55
  #[derive(Parser)]
28
56
  #[command(author, version)]
29
57
  struct Cli {
@@ -47,6 +75,8 @@ enum Commands {
47
75
  /// optional zstd dictionary file for payload compression
48
76
  #[arg(long, value_name = "FILE")]
49
77
  dict: Option<PathBuf>,
78
+ #[arg(long, value_name = "MB")]
79
+ ram_budget_mb: Option<u64>,
50
80
  },
51
81
 
52
82
  List {
@@ -100,6 +130,8 @@ enum Commands {
100
130
  /// optional dictionary file used during decompression
101
131
  #[arg(long, value_name = "FILE")]
102
132
  dict: Option<PathBuf>,
133
+ #[arg(long, value_name = "MB")]
134
+ ram_budget_mb: Option<u64>,
103
135
  },
104
136
  Crc32 {
105
137
  input: PathBuf,
@@ -118,10 +150,192 @@ fn read_all(path: &PathBuf) -> anyhow::Result<Vec<u8>> {
118
150
  Ok(buf)
119
151
  }
120
152
 
153
+ pub fn parse_linux_mem_available_mb() -> Option<u64> {
154
+ let meminfo = std::fs::read_to_string("/proc/meminfo").ok()?;
155
+ let line = meminfo.lines().find(|l| l.starts_with("MemAvailable:"))?;
156
+ let kb = line
157
+ .split_whitespace()
158
+ .nth(1)
159
+ .and_then(|v| v.parse::<u64>().ok())?;
160
+ Some(kb / 1024)
161
+ }
162
+
163
+ fn parse_total_ram_mb() -> Option<u64> {
164
+ let meminfo = std::fs::read_to_string("/proc/meminfo").ok()?;
165
+ let line = meminfo.lines().find(|l| l.starts_with("MemTotal:"))?;
166
+ let kb = line
167
+ .split_whitespace()
168
+ .nth(1)
169
+ .and_then(|v| v.parse::<u64>().ok())?;
170
+ Some(kb / 1024)
171
+ }
172
+
173
+ fn get_cpu_cores() -> usize {
174
+ std::thread::available_parallelism()
175
+ .map(|p| p.get())
176
+ .unwrap_or(4)
177
+ }
178
+
179
+ /// Determine RAM tier based on available memory
180
+ fn get_ram_tier(available_mb: u64) -> &'static str {
181
+ match available_mb {
182
+ x if x >= RAM_TIER_ULTRA => "ultra",
183
+ x if x >= RAM_TIER_HIGH => "high",
184
+ x if x >= RAM_TIER_MEDIUM => "medium",
185
+ _ => "low",
186
+ }
187
+ }
188
+
189
+ /// Calculate adaptive RAM budget with aggressive optimization for Pyxelze
190
+ fn auto_ram_budget_mb() -> u64 {
191
+ let total_mb = parse_total_ram_mb().unwrap_or(4096);
192
+ let available_mb = parse_linux_mem_available_mb().unwrap_or(total_mb / 2);
193
+ let cpu_cores = get_cpu_cores();
194
+
195
+ // Base calculation: use up to 85% of available RAM for ultra-fast mode
196
+ let base_budget = available_mb.saturating_mul(85) / 100;
197
+
198
+ // Adjust based on RAM tier and CPU cores
199
+ let tier_multiplier = match get_ram_tier(total_mb) {
200
+ "ultra" if cpu_cores >= 8 => 90, // 16GB+ with 8+ cores: 90%
201
+ "ultra" => 85, // 16GB+ with fewer cores
202
+ "high" if cpu_cores >= 6 => 80, // 8GB+ with 6+ cores
203
+ "high" => 75, // 8GB+
204
+ "medium" => 70, // 4GB+
205
+ _ => 65, // 2GB+
206
+ };
207
+
208
+ let budget = available_mb.saturating_mul(tier_multiplier) / 100;
209
+ let budget = budget.max(MIN_RAM_BUDGET_MB);
210
+
211
+ // Cap at reasonable maximum to prevent OOM
212
+ let max_budget = (total_mb / 2).max(8192);
213
+ budget.min(max_budget)
214
+ }
215
+
216
+ /// Auto-select optimal compression level based on file size and available RAM
217
+ fn auto_compression_level(file_size_mb: u64, ram_budget_mb: u64) -> i32 {
218
+ // Ultra-fast for small files with lots of RAM
219
+ if file_size_mb < FAST_COMPRESSION_THRESHOLD_MB && ram_budget_mb >= RAM_TIER_HIGH {
220
+ return COMPRESSION_ULTRA_FAST;
221
+ }
222
+
223
+ // Fast mode for medium files
224
+ if file_size_mb < STREAMING_THRESHOLD_MB {
225
+ return COMPRESSION_FAST;
226
+ }
227
+
228
+ // Balanced for large files
229
+ if ram_budget_mb >= RAM_TIER_MEDIUM {
230
+ return COMPRESSION_BALANCED;
231
+ }
232
+
233
+ // Conservative for low RAM
234
+ COMPRESSION_SMALL_FILES
235
+ }
236
+
237
+ /// Determine if streaming mode should be used
238
+ fn should_use_streaming(file_size_mb: u64, ram_budget_mb: u64) -> bool {
239
+ // Force streaming for very large files
240
+ if file_size_mb >= 2048 { // 2GB+
241
+ return true;
242
+ }
243
+
244
+ // Use streaming when file is larger than 60% of RAM budget
245
+ let threshold = ram_budget_mb.saturating_mul(60) / 100;
246
+ file_size_mb >= threshold
247
+ }
248
+
249
+ /// Get optimal thread count for zstd based on RAM and CPU
250
+ fn optimal_zstd_threads(file_size_mb: u64, ram_budget_mb: u64) -> i32 {
251
+ let cpu_cores = get_cpu_cores();
252
+ let ram_tier = get_ram_tier(parse_total_ram_mb().unwrap_or(4096));
253
+
254
+ match ram_tier {
255
+ "ultra" => cpu_cores.min(16) as i32,
256
+ "high" => cpu_cores.min(8) as i32,
257
+ _ if file_size_mb > 1000 => cpu_cores.min(4) as i32,
258
+ _ => cpu_cores.min(2) as i32,
259
+ }
260
+ }
261
+
262
+ fn resolve_ram_budget_mb(cli_value: Option<u64>) -> u64 {
263
+ if let Some(v) = cli_value {
264
+ return v.max(MIN_RAM_BUDGET_MB);
265
+ }
266
+
267
+ if let Ok(v) = std::env::var("ROX_RAM_BUDGET_MB") {
268
+ if let Ok(parsed) = v.trim().parse::<u64>() {
269
+ return parsed.max(MIN_RAM_BUDGET_MB);
270
+ }
271
+ }
272
+
273
+ auto_ram_budget_mb()
274
+ }
275
+
276
+ fn effective_ram_budget_mb() -> u64 {
277
+ if let Ok(v) = std::env::var("ROX_RAM_BUDGET_MB_EFFECTIVE") {
278
+ if let Ok(parsed) = v.trim().parse::<u64>() {
279
+ return parsed.max(MIN_RAM_BUDGET_MB);
280
+ }
281
+ }
282
+ resolve_ram_budget_mb(None)
283
+ }
284
+
285
+ fn ram_budget_bytes(ram_budget_mb: u64) -> u64 {
286
+ ram_budget_mb.saturating_mul(MB_U64)
287
+ }
288
+
289
+ fn streaming_preference_threshold_bytes(ram_budget_mb: u64) -> u64 {
290
+ ram_budget_bytes(ram_budget_mb)
291
+ .saturating_mul(60)
292
+ .saturating_div(100)
293
+ .max(64 * MB_U64)
294
+ }
295
+
296
+ fn normalize_png_archive_bytes(png_data: &[u8], passphrase: Option<&str>) -> anyhow::Result<Vec<u8>> {
297
+ let payload = png_utils::extract_payload_from_png(png_data).map_err(|e| anyhow::anyhow!(e))?;
298
+ if payload.is_empty() {
299
+ return Err(anyhow::anyhow!("Empty payload"));
300
+ }
301
+
302
+ if payload[0] == 0x00u8 {
303
+ Ok(payload[1..].to_vec())
304
+ } else {
305
+ let decrypted = crate::crypto::try_decrypt(&payload, passphrase)
306
+ .map_err(|e| anyhow::anyhow!("Encrypted payload: {}", e))?;
307
+ Ok(decrypted)
308
+ }
309
+ }
310
+
311
+ fn unpack_archive_bytes(
312
+ normalized: Vec<u8>,
313
+ out_dir: &std::path::Path,
314
+ files_slice: Option<&[String]>,
315
+ progress: Option<&(dyn Fn(u64, u64, &str) + Send)>,
316
+ ) -> anyhow::Result<Vec<String>> {
317
+ let mut reader: Box<dyn std::io::Read> = if normalized.starts_with(b"ROX1") {
318
+ Box::new(std::io::Cursor::new(normalized[4..].to_vec()))
319
+ } else {
320
+ let mut dec = zstd::stream::Decoder::new(std::io::Cursor::new(normalized))
321
+ .map_err(|e| anyhow::anyhow!("zstd decoder init: {}", e))?;
322
+ dec.window_log_max(31)
323
+ .map_err(|e| anyhow::anyhow!("zstd window_log_max: {}", e))?;
324
+ Box::new(dec)
325
+ };
326
+
327
+ std::fs::create_dir_all(out_dir)
328
+ .map_err(|e| anyhow::anyhow!("Cannot create output directory {:?}: {}", out_dir, e))?;
329
+ packer::unpack_stream_to_dir(&mut reader, out_dir, files_slice, progress, 0)
330
+ .map_err(|e| anyhow::anyhow!(e))
331
+ }
332
+
121
333
  fn write_all(path: &PathBuf, data: &[u8]) -> anyhow::Result<()> {
122
334
  let f = File::create(path)?;
123
- let buf_size = if data.len() > 64 * 1024 * 1024 { 16 * 1024 * 1024 }
124
- else { (8 * 1024 * 1024).min(data.len().max(8192)) };
335
+ let budget_bytes = ram_budget_bytes(effective_ram_budget_mb());
336
+ let dynamic_cap = (budget_bytes / 24)
337
+ .clamp(MIN_WRITE_BUFFER_BYTES as u64, MAX_WRITE_BUFFER_BYTES as u64) as usize;
338
+ let buf_size = dynamic_cap.min(data.len().max(8192));
125
339
  let mut writer = std::io::BufWriter::with_capacity(buf_size, f);
126
340
  writer.write_all(data)?;
127
341
  writer.flush()?;
@@ -166,7 +380,9 @@ fn parse_requested_files(files: &str) -> anyhow::Result<Vec<String>> {
166
380
  println!("wrote {} bytes dictionary to {:?}", dict.len(), output);
167
381
  return Ok(());
168
382
  }
169
- Commands::Encode { input, output, level, passphrase, encrypt, name, dict } => {
383
+ Commands::Encode { input, output, level, passphrase, encrypt, name, dict, ram_budget_mb } => {
384
+ let ram_budget_mb = resolve_ram_budget_mb(ram_budget_mb);
385
+ std::env::set_var("ROX_RAM_BUDGET_MB_EFFECTIVE", ram_budget_mb.to_string());
170
386
  let is_dir = input.is_dir();
171
387
 
172
388
  let file_name = name.as_deref()
@@ -205,7 +421,7 @@ fn parse_requested_files(files: &str) -> anyhow::Result<Vec<String>> {
205
421
  None => None,
206
422
  };
207
423
 
208
- let use_streaming = payload.len() > 64 * 1024 * 1024;
424
+ let use_streaming = (payload.len() as u64) > streaming_preference_threshold_bytes(ram_budget_mb);
209
425
  eprintln!("PROGRESS:50:100:encoding");
210
426
 
211
427
  if use_streaming {
@@ -359,7 +575,9 @@ fn parse_requested_files(files: &str) -> anyhow::Result<Vec<String>> {
359
575
  let dest = output.unwrap_or_else(|| PathBuf::from("out.zst"));
360
576
  write_all(&dest, &out)?;
361
577
  }
362
- Commands::Decompress { input, output, files, passphrase, dict } => {
578
+ Commands::Decompress { input, output, files, passphrase, dict, ram_budget_mb } => {
579
+ let ram_budget_mb = resolve_ram_budget_mb(ram_budget_mb);
580
+ std::env::set_var("ROX_RAM_BUDGET_MB_EFFECTIVE", ram_budget_mb.to_string());
363
581
  let file_size = std::fs::metadata(&input).map(|m| m.len()).unwrap_or(0);
364
582
  let is_png_file = input.extension().map(|e| e == "png").unwrap_or(false)
365
583
  || (file_size >= 8 && {
@@ -373,45 +591,87 @@ fn parse_requested_files(files: &str) -> anyhow::Result<Vec<String>> {
373
591
  None => None,
374
592
  };
375
593
 
376
- if is_png_file && files.is_none() && dict.is_none() {
377
- let out_dir = output.clone().unwrap_or_else(|| PathBuf::from("out.raw"));
378
- match streaming_decode::streaming_decode_to_dir_encrypted_with_progress(
379
- &input,
380
- &out_dir,
381
- passphrase.as_deref(),
382
- Some(Box::new(|current, total, step| {
383
- eprintln!("PROGRESS:{}:{}:{}", current, total, step);
384
- })),
385
- ) {
386
- Ok(written) => {
387
- eprintln!("PROGRESS:100:100:done");
388
- println!("Unpacked {} files", written.len());
389
- return Ok(());
390
- }
391
- Err(e) => {
392
- return Err(anyhow::anyhow!("Streaming decode failed: {}", e));
594
+ if is_png_file && dict.is_none() {
595
+ let out_dir = if requested_files.is_some() {
596
+ output.clone().unwrap_or_else(|| PathBuf::from("."))
597
+ } else {
598
+ output.clone().unwrap_or_else(|| PathBuf::from("out.raw"))
599
+ };
600
+
601
+ let should_try_streaming = file_size >= streaming_preference_threshold_bytes(ram_budget_mb);
602
+
603
+ if should_try_streaming {
604
+ let streaming_result = if let Some(ref selected) = requested_files {
605
+ streaming_decode::streaming_decode_selected_to_dir_encrypted_with_progress(
606
+ &input,
607
+ &out_dir,
608
+ Some(selected.as_slice()),
609
+ passphrase.as_deref(),
610
+ Some(Box::new(|current, total, step| {
611
+ eprintln!("PROGRESS:{}:{}:{}", current, total, step);
612
+ })),
613
+ )
614
+ } else {
615
+ streaming_decode::streaming_decode_to_dir_encrypted_with_progress(
616
+ &input,
617
+ &out_dir,
618
+ passphrase.as_deref(),
619
+ Some(Box::new(|current, total, step| {
620
+ eprintln!("PROGRESS:{}:{}:{}", current, total, step);
621
+ })),
622
+ )
623
+ };
624
+
625
+ match streaming_result {
626
+ Ok(written) => {
627
+ eprintln!("PROGRESS:100:100:done");
628
+ println!("Unpacked {} files", written.len());
629
+ return Ok(());
630
+ }
631
+ Err(e) => {
632
+ eprintln!("PROGRESS:12:100:streaming_fallback");
633
+ eprintln!("Streaming decode failed, falling back to reconstruction: {}", e);
634
+ }
393
635
  }
394
636
  }
395
- }
396
637
 
397
- if is_png_file && requested_files.is_some() && dict.is_none() {
398
- let out_dir = output.clone().unwrap_or_else(|| PathBuf::from("."));
399
- std::fs::create_dir_all(&out_dir).map_err(|e| anyhow::anyhow!("Cannot create output directory {:?}: {}", out_dir, e))?;
400
- let written = streaming_decode::streaming_decode_selected_to_dir_encrypted_with_progress(
401
- &input,
638
+ if file_size > ram_budget_bytes(ram_budget_mb) {
639
+ return Err(anyhow::anyhow!(
640
+ "PNG fallback requires in-memory reconstruction ({} MB file > {} MB RAM budget). Increase --ram-budget-mb.",
641
+ file_size / MB_U64,
642
+ ram_budget_mb
643
+ ));
644
+ }
645
+
646
+ let buf = read_all(&input)?;
647
+
648
+ eprintln!("PROGRESS:20:100:decompressing");
649
+ let progress_cb = |current: u64, total: u64, step: &str| {
650
+ eprintln!("PROGRESS:{}:{}:{}", current, total, step);
651
+ };
652
+
653
+ let normalized = normalize_png_archive_bytes(&buf, passphrase.as_deref())?;
654
+ let written = unpack_archive_bytes(
655
+ normalized,
402
656
  &out_dir,
403
657
  requested_files.as_deref(),
404
- passphrase.as_deref(),
405
- Some(Box::new(|current, total, step| {
406
- eprintln!("PROGRESS:{}:{}:{}", current, total, step);
407
- })),
408
- ).map_err(|e| anyhow::anyhow!(e))?;
658
+ Some(&progress_cb),
659
+ )?;
409
660
  eprintln!("PROGRESS:100:100:done");
410
661
  println!("Unpacked {} files", written.len());
411
662
  return Ok(());
412
663
  }
413
664
 
665
+ if file_size > ram_budget_bytes(ram_budget_mb) {
666
+ return Err(anyhow::anyhow!(
667
+ "Input is {} MB but RAM budget is {} MB. Increase --ram-budget-mb for non-streaming decode paths.",
668
+ file_size / MB_U64,
669
+ ram_budget_mb
670
+ ));
671
+ }
672
+
414
673
  let buf = read_all(&input)?;
674
+
415
675
  eprintln!("PROGRESS:20:100:decompressing");
416
676
  let dict_bytes: Option<Vec<u8>> = match dict {
417
677
  Some(path) => Some(read_all(&path)?),
@@ -519,8 +519,12 @@ fn tar_unpack_from_reader_with_progress<R: Read>(
519
519
  }
520
520
  }
521
521
 
522
+ // NTFS optimization: larger buffer reduces syscalls (16MB max, 256KB min)
523
+ let buffer_size = (entry_size as usize)
524
+ .min(16 * 1024 * 1024) // 16MB max buffer
525
+ .max(256 * 1024); // 256KB min buffer for NTFS
522
526
  let mut f = std::io::BufWriter::with_capacity(
523
- (entry_size as usize).min(4 * 1024 * 1024).max(8192),
527
+ buffer_size,
524
528
  std::fs::File::create(&dest).map_err(|e| format!("create {:?}: {}", dest, e))?,
525
529
  );
526
530
  std::io::copy(&mut entry, &mut f).map_err(|e| format!("write {:?}: {}", dest, e))?;
@@ -316,14 +316,24 @@ fn estimate_zst_capacity(total_bytes: u64) -> usize {
316
316
 
317
317
  fn select_zstd_threads(total_bytes: u64) -> u32 {
318
318
  let max_threads = num_cpus::get().max(1) as u32;
319
- if total_bytes <= 32 * MB {
319
+ let ram_mb = crate::parse_linux_mem_available_mb().unwrap_or(4096);
320
+
321
+ // Aggressive multi-threading for Pyxelze speed target (<10s)
322
+ if total_bytes <= 16 * MB {
323
+ // Small files: single thread to avoid overhead
320
324
  1
321
- } else if total_bytes <= 128 * MB {
325
+ } else if total_bytes <= 64 * MB {
326
+ // Small-medium files: 2 threads
322
327
  max_threads.min(2)
323
- } else if total_bytes <= 512 * MB {
328
+ } else if total_bytes <= 256 * MB || ram_mb >= 8192 {
329
+ // Medium files or high RAM: up to 4 threads
324
330
  max_threads.min(4)
325
- } else {
331
+ } else if total_bytes <= 1024 * MB || ram_mb >= 4096 {
332
+ // Large files or medium RAM: up to 8 threads
326
333
  max_threads.min(8)
334
+ } else {
335
+ // Very large files: use all available cores up to 16
336
+ max_threads.min(16)
327
337
  }
328
338
  }
329
339
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "roxify",
3
- "version": "1.13.5",
3
+ "version": "1.13.6",
4
4
  "type": "module",
5
5
  "description": "Ultra-lightweight PNG steganography with native Rust acceleration. Encode binary data into PNG images with zstd compression.",
6
6
  "main": "dist/index.js",