roxify 1.13.0 → 1.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.toml +1 -11
- package/README.md +20 -144
- package/dist/cli.js +86 -44
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/rox-macos-universal +0 -0
- package/dist/roxify_native +0 -0
- package/dist/roxify_native-macos-arm64 +0 -0
- package/dist/roxify_native-macos-x64 +0 -0
- package/dist/roxify_native.exe +0 -0
- package/dist/utils/rust-cli-wrapper.d.ts +3 -0
- package/dist/utils/rust-cli-wrapper.js +85 -55
- package/native/bench_hybrid.rs +1 -1
- package/native/hybrid.rs +17 -7
- package/native/lib.rs +0 -26
- package/native/main.rs +12 -1
- package/native/progress.rs +117 -18
- package/native/streaming_encode.rs +34 -2
- package/package.json +1 -1
- package/roxify_native-aarch64-apple-darwin.node +0 -0
- package/roxify_native-aarch64-pc-windows-msvc.node +0 -0
- package/roxify_native-aarch64-unknown-linux-gnu.node +0 -0
- package/roxify_native-i686-pc-windows-msvc.node +0 -0
- package/roxify_native-i686-unknown-linux-gnu.node +0 -0
- package/roxify_native-x86_64-apple-darwin.node +0 -0
- package/roxify_native-x86_64-pc-windows-msvc.node +0 -0
- package/roxify_native-x86_64-unknown-linux-gnu.node +0 -0
- package/native/gpu.rs +0 -116
package/Cargo.toml
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "roxify_native"
|
|
3
|
-
version = "1.13.
|
|
3
|
+
version = "1.13.2"
|
|
4
4
|
edition = "2021"
|
|
5
5
|
publish = false
|
|
6
6
|
|
|
@@ -52,25 +52,15 @@ rand = "0.8"
|
|
|
52
52
|
sha2 = "0.10"
|
|
53
53
|
mimalloc = "0.1"
|
|
54
54
|
|
|
55
|
-
wgpu = { version = "0.19", optional = true }
|
|
56
55
|
memmap2 = "0.9"
|
|
57
56
|
bytemuck = { version = "1.14", features = ["derive"] }
|
|
58
|
-
# tokio is optional now; enable via the 'async' feature
|
|
59
57
|
tokio = { version = "1", features = ["sync", "rt"], optional = true }
|
|
60
58
|
parking_lot = "0.12"
|
|
61
|
-
pollster = { version = "0.3", optional = true }
|
|
62
59
|
libsais = { version = "0.2.0", default-features = false }
|
|
63
60
|
|
|
64
61
|
[features]
|
|
65
|
-
# default is intentionally empty so the crate compiles fast for local checks.
|
|
66
|
-
# Enable 'gpu' to pull in the WGPU and pollster dependencies (heavy).
|
|
67
|
-
# Enable 'async' to include tokio runtime (optional).
|
|
68
|
-
# Example: `cargo build -p roxify_native --features gpu`
|
|
69
62
|
default = []
|
|
70
|
-
|
|
71
|
-
gpu = ["wgpu", "pollster"]
|
|
72
63
|
async = ["tokio"]
|
|
73
|
-
full = ["gpu", "async"]
|
|
74
64
|
|
|
75
65
|
[profile.release]
|
|
76
66
|
opt-level = 3
|
package/README.md
CHANGED
|
@@ -57,151 +57,27 @@ The core compression and image-processing logic is written in Rust and exposed t
|
|
|
57
57
|
|
|
58
58
|
## Benchmarks
|
|
59
59
|
|
|
60
|
-
All measurements
|
|
61
|
-
|
|
62
|
-
### BWT-ANS Native Compression (Rust, via N-API)
|
|
63
|
-
|
|
64
|
-
Direct API calls through the native module (BWT → MTF → RLE0 → rANS, 1 MB blocks):
|
|
65
|
-
|
|
66
|
-
| Dataset | Original | Compressed | Ratio | Encode | Decode | Enc Throughput | Dec Throughput |
|
|
67
|
-
| ------- | -------- | ---------- | ----- | ------ | ------ | -------------- | -------------- |
|
|
68
|
-
| Repetitive text 45 KB | 45 KB | 207 B | 0.5% | < 1 ms | < 1 ms | — | — |
|
|
69
|
-
| Rust source 6 KB | 6 KB | 2.0 KB | 33.7% | < 1 ms | < 1 ms | — | — |
|
|
70
|
-
| node_modules tar 175 MB | 175.5 MB | 38.4 MB | 21.9% | 9.68 s | 5.62 s | 18.1 MB/s | 31.2 MB/s |
|
|
71
|
-
| Random 100 KB | 100 KB | 101 KB | 101.5% | < 1 ms | < 1 ms | — | — |
|
|
72
|
-
| Zeros 100 KB | 100 KB | 51 B | 0.1% | < 1 ms | < 1 ms | — | — |
|
|
73
|
-
|
|
74
|
-
> BWT-ANS achieves **21.9% on real-world node_modules** (175 MB), with **18 MB/s encode** and **31 MB/s decode** throughput. 100% lossless roundtrip verified on all datasets.
|
|
75
|
-
|
|
76
|
-
### Compression Ratio (Maximum Compression for All Tools)
|
|
77
|
-
|
|
78
|
-
| Dataset | Original | zip -9 | gzip -9 | 7z LZMA2 -9 | Roxify PNG | Roxify WAV |
|
|
79
|
-
| ----------- | -------- | --------------- | --------------- | --------------- | ------------------- | ------------------- |
|
|
80
|
-
| Text 1 MB | 1.00 MB | 219 KB (21.4%) | 219 KB (21.4%) | 187 KB (18.3%) | **188 KB (18.3%)** | **187 KB (18.3%)** |
|
|
81
|
-
| JSON 1 MB | 1.00 MB | 263 KB (25.7%) | 263 KB (25.7%) | 225 KB (22.0%) | **220 KB (21.5%)** | **219 KB (21.4%)** |
|
|
82
|
-
| Binary 1 MB | 1.00 MB | 1.00 MB (100%) | 1.00 MB (100%) | 1.00 MB (100%) | 1.00 MB (100%) | 1.00 MB (100%) |
|
|
83
|
-
| Mixed 5 MB | 5.00 MB | 2.45 MB (49.0%) | 2.45 MB (49.1%) | 2.33 MB (46.6%) | 2.38 MB (47.6%) | 2.38 MB (47.6%) |
|
|
84
|
-
| Text 10 MB | 10.00 MB | 2.13 MB (21.3%) | 2.13 MB (21.3%) | 1.71 MB (17.1%) | **1.71 MB (17.1%)** | **1.70 MB (17.0%)** |
|
|
85
|
-
| Mixed 10 MB | 10.00 MB | 4.90 MB (49.0%) | 4.90 MB (49.0%) | 4.65 MB (46.5%) | 4.73 MB (47.3%) | 4.73 MB (47.3%) |
|
|
86
|
-
|
|
87
|
-
> **Roxify matches 7z LZMA2 ultra-compression on text** (18.3% for both at 1 MB) and **beats LZMA2 on JSON** (21.4% vs 22.0%). On mixed data, Roxify is within 1 percentage point of LZMA2 while producing a shareable PNG/WAV instead of an archive.
|
|
88
|
-
|
|
89
|
-
### Encode and Decode Speed (CLI)
|
|
90
|
-
|
|
91
|
-
| Dataset | Tool | Encode | Decode | Enc Throughput | Dec Throughput |
|
|
92
|
-
| ---------- | -------------- | ---------- | ---------- | -------------- | -------------- |
|
|
93
|
-
| Text 1 MB | zip -9 | 112 ms | 36 ms | 8.9 MB/s | 27.6 MB/s |
|
|
94
|
-
| | gzip -9 | 146 ms | 38 ms | 6.9 MB/s | 26.0 MB/s |
|
|
95
|
-
| | 7z LZMA -9 | 303 ms | 21 ms | 3.3 MB/s | 46.6 MB/s |
|
|
96
|
-
| | **Roxify PNG** | **859 ms** | **577 ms** | **1.2 MB/s** | **1.7 MB/s** |
|
|
97
|
-
| | **Roxify WAV** | **794 ms** | **480 ms** | **1.3 MB/s** | **2.1 MB/s** |
|
|
98
|
-
| JSON 1 MB | zip -9 | 79 ms | 20 ms | 12.7 MB/s | 50.5 MB/s |
|
|
99
|
-
| | 7z LZMA -9 | 197 ms | 26 ms | 5.1 MB/s | 37.9 MB/s |
|
|
100
|
-
| | **Roxify PNG** | **1.14 s** | **755 ms** | **0.9 MB/s** | **1.3 MB/s** |
|
|
101
|
-
| | **Roxify WAV** | **1.49 s** | **518 ms** | **0.7 MB/s** | **1.9 MB/s** |
|
|
102
|
-
| Text 10 MB | zip -9 | 1.21 s | 70 ms | 8.2 MB/s | 143.8 MB/s |
|
|
103
|
-
| | 7z LZMA -9 | 5.05 s | 99 ms | 2.0 MB/s | 100.8 MB/s |
|
|
104
|
-
| | **Roxify PNG** | **9.05 s** | **4.53 s** | **1.1 MB/s** | **2.2 MB/s** |
|
|
105
|
-
| | **Roxify WAV** | **9.22 s** | **2.59 s** | **1.1 MB/s** | **3.9 MB/s** |
|
|
106
|
-
|
|
107
|
-
> Roxify CLI includes Node.js startup overhead (~400 ms). In the JS API (below), the same operations are significantly faster. WAV decode is consistently faster than PNG decode due to simpler container parsing.
|
|
108
|
-
|
|
109
|
-
### JavaScript API Throughput
|
|
110
|
-
|
|
111
|
-
Direct API calls (no CLI startup overhead):
|
|
112
|
-
|
|
113
|
-
| Size | Container | Encode | Decode | Enc Throughput | Dec Throughput | Output | Ratio | Integrity |
|
|
114
|
-
| ------ | --------- | ------ | ------- | -------------- | -------------- | --------- | ------ | --------- |
|
|
115
|
-
| 1 KB | PNG | 9 ms | 12 ms | 0.1 MB/s | 0.1 MB/s | 1.14 KB | 114.3% | ✓ |
|
|
116
|
-
| 10 KB | PNG | 18 ms | 34 ms | 0.5 MB/s | 0.3 MB/s | 10.32 KB | 103.2% | ✓ |
|
|
117
|
-
| 100 KB | PNG | 52 ms | 109 ms | 1.9 MB/s | 0.9 MB/s | 100.52 KB | 100.5% | ✓ |
|
|
118
|
-
| 500 KB | PNG | 339 ms | 541 ms | 1.4 MB/s | 0.9 MB/s | 502.64 KB | 100.5% | ✓ |
|
|
119
|
-
| 1 MB | PNG | 875 ms | 1.24 s | 1.1 MB/s | 0.8 MB/s | 1.00 MB | 100.3% | ✓ |
|
|
120
|
-
| 5 MB | PNG | 3.39 s | 4.12 s | 1.5 MB/s | 1.2 MB/s | 5.01 MB | 100.2% | ✓ |
|
|
121
|
-
| 10 MB | PNG | 6.84 s | 12.28 s | 1.5 MB/s | 0.8 MB/s | 10.01 MB | 100.1% | ✓ |
|
|
122
|
-
| 1 KB | WAV | 2 ms | 2 ms | 0.6 MB/s | 0.6 MB/s | 1.08 KB | 107.5% | ✓ |
|
|
123
|
-
| 10 KB | WAV | 4 ms | 5 ms | 2.3 MB/s | 1.8 MB/s | 10.08 KB | 100.8% | ✓ |
|
|
124
|
-
| 100 KB | WAV | 39 ms | 28 ms | 2.5 MB/s | 3.5 MB/s | 100.08 KB | 100.1% | ✓ |
|
|
125
|
-
| 500 KB | WAV | 172 ms | 190 ms | 2.8 MB/s | 2.6 MB/s | 500.09 KB | 100.0% | ✓ |
|
|
126
|
-
| 1 MB | WAV | 452 ms | 276 ms | 2.2 MB/s | 3.6 MB/s | 1.00 MB | 100.0% | ✓ |
|
|
127
|
-
| 5 MB | WAV | 2.70 s | 1.65 s | 1.8 MB/s | 3.0 MB/s | 5.00 MB | 100.0% | ✓ |
|
|
128
|
-
| 10 MB | WAV | 4.81 s | 2.56 s | 2.1 MB/s | 3.9 MB/s | 10.00 MB | 100.0% | ✓ |
|
|
129
|
-
|
|
130
|
-
> WAV container is **2–4× faster** than PNG for decoding at large sizes, and produces slightly smaller output thanks to simpler framing.
|
|
131
|
-
|
|
132
|
-
### Reed-Solomon ECC Throughput
|
|
133
|
-
|
|
134
|
-
| Size | Encode | Decode | Enc Throughput | Dec Throughput | Overhead |
|
|
135
|
-
| ------ | ------ | ------ | -------------- | -------------- | -------- |
|
|
136
|
-
| 1 KB | 6 ms | 4 ms | 0.2 MB/s | 0.2 MB/s | 125.7% |
|
|
137
|
-
| 10 KB | 7 ms | 6 ms | 1.3 MB/s | 1.5 MB/s | 119.6% |
|
|
138
|
-
| 100 KB | 49 ms | 45 ms | 2.0 MB/s | 2.1 MB/s | 118.8% |
|
|
139
|
-
| 1 MB | 483 ms | 377 ms | 2.1 MB/s | 2.7 MB/s | 118.6% |
|
|
140
|
-
|
|
141
|
-
### Lossy-Resilient Encoding
|
|
142
|
-
|
|
143
|
-
#### Robust Image (QR-code-style, block size 4×4)
|
|
144
|
-
|
|
145
|
-
| Data Size | Encode Time | Output (PNG) |
|
|
146
|
-
| --------- | ----------- | ------------ |
|
|
147
|
-
| 32 B | 32 ms | 122 KB |
|
|
148
|
-
| 128 B | 39 ms | 122 KB |
|
|
149
|
-
| 512 B | 76 ms | 316 KB |
|
|
150
|
-
| 1 KB | 139 ms | 508 KB |
|
|
151
|
-
| 2 KB | 251 ms | 986 KB |
|
|
152
|
-
|
|
153
|
-
#### Robust Audio (MFSK 8-channel, medium ECC)
|
|
154
|
-
|
|
155
|
-
| Data Size | Encode | Decode | Output (WAV) | Integrity |
|
|
156
|
-
| --------- | ------ | ------ | ------------ | --------- |
|
|
157
|
-
| 10 B | 33 ms | 44 ms | 1.35 MB | ✓ |
|
|
158
|
-
| 32 B | 19 ms | 31 ms | 1.35 MB | ✓ |
|
|
159
|
-
| 64 B | 22 ms | 24 ms | 1.35 MB | ✓ |
|
|
160
|
-
| 128 B | 21 ms | 28 ms | 1.35 MB | ✓ |
|
|
161
|
-
| 256 B | 40 ms | 45 ms | 2.59 MB | ✓ |
|
|
162
|
-
|
|
163
|
-
### Data Integrity Verification
|
|
164
|
-
|
|
165
|
-
All encode/decode roundtrips produce bit-exact output, verified by SHA-256:
|
|
166
|
-
|
|
167
|
-
| Test Case | PNG | WAV |
|
|
168
|
-
| ----------------------- | --- | --- |
|
|
169
|
-
| Empty buffer (0 B) | ✓ | ✓ |
|
|
170
|
-
| Single byte (1 B) | ✓ | ✓ |
|
|
171
|
-
| All byte values (256 B) | ✓ | ✓ |
|
|
172
|
-
| 1 KB text | ✓ | ✓ |
|
|
173
|
-
| 100 KB random | ✓ | ✓ |
|
|
174
|
-
| 1 MB random | ✓ | ✓ |
|
|
175
|
-
| 5 MB random | ✓ | ✓ |
|
|
176
|
-
|
|
177
|
-
**14 / 14 integrity tests passed** across both containers.
|
|
178
|
-
|
|
179
|
-
### Key Observations
|
|
180
|
-
|
|
181
|
-
- **Roxify matches LZMA2 ultra-compression** on text data (18.3%) and **outperforms it on JSON** (21.4% vs 22.0%), while producing a standard PNG or WAV file instead of an archive.
|
|
182
|
-
- **WAV container decode is 2–4× faster** than PNG decode at large sizes (3.9 MB/s vs 0.8 MB/s for 10 MB).
|
|
183
|
-
- **WAV encode for 1 KB data completes in 2 ms** — well under the sub-second target.
|
|
184
|
-
- **Lossy-resilient audio** encode/decode completes in under 50 ms for data up to 256 bytes, with full integrity.
|
|
185
|
-
- **100% data integrity** across all sizes and containers — every byte is recovered exactly.
|
|
186
|
-
- The CLI overhead (~400 ms Node.js startup) is amortized on larger inputs. For programmatic use, the JS API eliminates this entirely.
|
|
187
|
-
- On incompressible (random) data, all tools converge to ~100% as expected. No compression algorithm can shrink truly random data.
|
|
188
|
-
|
|
189
|
-
### Methodology
|
|
190
|
-
|
|
191
|
-
Benchmarks were generated using `test/benchmark-detailed.cjs`. Datasets consist of procedurally generated text, JSON, and random binary data. Each tool was invoked with its maximum compression setting:
|
|
192
|
-
|
|
193
|
-
| Tool | Command / Setting |
|
|
194
|
-
| -------- | --------------------------- |
|
|
195
|
-
| zip | `zip -r -q -9` |
|
|
196
|
-
| tar/gzip | `tar -cf - \| gzip -9` |
|
|
197
|
-
| 7z | `7z a -mx=9` (LZMA2 ultra) |
|
|
198
|
-
| Roxify | Zstd level 19, compact mode |
|
|
199
|
-
|
|
200
|
-
To reproduce:
|
|
60
|
+
All measurements taken on Linux x64 (Intel i7-6700K @ 4.0 GHz, 32 GB RAM). Roxify uses its native Rust CLI (`roxify_native`) with streaming Zstd L3 + multi-threaded + LDM + window_log(30). ZIP uses `zip -r -q -9` (maximum compression).
|
|
201
61
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
62
|
+
### Real-world directory encoding: Roxify vs ZIP
|
|
63
|
+
|
|
64
|
+
| Dataset | Original | ZIP -9 | Roxify PNG | ZIP time | Roxify time | Speedup |
|
|
65
|
+
| --- | --- | --- | --- | --- | --- | --- |
|
|
66
|
+
| Test A (19 638 files, 177 MB) | 177 MB | 87.7 MB (49.6%) | 54.9 MB (31.0%) | 17.6 s | 1.2 s | 14.7x |
|
|
67
|
+
| Test B (3 936 files, 1.4 GB) | 1.4 GB | 513 MB (36.7%) | 409 MB (29.2%) | 1 min 46 s | 6.7 s | 15.9x |
|
|
68
|
+
|
|
69
|
+
### Decompression
|
|
70
|
+
|
|
71
|
+
| Dataset | unzip | Roxify decode | Speedup |
|
|
72
|
+
| --- | --- | --- | --- |
|
|
73
|
+
| Test A (177 MB) | 2.4 s | 1.8 s | 1.3x |
|
|
74
|
+
| Test B (1.4 GB) | 8.4 s | 2.2 s | 3.8x |
|
|
75
|
+
|
|
76
|
+
Roxify produces a valid PNG image instead of a ZIP archive. On these real-world datasets it compresses 20-37% smaller than ZIP -9 while encoding 15x faster, thanks to multi-threaded Zstd with long-distance matching.
|
|
77
|
+
|
|
78
|
+
### Data integrity
|
|
79
|
+
|
|
80
|
+
100% lossless roundtrip verified by byte-exact diff on all datasets. Start and end markers verified in every output PNG.
|
|
205
81
|
|
|
206
82
|
---
|
|
207
83
|
|
package/dist/cli.js
CHANGED
|
@@ -2,11 +2,25 @@
|
|
|
2
2
|
import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync, } from 'fs';
|
|
3
3
|
import { open } from 'fs/promises';
|
|
4
4
|
import { basename, dirname, join, resolve } from 'path';
|
|
5
|
-
import { DataFormatError, decodePngToBinary, encodeBinaryToPng, hasPassphraseInPng, IncorrectPassphraseError, listFilesInPng, PassphraseRequiredError, } from './index.js';
|
|
6
|
-
import { packPathsGenerator, unpackBuffer } from './pack.js';
|
|
7
5
|
import * as cliProgress from './stub-progress.js';
|
|
8
|
-
import { encodeWithRustCLI, isRustBinaryAvailable, } from './utils/rust-cli-wrapper.js';
|
|
9
|
-
|
|
6
|
+
import { decodeWithRustCLI, encodeWithRustCLI, havepassphraseWithRustCLI, isRustBinaryAvailable, listWithRustCLI, } from './utils/rust-cli-wrapper.js';
|
|
7
|
+
async function loadJsEngine() {
|
|
8
|
+
const indexMod = await import('./index.js');
|
|
9
|
+
const packMod = await import('./pack.js');
|
|
10
|
+
return {
|
|
11
|
+
decodePngToBinary: indexMod.decodePngToBinary,
|
|
12
|
+
encodeBinaryToPng: indexMod.encodeBinaryToPng,
|
|
13
|
+
hasPassphraseInPng: indexMod.hasPassphraseInPng,
|
|
14
|
+
listFilesInPng: indexMod.listFilesInPng,
|
|
15
|
+
DataFormatError: indexMod.DataFormatError,
|
|
16
|
+
IncorrectPassphraseError: indexMod.IncorrectPassphraseError,
|
|
17
|
+
PassphraseRequiredError: indexMod.PassphraseRequiredError,
|
|
18
|
+
packPathsGenerator: packMod.packPathsGenerator,
|
|
19
|
+
unpackBuffer: packMod.unpackBuffer,
|
|
20
|
+
VFSIndexEntry: undefined,
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
const VERSION = '1.13.2';
|
|
10
24
|
function getDirectorySize(dirPath) {
|
|
11
25
|
let totalSize = 0;
|
|
12
26
|
try {
|
|
@@ -280,8 +294,9 @@ async function encodeCommand(args) {
|
|
|
280
294
|
return false;
|
|
281
295
|
}
|
|
282
296
|
});
|
|
283
|
-
if (anyDir) {
|
|
284
|
-
const
|
|
297
|
+
if (anyDir && !isRustBinaryAvailable()) {
|
|
298
|
+
const js = await loadJsEngine();
|
|
299
|
+
const { index } = await js.packPathsGenerator(inputPaths, undefined, () => { });
|
|
285
300
|
if (!index || index.length === 0) {
|
|
286
301
|
console.log(' ');
|
|
287
302
|
console.error('Error: No files found in specified input paths.');
|
|
@@ -357,6 +372,7 @@ async function encodeCommand(args) {
|
|
|
357
372
|
}
|
|
358
373
|
}
|
|
359
374
|
try {
|
|
375
|
+
const js = await loadJsEngine();
|
|
360
376
|
const encodeBar = new cliProgress.SingleBar({
|
|
361
377
|
format: ' {bar} {percentage}% | {step} | {elapsed}s',
|
|
362
378
|
}, cliProgress.Presets.shades_classic);
|
|
@@ -428,7 +444,7 @@ async function encodeCommand(args) {
|
|
|
428
444
|
};
|
|
429
445
|
if (inputPaths.length > 1) {
|
|
430
446
|
currentEncodeStep = 'Reading files';
|
|
431
|
-
const { index, stream, totalSize } = await packPathsGenerator(inputPaths, undefined, onProgress);
|
|
447
|
+
const { index, stream, totalSize } = await js.packPathsGenerator(inputPaths, undefined, onProgress);
|
|
432
448
|
if (!index || index.length === 0) {
|
|
433
449
|
console.log(' ');
|
|
434
450
|
console.error('Error: No files found in specified input paths.');
|
|
@@ -448,7 +464,7 @@ async function encodeCommand(args) {
|
|
|
448
464
|
const st = statSync(resolvedInput);
|
|
449
465
|
if (st.isDirectory()) {
|
|
450
466
|
currentEncodeStep = 'Reading files';
|
|
451
|
-
const { index, stream, totalSize } = await packPathsGenerator([resolvedInput], dirname(resolvedInput), onProgress);
|
|
467
|
+
const { index, stream, totalSize } = await js.packPathsGenerator([resolvedInput], dirname(resolvedInput), onProgress);
|
|
452
468
|
if (!index || index.length === 0) {
|
|
453
469
|
console.log(' ');
|
|
454
470
|
console.error(`Error: No files found in ${resolvedInput}`);
|
|
@@ -535,7 +551,7 @@ async function encodeCommand(args) {
|
|
|
535
551
|
else {
|
|
536
552
|
inputBuffer = inputData;
|
|
537
553
|
}
|
|
538
|
-
const output = await encodeBinaryToPng(inputBuffer, options);
|
|
554
|
+
const output = await js.encodeBinaryToPng(inputBuffer, options);
|
|
539
555
|
const encodeTime = Date.now() - startEncode;
|
|
540
556
|
clearInterval(encodeHeartbeat);
|
|
541
557
|
if (barStarted) {
|
|
@@ -574,7 +590,38 @@ async function decodeCommand(args) {
|
|
|
574
590
|
process.exit(1);
|
|
575
591
|
}
|
|
576
592
|
const resolvedInput = resolve(inputPath);
|
|
577
|
-
const resolvedOutput = parsed.output || outputPath || '
|
|
593
|
+
const resolvedOutput = parsed.output || outputPath || '.';
|
|
594
|
+
if (isRustBinaryAvailable() && !parsed.forceTs && !parsed.lossyResilient) {
|
|
595
|
+
try {
|
|
596
|
+
console.log(' ');
|
|
597
|
+
console.log('Decoding... (Using native Rust decoder)\n');
|
|
598
|
+
const startTime = Date.now();
|
|
599
|
+
const decodeBar = new cliProgress.SingleBar({ format: ' {bar} {percentage}% | {step} | {elapsed}s' }, cliProgress.Presets.shades_classic);
|
|
600
|
+
let barValue = 0;
|
|
601
|
+
decodeBar.start(100, 0, { step: 'Decoding', elapsed: '0' });
|
|
602
|
+
const progressInterval = setInterval(() => {
|
|
603
|
+
barValue = Math.min(barValue + 2, 99);
|
|
604
|
+
decodeBar.update(barValue, {
|
|
605
|
+
step: 'Decoding',
|
|
606
|
+
elapsed: String(Math.floor((Date.now() - startTime) / 1000)),
|
|
607
|
+
});
|
|
608
|
+
}, 300);
|
|
609
|
+
await decodeWithRustCLI(resolvedInput, resolvedOutput, parsed.passphrase, parsed.files, parsed.dict);
|
|
610
|
+
clearInterval(progressInterval);
|
|
611
|
+
const decodeTime = Date.now() - startTime;
|
|
612
|
+
decodeBar.update(100, { step: 'done', elapsed: String(Math.floor(decodeTime / 1000)) });
|
|
613
|
+
decodeBar.stop();
|
|
614
|
+
console.log(`\nSuccess!`);
|
|
615
|
+
console.log(` Time: ${decodeTime}ms`);
|
|
616
|
+
console.log(` Output: ${resolve(resolvedOutput)}`);
|
|
617
|
+
console.log(' ');
|
|
618
|
+
return;
|
|
619
|
+
}
|
|
620
|
+
catch (err) {
|
|
621
|
+
console.warn('\nRust decoder failed, falling back to TypeScript decoder...');
|
|
622
|
+
console.warn(`Reason: ${err.message}\n`);
|
|
623
|
+
}
|
|
624
|
+
}
|
|
578
625
|
try {
|
|
579
626
|
const options = {};
|
|
580
627
|
if (parsed.passphrase) {
|
|
@@ -645,7 +692,8 @@ async function decodeCommand(args) {
|
|
|
645
692
|
}
|
|
646
693
|
};
|
|
647
694
|
const inputBuffer = await readLargeFile(resolvedInput);
|
|
648
|
-
const
|
|
695
|
+
const js = await loadJsEngine();
|
|
696
|
+
const result = await js.decodePngToBinary(inputBuffer, options);
|
|
649
697
|
const decodeTime = Date.now() - startDecode;
|
|
650
698
|
clearInterval(heartbeat);
|
|
651
699
|
if (barStarted) {
|
|
@@ -684,7 +732,7 @@ async function decodeCommand(args) {
|
|
|
684
732
|
console.log(`Time: ${decodeTime}ms`);
|
|
685
733
|
}
|
|
686
734
|
else if (result.buf) {
|
|
687
|
-
const unpacked = unpackBuffer(result.buf);
|
|
735
|
+
const unpacked = js.unpackBuffer(result.buf);
|
|
688
736
|
if (unpacked) {
|
|
689
737
|
const baseDir = parsed.output || outputPath || '.';
|
|
690
738
|
for (const file of unpacked.files) {
|
|
@@ -720,17 +768,17 @@ async function decodeCommand(args) {
|
|
|
720
768
|
console.log(' ');
|
|
721
769
|
}
|
|
722
770
|
catch (err) {
|
|
723
|
-
if (err
|
|
771
|
+
if ((err.message && err.message.includes('passphrase required')) ||
|
|
724
772
|
(err.message && err.message.includes('passphrase') && !parsed.passphrase)) {
|
|
725
773
|
console.log(' ');
|
|
726
774
|
console.error('File appears to be encrypted. Provide a passphrase with -p');
|
|
727
775
|
}
|
|
728
|
-
else if (err
|
|
776
|
+
else if ((err.message && err.message.includes('Incorrect passphrase')) ||
|
|
729
777
|
(err.message && err.message.includes('Incorrect passphrase'))) {
|
|
730
778
|
console.log(' ');
|
|
731
779
|
console.error('Incorrect passphrase');
|
|
732
780
|
}
|
|
733
|
-
else if (err
|
|
781
|
+
else if ((err.message && err.message.includes('data format error')) ||
|
|
734
782
|
(err.message &&
|
|
735
783
|
(err.message.includes('decompression failed') ||
|
|
736
784
|
err.message.includes('missing ROX1') ||
|
|
@@ -761,40 +809,25 @@ async function listCommand(args) {
|
|
|
761
809
|
const resolvedInput = resolve(inputPath);
|
|
762
810
|
if (isRustBinaryAvailable()) {
|
|
763
811
|
try {
|
|
764
|
-
const
|
|
765
|
-
const
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
}
|
|
773
|
-
const output = execSync(`"${cliPath}" list "${resolvedInput}"`, {
|
|
774
|
-
encoding: 'utf-8',
|
|
775
|
-
stdio: ['pipe', 'pipe', 'inherit'],
|
|
776
|
-
timeout: 30000,
|
|
777
|
-
});
|
|
778
|
-
const fileList = JSON.parse(output.trim());
|
|
779
|
-
console.log(`Files in ${resolvedInput}:`);
|
|
780
|
-
for (const file of fileList) {
|
|
781
|
-
if (typeof file === 'string') {
|
|
782
|
-
console.log(` ${file}`);
|
|
783
|
-
}
|
|
784
|
-
else {
|
|
785
|
-
console.log(` ${file.name} (${file.size} bytes)`);
|
|
786
|
-
}
|
|
787
|
-
}
|
|
788
|
-
return;
|
|
812
|
+
const output = await listWithRustCLI(resolvedInput);
|
|
813
|
+
const fileList = JSON.parse(output.trim());
|
|
814
|
+
console.log(`Files in ${resolvedInput}:`);
|
|
815
|
+
for (const file of fileList) {
|
|
816
|
+
if (typeof file === 'string') {
|
|
817
|
+
console.log(` ${file}`);
|
|
818
|
+
}
|
|
819
|
+
else {
|
|
820
|
+
console.log(` ${file.name} (${file.size} bytes)`);
|
|
789
821
|
}
|
|
790
|
-
catch (e) { }
|
|
791
822
|
}
|
|
823
|
+
return;
|
|
792
824
|
}
|
|
793
|
-
catch (
|
|
825
|
+
catch (e) { }
|
|
794
826
|
}
|
|
795
827
|
try {
|
|
796
828
|
const inputBuffer = readFileSync(resolvedInput);
|
|
797
|
-
const
|
|
829
|
+
const js = await loadJsEngine();
|
|
830
|
+
const fileList = await js.listFilesInPng(inputBuffer, {
|
|
798
831
|
includeSizes: parsed.sizes !== false,
|
|
799
832
|
});
|
|
800
833
|
if (fileList) {
|
|
@@ -831,9 +864,18 @@ async function havePassphraseCommand(args) {
|
|
|
831
864
|
process.exit(1);
|
|
832
865
|
}
|
|
833
866
|
const resolvedInput = resolve(inputPath);
|
|
867
|
+
if (isRustBinaryAvailable()) {
|
|
868
|
+
try {
|
|
869
|
+
const output = await havepassphraseWithRustCLI(resolvedInput);
|
|
870
|
+
console.log(output.trim());
|
|
871
|
+
return;
|
|
872
|
+
}
|
|
873
|
+
catch (e) { }
|
|
874
|
+
}
|
|
834
875
|
try {
|
|
835
876
|
const inputBuffer = readFileSync(resolvedInput);
|
|
836
|
-
const
|
|
877
|
+
const js = await loadJsEngine();
|
|
878
|
+
const has = await js.hasPassphraseInPng(inputBuffer);
|
|
837
879
|
console.log(has ? 'Passphrase detected.' : 'No passphrase detected.');
|
|
838
880
|
}
|
|
839
881
|
catch (err) {
|
package/dist/index.d.ts
CHANGED
|
@@ -12,7 +12,7 @@ export * from './utils/optimization.js';
|
|
|
12
12
|
export * from './utils/reconstitution.js';
|
|
13
13
|
export * from './utils/robust-audio.js';
|
|
14
14
|
export * from './utils/robust-image.js';
|
|
15
|
-
export { encodeWithRustCLI, isRustBinaryAvailable } from './utils/rust-cli-wrapper.js';
|
|
15
|
+
export { decodeWithRustCLI, encodeWithRustCLI, havepassphraseWithRustCLI, isRustBinaryAvailable, listWithRustCLI } from './utils/rust-cli-wrapper.js';
|
|
16
16
|
export * from './utils/types.js';
|
|
17
17
|
export * from './utils/zstd.js';
|
|
18
18
|
export { packPaths, packPathsToParts, unpackBuffer } from './pack.js';
|
package/dist/index.js
CHANGED
|
@@ -12,7 +12,7 @@ export * from './utils/optimization.js';
|
|
|
12
12
|
export * from './utils/reconstitution.js';
|
|
13
13
|
export * from './utils/robust-audio.js';
|
|
14
14
|
export * from './utils/robust-image.js';
|
|
15
|
-
export { encodeWithRustCLI, isRustBinaryAvailable } from './utils/rust-cli-wrapper.js';
|
|
15
|
+
export { decodeWithRustCLI, encodeWithRustCLI, havepassphraseWithRustCLI, isRustBinaryAvailable, listWithRustCLI } from './utils/rust-cli-wrapper.js';
|
|
16
16
|
export * from './utils/types.js';
|
|
17
17
|
export * from './utils/zstd.js';
|
|
18
18
|
export { packPaths, packPathsToParts, unpackBuffer } from './pack.js';
|
package/dist/rox-macos-universal
CHANGED
|
Binary file
|
package/dist/roxify_native
CHANGED
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/dist/roxify_native.exe
CHANGED
|
Binary file
|
|
@@ -2,3 +2,6 @@ declare function findRustBinary(): string | null;
|
|
|
2
2
|
export { findRustBinary };
|
|
3
3
|
export declare function isRustBinaryAvailable(): boolean;
|
|
4
4
|
export declare function encodeWithRustCLI(inputPath: string, outputPath: string, compressionLevel?: number, passphrase?: string, encryptType?: 'aes' | 'xor', name?: string): Promise<void>;
|
|
5
|
+
export declare function decodeWithRustCLI(inputPath: string, outputPath: string, passphrase?: string, files?: string[], dict?: string): Promise<void>;
|
|
6
|
+
export declare function listWithRustCLI(inputPath: string): Promise<string>;
|
|
7
|
+
export declare function havepassphraseWithRustCLI(inputPath: string): Promise<string>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { execSync, spawn } from 'child_process';
|
|
2
|
-
import { existsSync } from 'fs';
|
|
2
|
+
import { accessSync, constants, existsSync } from 'fs';
|
|
3
3
|
import { dirname, join } from 'path';
|
|
4
4
|
import { fileURLToPath } from 'url';
|
|
5
5
|
let moduleDir;
|
|
@@ -14,6 +14,19 @@ else {
|
|
|
14
14
|
moduleDir = process.cwd();
|
|
15
15
|
}
|
|
16
16
|
}
|
|
17
|
+
function canExecute(p) {
|
|
18
|
+
if (!existsSync(p))
|
|
19
|
+
return false;
|
|
20
|
+
if (process.platform === 'win32')
|
|
21
|
+
return true;
|
|
22
|
+
try {
|
|
23
|
+
accessSync(p, constants.X_OK);
|
|
24
|
+
return true;
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
17
30
|
function findRustBinary() {
|
|
18
31
|
const platformBins = {
|
|
19
32
|
win32: ['roxify_native.exe', 'roxify-cli.exe', 'roxify_cli.exe'],
|
|
@@ -24,13 +37,13 @@ function findRustBinary() {
|
|
|
24
37
|
const baseDir = moduleDir;
|
|
25
38
|
for (const name of binNames) {
|
|
26
39
|
const sameDirPath = join(baseDir, name);
|
|
27
|
-
if (
|
|
40
|
+
if (canExecute(sameDirPath))
|
|
28
41
|
return sameDirPath;
|
|
29
42
|
const parentPath = join(baseDir, '..', name);
|
|
30
|
-
if (
|
|
43
|
+
if (canExecute(parentPath))
|
|
31
44
|
return parentPath;
|
|
32
45
|
const parentDistPath = join(baseDir, '..', 'dist', name);
|
|
33
|
-
if (
|
|
46
|
+
if (canExecute(parentDistPath))
|
|
34
47
|
return parentDistPath;
|
|
35
48
|
}
|
|
36
49
|
if (process.pkg) {
|
|
@@ -42,7 +55,7 @@ function findRustBinary() {
|
|
|
42
55
|
for (const basePath of snapshotPaths) {
|
|
43
56
|
for (const name of binNames) {
|
|
44
57
|
const binPath = join(basePath, name);
|
|
45
|
-
if (
|
|
58
|
+
if (canExecute(binPath))
|
|
46
59
|
return binPath;
|
|
47
60
|
}
|
|
48
61
|
}
|
|
@@ -58,7 +71,7 @@ function findRustBinary() {
|
|
|
58
71
|
for (const c of execCandidates) {
|
|
59
72
|
for (const name of binNames) {
|
|
60
73
|
const p = join(c, name);
|
|
61
|
-
if (
|
|
74
|
+
if (canExecute(p))
|
|
62
75
|
return p;
|
|
63
76
|
}
|
|
64
77
|
}
|
|
@@ -97,7 +110,7 @@ function findRustBinary() {
|
|
|
97
110
|
for (const c of candidates) {
|
|
98
111
|
for (const name of binNames) {
|
|
99
112
|
const candidate = join(c, name);
|
|
100
|
-
if (
|
|
113
|
+
if (canExecute(candidate))
|
|
101
114
|
return candidate;
|
|
102
115
|
}
|
|
103
116
|
}
|
|
@@ -108,16 +121,16 @@ function findRustBinary() {
|
|
|
108
121
|
catch { }
|
|
109
122
|
for (const name of binNames) {
|
|
110
123
|
const parentParentLocal = join(baseDir, '..', '..', name);
|
|
111
|
-
if (
|
|
124
|
+
if (canExecute(parentParentLocal))
|
|
112
125
|
return parentParentLocal;
|
|
113
126
|
const nodeModulesPath = join(baseDir, '..', '..', '..', '..', name);
|
|
114
|
-
if (
|
|
127
|
+
if (canExecute(nodeModulesPath))
|
|
115
128
|
return nodeModulesPath;
|
|
116
129
|
}
|
|
117
130
|
const targetRelease = join(baseDir, '..', '..', 'target', 'release');
|
|
118
131
|
for (const name of binNames) {
|
|
119
132
|
const targetPath = join(targetRelease, name);
|
|
120
|
-
if (
|
|
133
|
+
if (canExecute(targetPath))
|
|
121
134
|
return targetPath;
|
|
122
135
|
}
|
|
123
136
|
return null;
|
|
@@ -128,52 +141,32 @@ export function isRustBinaryAvailable() {
|
|
|
128
141
|
}
|
|
129
142
|
import { chmodSync, mkdtempSync, readFileSync, unlinkSync, writeFileSync, } from 'fs';
|
|
130
143
|
import { tmpdir } from 'os';
|
|
131
|
-
|
|
144
|
+
function extractToTemp(pathToRead) {
|
|
145
|
+
const buf = readFileSync(pathToRead);
|
|
146
|
+
const tmp = mkdtempSync(join(tmpdir(), 'roxify-'));
|
|
147
|
+
const dest = join(tmp, pathToRead.replace(/.*[\\/]/, ''));
|
|
148
|
+
writeFileSync(dest, buf);
|
|
149
|
+
try {
|
|
150
|
+
chmodSync(dest, 0o755);
|
|
151
|
+
}
|
|
152
|
+
catch (e) { }
|
|
153
|
+
return dest;
|
|
154
|
+
}
|
|
155
|
+
function spawnRustCLI(args, options) {
|
|
132
156
|
const cliPath = findRustBinary();
|
|
133
|
-
if (!cliPath)
|
|
157
|
+
if (!cliPath)
|
|
134
158
|
throw new Error('Rust CLI binary not found');
|
|
135
|
-
}
|
|
136
|
-
function extractToTemp(pathToRead) {
|
|
137
|
-
const buf = readFileSync(pathToRead);
|
|
138
|
-
const tmp = mkdtempSync(join(tmpdir(), 'roxify-'));
|
|
139
|
-
const dest = join(tmp, pathToRead.replace(/.*[\\/]/, ''));
|
|
140
|
-
writeFileSync(dest, buf);
|
|
141
|
-
try {
|
|
142
|
-
chmodSync(dest, 0o755);
|
|
143
|
-
}
|
|
144
|
-
catch (e) { }
|
|
145
|
-
return dest;
|
|
146
|
-
}
|
|
147
159
|
return new Promise((resolve, reject) => {
|
|
148
|
-
const args = ['encode', '--level', String(compressionLevel)];
|
|
149
|
-
let supportsName = false;
|
|
150
|
-
if (name) {
|
|
151
|
-
try {
|
|
152
|
-
const helpOut = execSync(`"${cliPath}" --help`, {
|
|
153
|
-
encoding: 'utf8',
|
|
154
|
-
timeout: 2000,
|
|
155
|
-
});
|
|
156
|
-
if (helpOut && helpOut.includes('--name'))
|
|
157
|
-
supportsName = true;
|
|
158
|
-
}
|
|
159
|
-
catch (e) {
|
|
160
|
-
supportsName = false;
|
|
161
|
-
}
|
|
162
|
-
if (supportsName) {
|
|
163
|
-
args.push('--name', name);
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
if (passphrase) {
|
|
167
|
-
args.push('--passphrase', passphrase);
|
|
168
|
-
args.push('--encrypt', encryptType);
|
|
169
|
-
}
|
|
170
|
-
args.push(inputPath, outputPath);
|
|
171
160
|
let triedExtract = false;
|
|
172
161
|
let tempExe;
|
|
162
|
+
let stdout = '';
|
|
173
163
|
const runSpawn = (exePath) => {
|
|
174
164
|
let proc;
|
|
165
|
+
const stdio = options?.collectStdout
|
|
166
|
+
? ['pipe', 'pipe', 'inherit']
|
|
167
|
+
: 'inherit';
|
|
175
168
|
try {
|
|
176
|
-
proc = spawn(exePath, args, { stdio
|
|
169
|
+
proc = spawn(exePath, args, { stdio });
|
|
177
170
|
}
|
|
178
171
|
catch (err) {
|
|
179
172
|
if (!triedExtract) {
|
|
@@ -188,6 +181,9 @@ export async function encodeWithRustCLI(inputPath, outputPath, compressionLevel
|
|
|
188
181
|
}
|
|
189
182
|
return reject(err);
|
|
190
183
|
}
|
|
184
|
+
if (options?.collectStdout && proc.stdout) {
|
|
185
|
+
proc.stdout.on('data', (chunk) => { stdout += chunk.toString(); });
|
|
186
|
+
}
|
|
191
187
|
proc.on('error', (err) => {
|
|
192
188
|
if (!triedExtract) {
|
|
193
189
|
triedExtract = true;
|
|
@@ -201,21 +197,55 @@ export async function encodeWithRustCLI(inputPath, outputPath, compressionLevel
|
|
|
201
197
|
}
|
|
202
198
|
reject(err);
|
|
203
199
|
});
|
|
204
|
-
proc.on('close', (code) => {
|
|
200
|
+
proc.on('close', (code, signal) => {
|
|
205
201
|
if (tempExe) {
|
|
206
202
|
try {
|
|
207
203
|
unlinkSync(tempExe);
|
|
208
204
|
}
|
|
209
205
|
catch (e) { }
|
|
210
206
|
}
|
|
211
|
-
if (code === 0)
|
|
212
|
-
resolve();
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
reject(new Error(`Rust encoder exited with status ${code}`));
|
|
216
|
-
}
|
|
207
|
+
if (code === 0 || (code === null && signal === null))
|
|
208
|
+
resolve(stdout);
|
|
209
|
+
else
|
|
210
|
+
reject(new Error(`Rust CLI exited with status ${code ?? signal}`));
|
|
217
211
|
});
|
|
218
212
|
};
|
|
219
213
|
runSpawn(cliPath);
|
|
220
214
|
});
|
|
221
215
|
}
|
|
216
|
+
export async function encodeWithRustCLI(inputPath, outputPath, compressionLevel = 3, passphrase, encryptType = 'aes', name) {
|
|
217
|
+
const cliPath = findRustBinary();
|
|
218
|
+
if (!cliPath)
|
|
219
|
+
throw new Error('Rust CLI binary not found');
|
|
220
|
+
const args = ['encode', '--level', String(compressionLevel)];
|
|
221
|
+
if (name) {
|
|
222
|
+
try {
|
|
223
|
+
const helpOut = execSync(`"${cliPath}" --help`, { encoding: 'utf8', timeout: 2000 });
|
|
224
|
+
if (helpOut && helpOut.includes('--name'))
|
|
225
|
+
args.push('--name', name);
|
|
226
|
+
}
|
|
227
|
+
catch (e) { }
|
|
228
|
+
}
|
|
229
|
+
if (passphrase) {
|
|
230
|
+
args.push('--passphrase', passphrase);
|
|
231
|
+
args.push('--encrypt', encryptType);
|
|
232
|
+
}
|
|
233
|
+
args.push(inputPath, outputPath);
|
|
234
|
+
await spawnRustCLI(args);
|
|
235
|
+
}
|
|
236
|
+
export async function decodeWithRustCLI(inputPath, outputPath, passphrase, files, dict) {
|
|
237
|
+
const args = ['decompress', inputPath, outputPath];
|
|
238
|
+
if (passphrase)
|
|
239
|
+
args.push('--passphrase', passphrase);
|
|
240
|
+
if (files && files.length > 0)
|
|
241
|
+
args.push('--files', JSON.stringify(files));
|
|
242
|
+
if (dict)
|
|
243
|
+
args.push('--dict', dict);
|
|
244
|
+
await spawnRustCLI(args);
|
|
245
|
+
}
|
|
246
|
+
export async function listWithRustCLI(inputPath) {
|
|
247
|
+
return spawnRustCLI(['list', inputPath], { collectStdout: true });
|
|
248
|
+
}
|
|
249
|
+
export async function havepassphraseWithRustCLI(inputPath) {
|
|
250
|
+
return spawnRustCLI(['havepassphrase', inputPath], { collectStdout: true });
|
|
251
|
+
}
|
package/native/bench_hybrid.rs
CHANGED
|
@@ -8,7 +8,7 @@ mod pool;
|
|
|
8
8
|
mod hybrid;
|
|
9
9
|
|
|
10
10
|
fn bench_roundtrip(name: &str, data: &[u8]) {
|
|
11
|
-
let compressor = hybrid::HybridCompressor::new(
|
|
11
|
+
let compressor = hybrid::HybridCompressor::new();
|
|
12
12
|
|
|
13
13
|
let start = Instant::now();
|
|
14
14
|
let (compressed, stats) = compressor.compress(data).unwrap();
|
package/native/hybrid.rs
CHANGED
|
@@ -28,7 +28,7 @@ pub struct HybridCompressor {
|
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
impl HybridCompressor {
|
|
31
|
-
pub fn new(
|
|
31
|
+
pub fn new() -> Self {
|
|
32
32
|
HybridCompressor {
|
|
33
33
|
block_size: BLOCK_SIZE,
|
|
34
34
|
}
|
|
@@ -36,7 +36,6 @@ impl HybridCompressor {
|
|
|
36
36
|
|
|
37
37
|
pub fn compress(&self, data: &[u8]) -> Result<(Vec<u8>, CompressionStats)> {
|
|
38
38
|
let original_size = data.len() as u64;
|
|
39
|
-
|
|
40
39
|
let blocks: Vec<&[u8]> = data.chunks(self.block_size).collect();
|
|
41
40
|
let blocks_count = blocks.len();
|
|
42
41
|
|
|
@@ -136,13 +135,11 @@ impl HybridCompressor {
|
|
|
136
135
|
}
|
|
137
136
|
}
|
|
138
137
|
|
|
139
|
-
fn
|
|
138
|
+
fn compress_block_with_entropy(block: &[u8], entropy: f32) -> Result<Vec<u8>> {
|
|
140
139
|
if block.is_empty() {
|
|
141
140
|
return Ok(vec![BLOCK_FLAG_STORE]);
|
|
142
141
|
}
|
|
143
142
|
|
|
144
|
-
let entropy = analyze_entropy(block);
|
|
145
|
-
|
|
146
143
|
if entropy >= ENTROPY_THRESHOLD_STORE {
|
|
147
144
|
let mut result = Vec::with_capacity(1 + block.len());
|
|
148
145
|
result.push(BLOCK_FLAG_STORE);
|
|
@@ -165,6 +162,19 @@ fn compress_block(block: &[u8]) -> Result<Vec<u8>> {
|
|
|
165
162
|
return Ok(result);
|
|
166
163
|
}
|
|
167
164
|
|
|
165
|
+
try_bwt_or_zstd(block)
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
fn compress_block(block: &[u8]) -> Result<Vec<u8>> {
|
|
169
|
+
if block.is_empty() {
|
|
170
|
+
return Ok(vec![BLOCK_FLAG_STORE]);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
let entropy = analyze_entropy(block);
|
|
174
|
+
compress_block_with_entropy(block, entropy)
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
fn try_bwt_or_zstd(block: &[u8]) -> Result<Vec<u8>> {
|
|
168
178
|
let bwt = bwt_encode(block)?;
|
|
169
179
|
let mtf_data = mtf_encode(&bwt.transformed);
|
|
170
180
|
let rle_data = rle0_encode(&mtf_data);
|
|
@@ -277,11 +287,11 @@ fn decompress_block_v1(block: &[u8]) -> Result<Vec<u8>> {
|
|
|
277
287
|
}
|
|
278
288
|
|
|
279
289
|
pub fn compress_high_performance(data: &[u8]) -> Result<(Vec<u8>, CompressionStats)> {
|
|
280
|
-
let compressor = HybridCompressor::new(
|
|
290
|
+
let compressor = HybridCompressor::new();
|
|
281
291
|
compressor.compress(data)
|
|
282
292
|
}
|
|
283
293
|
|
|
284
294
|
pub fn decompress_high_performance(data: &[u8]) -> Result<Vec<u8>> {
|
|
285
|
-
let compressor = HybridCompressor::new(
|
|
295
|
+
let compressor = HybridCompressor::new();
|
|
286
296
|
compressor.decompress(data)
|
|
287
297
|
}
|
package/native/lib.rs
CHANGED
|
@@ -6,8 +6,6 @@ use napi_derive::napi;
|
|
|
6
6
|
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
|
7
7
|
|
|
8
8
|
mod core;
|
|
9
|
-
#[cfg(feature = "gpu")]
|
|
10
|
-
mod gpu;
|
|
11
9
|
mod rans;
|
|
12
10
|
mod rans_byte;
|
|
13
11
|
mod bwt;
|
|
@@ -27,15 +25,6 @@ mod archive;
|
|
|
27
25
|
mod streaming;
|
|
28
26
|
|
|
29
27
|
pub use core::*;
|
|
30
|
-
#[cfg(feature = "gpu")]
|
|
31
|
-
pub use gpu::*;
|
|
32
|
-
#[cfg(not(feature = "gpu"))]
|
|
33
|
-
mod gpu {
|
|
34
|
-
pub fn gpu_available() -> bool {
|
|
35
|
-
false
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
|
|
39
28
|
pub use rans::*;
|
|
40
29
|
pub use bwt::*;
|
|
41
30
|
pub use context_mixing::*;
|
|
@@ -57,12 +46,6 @@ pub struct CompressionReport {
|
|
|
57
46
|
pub blocks_count: u32,
|
|
58
47
|
}
|
|
59
48
|
|
|
60
|
-
#[napi(object)]
|
|
61
|
-
pub struct GpuStatus {
|
|
62
|
-
pub available: bool,
|
|
63
|
-
pub adapter_info: Option<String>,
|
|
64
|
-
}
|
|
65
|
-
|
|
66
49
|
#[cfg(not(test))]
|
|
67
50
|
#[napi]
|
|
68
51
|
pub fn scan_pixels(buffer: Buffer, channels: u32, marker_bytes: Option<Buffer>) -> Result<ScanResult> {
|
|
@@ -122,15 +105,6 @@ pub fn native_zstd_decompress_with_dict(buffer: Buffer, dict: Buffer) -> Result<
|
|
|
122
105
|
core::zstd_decompress_bytes(&buffer, Some(dict_slice)).map_err(|e| Error::from_reason(e))
|
|
123
106
|
}
|
|
124
107
|
|
|
125
|
-
#[cfg(not(test))]
|
|
126
|
-
#[napi]
|
|
127
|
-
pub fn check_gpu_status() -> GpuStatus {
|
|
128
|
-
GpuStatus {
|
|
129
|
-
available: gpu::gpu_available(),
|
|
130
|
-
adapter_info: None,
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
|
|
134
108
|
#[cfg(not(test))]
|
|
135
109
|
#[napi]
|
|
136
110
|
pub fn bwt_transform(buffer: Buffer) -> Result<Vec<u8>> {
|
package/native/main.rs
CHANGED
|
@@ -17,6 +17,7 @@ mod archive;
|
|
|
17
17
|
mod streaming;
|
|
18
18
|
mod streaming_decode;
|
|
19
19
|
mod streaming_encode;
|
|
20
|
+
mod progress;
|
|
20
21
|
|
|
21
22
|
use crate::encoder::ImageFormat;
|
|
22
23
|
use std::path::PathBuf;
|
|
@@ -157,13 +158,16 @@ fn main() -> anyhow::Result<()> {
|
|
|
157
158
|
.or_else(|| input.file_name().and_then(|n| n.to_str()));
|
|
158
159
|
|
|
159
160
|
if is_dir && dict.is_none() {
|
|
160
|
-
streaming_encode::
|
|
161
|
+
streaming_encode::encode_dir_to_png_encrypted_with_progress(
|
|
161
162
|
&input,
|
|
162
163
|
&output,
|
|
163
164
|
level,
|
|
164
165
|
file_name,
|
|
165
166
|
passphrase.as_deref(),
|
|
166
167
|
Some(&encrypt),
|
|
168
|
+
Some(Box::new(|current, total, step| {
|
|
169
|
+
eprintln!("PROGRESS:{}:{}:{}", current, total, step);
|
|
170
|
+
})),
|
|
167
171
|
)?;
|
|
168
172
|
println!("(TAR archive, rXFL chunk embedded)");
|
|
169
173
|
return Ok(());
|
|
@@ -187,6 +191,7 @@ fn main() -> anyhow::Result<()> {
|
|
|
187
191
|
};
|
|
188
192
|
|
|
189
193
|
let use_streaming = payload.len() > 64 * 1024 * 1024;
|
|
194
|
+
eprintln!("PROGRESS:50:100:encoding");
|
|
190
195
|
|
|
191
196
|
if use_streaming {
|
|
192
197
|
streaming::encode_to_png_file(
|
|
@@ -227,6 +232,7 @@ fn main() -> anyhow::Result<()> {
|
|
|
227
232
|
}
|
|
228
233
|
|
|
229
234
|
if file_list_json.is_some() {
|
|
235
|
+
eprintln!("PROGRESS:100:100:done");
|
|
230
236
|
if is_dir {
|
|
231
237
|
println!("(TAR archive, rXFL chunk embedded)");
|
|
232
238
|
} else {
|
|
@@ -339,8 +345,10 @@ fn main() -> anyhow::Result<()> {
|
|
|
339
345
|
|
|
340
346
|
if is_png_file && files.is_none() && dict.is_none() && file_size > 100_000_000 {
|
|
341
347
|
let out_dir = output.clone().unwrap_or_else(|| PathBuf::from("out.raw"));
|
|
348
|
+
eprintln!("PROGRESS:5:100:decoding");
|
|
342
349
|
match streaming_decode::streaming_decode_to_dir_encrypted(&input, &out_dir, passphrase.as_deref()) {
|
|
343
350
|
Ok(written) => {
|
|
351
|
+
eprintln!("PROGRESS:100:100:done");
|
|
344
352
|
println!("Unpacked {} files (TAR)", written.len());
|
|
345
353
|
return Ok(());
|
|
346
354
|
}
|
|
@@ -351,6 +359,7 @@ fn main() -> anyhow::Result<()> {
|
|
|
351
359
|
}
|
|
352
360
|
|
|
353
361
|
let buf = read_all(&input)?;
|
|
362
|
+
eprintln!("PROGRESS:20:100:decompressing");
|
|
354
363
|
let dict_bytes: Option<Vec<u8>> = match dict {
|
|
355
364
|
Some(path) => Some(read_all(&path)?),
|
|
356
365
|
None => None,
|
|
@@ -413,6 +422,7 @@ fn main() -> anyhow::Result<()> {
|
|
|
413
422
|
let files_slice = file_list.as_ref().map(|v| v.as_slice());
|
|
414
423
|
|
|
415
424
|
let written = packer::unpack_stream_to_dir(&mut reader, &out_dir, files_slice).map_err(|e| anyhow::anyhow!(e))?;
|
|
425
|
+
eprintln!("PROGRESS:100:100:done");
|
|
416
426
|
println!("Unpacked {} files", written.len());
|
|
417
427
|
} else {
|
|
418
428
|
let is_png = buf.len() >= 8 && &buf[0..8] == &[137, 80, 78, 71, 13, 10, 26, 10];
|
|
@@ -511,6 +521,7 @@ fn main() -> anyhow::Result<()> {
|
|
|
511
521
|
} else {
|
|
512
522
|
write_all(&dest, &out_bytes)?;
|
|
513
523
|
}
|
|
524
|
+
eprintln!("PROGRESS:100:100:done");
|
|
514
525
|
}
|
|
515
526
|
}
|
|
516
527
|
Commands::Crc32 { input } => {
|
package/native/progress.rs
CHANGED
|
@@ -1,43 +1,142 @@
|
|
|
1
1
|
use parking_lot::Mutex;
|
|
2
2
|
use std::sync::Arc;
|
|
3
|
+
use std::time::Instant;
|
|
3
4
|
|
|
4
|
-
|
|
5
|
+
#[derive(Clone, Debug)]
|
|
6
|
+
pub struct ProgressSnapshot {
|
|
7
|
+
pub current: u64,
|
|
8
|
+
pub total: u64,
|
|
9
|
+
pub percentage: f64,
|
|
10
|
+
pub elapsed_ms: u64,
|
|
11
|
+
pub eta_ms: Option<u64>,
|
|
12
|
+
pub speed_bytes_per_sec: f64,
|
|
13
|
+
pub step: String,
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
struct ProgressInner {
|
|
5
17
|
total: u64,
|
|
6
|
-
current:
|
|
7
|
-
|
|
18
|
+
current: u64,
|
|
19
|
+
step: String,
|
|
20
|
+
start: Instant,
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
pub struct ProgressBar {
|
|
24
|
+
inner: Arc<Mutex<ProgressInner>>,
|
|
25
|
+
callback: Arc<Mutex<Option<Box<dyn Fn(ProgressSnapshot) + Send>>>>,
|
|
8
26
|
}
|
|
9
27
|
|
|
10
28
|
impl ProgressBar {
|
|
11
29
|
pub fn new(total: u64) -> Self {
|
|
12
30
|
Self {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
31
|
+
inner: Arc::new(Mutex::new(ProgressInner {
|
|
32
|
+
total,
|
|
33
|
+
current: 0,
|
|
34
|
+
step: String::new(),
|
|
35
|
+
start: Instant::now(),
|
|
36
|
+
})),
|
|
37
|
+
callback: Arc::new(Mutex::new(None)),
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
pub fn with_callback<F: Fn(ProgressSnapshot) + Send + 'static>(total: u64, cb: F) -> Self {
|
|
42
|
+
Self {
|
|
43
|
+
inner: Arc::new(Mutex::new(ProgressInner {
|
|
44
|
+
total,
|
|
45
|
+
current: 0,
|
|
46
|
+
step: String::new(),
|
|
47
|
+
start: Instant::now(),
|
|
48
|
+
})),
|
|
49
|
+
callback: Arc::new(Mutex::new(Some(Box::new(cb)))),
|
|
16
50
|
}
|
|
17
51
|
}
|
|
18
52
|
|
|
19
53
|
pub fn inc(&self, delta: u64) {
|
|
20
|
-
let
|
|
21
|
-
|
|
54
|
+
let snapshot = {
|
|
55
|
+
let mut inner = self.inner.lock();
|
|
56
|
+
inner.current += delta;
|
|
57
|
+
self.snapshot_inner(&inner)
|
|
58
|
+
};
|
|
59
|
+
self.emit(snapshot);
|
|
22
60
|
}
|
|
23
61
|
|
|
24
62
|
pub fn set(&self, value: u64) {
|
|
25
|
-
let
|
|
26
|
-
|
|
63
|
+
let snapshot = {
|
|
64
|
+
let mut inner = self.inner.lock();
|
|
65
|
+
inner.current = value;
|
|
66
|
+
self.snapshot_inner(&inner)
|
|
67
|
+
};
|
|
68
|
+
self.emit(snapshot);
|
|
27
69
|
}
|
|
28
70
|
|
|
29
|
-
pub fn
|
|
30
|
-
let
|
|
31
|
-
|
|
71
|
+
pub fn set_step(&self, step: &str) {
|
|
72
|
+
let snapshot = {
|
|
73
|
+
let mut inner = self.inner.lock();
|
|
74
|
+
inner.step = step.to_string();
|
|
75
|
+
self.snapshot_inner(&inner)
|
|
76
|
+
};
|
|
77
|
+
self.emit(snapshot);
|
|
32
78
|
}
|
|
33
79
|
|
|
34
|
-
pub fn
|
|
35
|
-
let
|
|
36
|
-
|
|
80
|
+
pub fn snapshot(&self) -> ProgressSnapshot {
|
|
81
|
+
let inner = self.inner.lock();
|
|
82
|
+
self.snapshot_inner(&inner)
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
fn snapshot_inner(&self, inner: &ProgressInner) -> ProgressSnapshot {
|
|
86
|
+
let elapsed = inner.start.elapsed();
|
|
87
|
+
let elapsed_ms = elapsed.as_millis() as u64;
|
|
88
|
+
let percentage = if inner.total > 0 {
|
|
89
|
+
(inner.current as f64 / inner.total as f64) * 100.0
|
|
90
|
+
} else {
|
|
91
|
+
0.0
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
let elapsed_secs = elapsed.as_secs_f64();
|
|
95
|
+
let speed = if elapsed_secs > 0.01 {
|
|
96
|
+
inner.current as f64 / elapsed_secs
|
|
97
|
+
} else {
|
|
98
|
+
0.0
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
let eta_ms = if speed > 0.0 && inner.current > 0 && inner.current < inner.total {
|
|
102
|
+
let remaining = inner.total - inner.current;
|
|
103
|
+
Some((remaining as f64 / speed * 1000.0) as u64)
|
|
104
|
+
} else {
|
|
105
|
+
None
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
ProgressSnapshot {
|
|
109
|
+
current: inner.current,
|
|
110
|
+
total: inner.total,
|
|
111
|
+
percentage,
|
|
112
|
+
elapsed_ms,
|
|
113
|
+
eta_ms,
|
|
114
|
+
speed_bytes_per_sec: speed,
|
|
115
|
+
step: inner.step.clone(),
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
fn emit(&self, snapshot: ProgressSnapshot) {
|
|
120
|
+
if let Some(ref cb) = *self.callback.lock() {
|
|
121
|
+
cb(snapshot);
|
|
122
|
+
}
|
|
37
123
|
}
|
|
38
124
|
|
|
39
125
|
pub fn finish(&self) {
|
|
40
|
-
let
|
|
41
|
-
|
|
126
|
+
let snapshot = {
|
|
127
|
+
let mut inner = self.inner.lock();
|
|
128
|
+
inner.current = inner.total;
|
|
129
|
+
self.snapshot_inner(&inner)
|
|
130
|
+
};
|
|
131
|
+
self.emit(snapshot);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
pub fn get_progress(&self) -> (u64, u64) {
|
|
135
|
+
let inner = self.inner.lock();
|
|
136
|
+
(inner.current, inner.total)
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
pub fn set_message(&self, msg: String) {
|
|
140
|
+
self.set_step(&msg);
|
|
42
141
|
}
|
|
43
142
|
}
|
|
@@ -11,6 +11,8 @@ const MARKER_END: [(u8, u8, u8); 3] = [(0, 0, 255), (0, 255, 0), (255, 0, 0)];
|
|
|
11
11
|
const MARKER_ZSTD: (u8, u8, u8) = (0, 255, 0);
|
|
12
12
|
const MAGIC: &[u8] = b"ROX1";
|
|
13
13
|
|
|
14
|
+
pub type ProgressCallback = Box<dyn Fn(u64, u64, &str) + Send>;
|
|
15
|
+
|
|
14
16
|
pub fn encode_dir_to_png(
|
|
15
17
|
dir_path: &Path,
|
|
16
18
|
output_path: &Path,
|
|
@@ -27,10 +29,27 @@ pub fn encode_dir_to_png_encrypted(
|
|
|
27
29
|
name: Option<&str>,
|
|
28
30
|
passphrase: Option<&str>,
|
|
29
31
|
encrypt_type: Option<&str>,
|
|
32
|
+
) -> anyhow::Result<()> {
|
|
33
|
+
encode_dir_to_png_encrypted_with_progress(dir_path, output_path, compression_level, name, passphrase, encrypt_type, None)
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
pub fn encode_dir_to_png_encrypted_with_progress(
|
|
37
|
+
dir_path: &Path,
|
|
38
|
+
output_path: &Path,
|
|
39
|
+
compression_level: i32,
|
|
40
|
+
name: Option<&str>,
|
|
41
|
+
passphrase: Option<&str>,
|
|
42
|
+
encrypt_type: Option<&str>,
|
|
43
|
+
progress: Option<ProgressCallback>,
|
|
30
44
|
) -> anyhow::Result<()> {
|
|
31
45
|
let tmp_zst = output_path.with_extension("tmp.zst");
|
|
32
46
|
|
|
33
|
-
let file_list = compress_dir_to_zst(dir_path, &tmp_zst, compression_level)?;
|
|
47
|
+
let file_list = compress_dir_to_zst(dir_path, &tmp_zst, compression_level, &progress)?;
|
|
48
|
+
|
|
49
|
+
if let Some(ref cb) = progress {
|
|
50
|
+
cb(90, 100, "writing_png");
|
|
51
|
+
}
|
|
52
|
+
|
|
34
53
|
let file_list_json = serde_json::to_string(&file_list)?;
|
|
35
54
|
|
|
36
55
|
let result = write_png_from_zst(
|
|
@@ -38,6 +57,11 @@ pub fn encode_dir_to_png_encrypted(
|
|
|
38
57
|
passphrase, encrypt_type,
|
|
39
58
|
);
|
|
40
59
|
let _ = std::fs::remove_file(&tmp_zst);
|
|
60
|
+
|
|
61
|
+
if let Some(ref cb) = progress {
|
|
62
|
+
cb(100, 100, "done");
|
|
63
|
+
}
|
|
64
|
+
|
|
41
65
|
result
|
|
42
66
|
}
|
|
43
67
|
|
|
@@ -45,6 +69,7 @@ fn compress_dir_to_zst(
|
|
|
45
69
|
dir_path: &Path,
|
|
46
70
|
zst_path: &Path,
|
|
47
71
|
compression_level: i32,
|
|
72
|
+
progress: &Option<ProgressCallback>,
|
|
48
73
|
) -> anyhow::Result<Vec<serde_json::Value>> {
|
|
49
74
|
let base = dir_path;
|
|
50
75
|
|
|
@@ -55,6 +80,8 @@ fn compress_dir_to_zst(
|
|
|
55
80
|
.filter(|e| e.file_type().is_file())
|
|
56
81
|
.collect();
|
|
57
82
|
|
|
83
|
+
let total_files = entries.len() as u64;
|
|
84
|
+
|
|
58
85
|
let zst_file = File::create(zst_path)?;
|
|
59
86
|
let buf_writer = BufWriter::with_capacity(16 * 1024 * 1024, zst_file);
|
|
60
87
|
|
|
@@ -74,7 +101,7 @@ fn compress_dir_to_zst(
|
|
|
74
101
|
let mut file_list = Vec::new();
|
|
75
102
|
{
|
|
76
103
|
let mut tar_builder = Builder::new(&mut encoder);
|
|
77
|
-
for entry in
|
|
104
|
+
for (idx, entry) in entries.iter().enumerate() {
|
|
78
105
|
let full = entry.path();
|
|
79
106
|
let rel = full.strip_prefix(base).unwrap_or(full);
|
|
80
107
|
let rel_str = rel.to_string_lossy().replace('\\', "/");
|
|
@@ -103,6 +130,11 @@ fn compress_dir_to_zst(
|
|
|
103
130
|
.map_err(|e| anyhow::anyhow!("tar append {}: {}", rel_str, e))?;
|
|
104
131
|
|
|
105
132
|
file_list.push(serde_json::json!({"name": rel_str, "size": size}));
|
|
133
|
+
|
|
134
|
+
if let Some(ref cb) = progress {
|
|
135
|
+
let pct = ((idx as u64 + 1) * 85 / total_files.max(1)).min(85);
|
|
136
|
+
cb(pct, 100, "compressing");
|
|
137
|
+
}
|
|
106
138
|
}
|
|
107
139
|
tar_builder.finish().map_err(|e| anyhow::anyhow!("tar finish: {}", e))?;
|
|
108
140
|
}
|
package/package.json
CHANGED
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/native/gpu.rs
DELETED
|
@@ -1,116 +0,0 @@
|
|
|
1
|
-
use anyhow::{anyhow, Result};
|
|
2
|
-
use parking_lot::RwLock;
|
|
3
|
-
use std::sync::Arc;
|
|
4
|
-
use wgpu::*;
|
|
5
|
-
use wgpu::util::DeviceExt;
|
|
6
|
-
|
|
7
|
-
pub struct GpuDevice {
|
|
8
|
-
device: Device,
|
|
9
|
-
queue: Queue,
|
|
10
|
-
supported: bool,
|
|
11
|
-
adapter_info: String,
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
pub struct GpuContext {
|
|
15
|
-
inner: Arc<RwLock<Option<GpuDevice>>>,
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
impl GpuContext {
|
|
19
|
-
pub async fn new() -> Self {
|
|
20
|
-
let instance = Instance::new(InstanceDescriptor {
|
|
21
|
-
backends: Backends::all(),
|
|
22
|
-
..Default::default()
|
|
23
|
-
});
|
|
24
|
-
|
|
25
|
-
let adapter = instance.request_adapter(&RequestAdapterOptions {
|
|
26
|
-
power_preference: PowerPreference::HighPerformance,
|
|
27
|
-
compatible_surface: None,
|
|
28
|
-
force_fallback_adapter: false,
|
|
29
|
-
}).await;
|
|
30
|
-
|
|
31
|
-
let device_info = if let Some(adapter) = adapter {
|
|
32
|
-
match adapter.request_device(&DeviceDescriptor {
|
|
33
|
-
label: Some("roxify-compute"),
|
|
34
|
-
required_features: Features::empty(),
|
|
35
|
-
required_limits: Limits::default(),
|
|
36
|
-
}, None).await {
|
|
37
|
-
Ok((device, queue)) => {
|
|
38
|
-
let info = adapter.get_info();
|
|
39
|
-
Some(GpuDevice {
|
|
40
|
-
device,
|
|
41
|
-
queue,
|
|
42
|
-
supported: true,
|
|
43
|
-
adapter_info: format!("{:?}", info.driver),
|
|
44
|
-
})
|
|
45
|
-
}
|
|
46
|
-
Err(_) => None,
|
|
47
|
-
}
|
|
48
|
-
} else {
|
|
49
|
-
None
|
|
50
|
-
};
|
|
51
|
-
|
|
52
|
-
GpuContext {
|
|
53
|
-
inner: Arc::new(RwLock::new(device_info)),
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
pub fn is_available(&self) -> bool {
|
|
58
|
-
self.inner.read().is_some()
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
pub fn get_adapter_info(&self) -> Option<String> {
|
|
62
|
-
self.inner.read().as_ref().map(|d| d.adapter_info.clone())
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
pub async fn create_compute_pipeline(
|
|
66
|
-
&self,
|
|
67
|
-
shader_src: &str,
|
|
68
|
-
entry_point: &str,
|
|
69
|
-
) -> Result<ComputePipeline> {
|
|
70
|
-
let gpu = self.inner.read();
|
|
71
|
-
let gpu = gpu.as_ref().ok_or_else(|| anyhow!("No GPU device available"))?;
|
|
72
|
-
|
|
73
|
-
let shader_module = gpu.device.create_shader_module(ShaderModuleDescriptor {
|
|
74
|
-
label: Some("compute-shader"),
|
|
75
|
-
source: ShaderSource::Wgsl(std::borrow::Cow::Borrowed(shader_src)),
|
|
76
|
-
});
|
|
77
|
-
|
|
78
|
-
let pipeline_layout = gpu.device.create_pipeline_layout(&PipelineLayoutDescriptor {
|
|
79
|
-
label: Some("compute-layout"),
|
|
80
|
-
bind_group_layouts: &[],
|
|
81
|
-
push_constant_ranges: &[],
|
|
82
|
-
});
|
|
83
|
-
|
|
84
|
-
Ok(gpu.device.create_compute_pipeline(&ComputePipelineDescriptor {
|
|
85
|
-
label: Some("compute-pipeline"),
|
|
86
|
-
layout: Some(&pipeline_layout),
|
|
87
|
-
module: &shader_module,
|
|
88
|
-
entry_point,
|
|
89
|
-
}))
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
pub fn create_buffer_init(&self, data: &[u8], usage: BufferUsages) -> Result<Buffer> {
|
|
93
|
-
let gpu = self.inner.read();
|
|
94
|
-
let gpu = gpu.as_ref().ok_or_else(|| anyhow!("No GPU device available"))?;
|
|
95
|
-
|
|
96
|
-
Ok(gpu.device.create_buffer_init(&util::BufferInitDescriptor {
|
|
97
|
-
label: None,
|
|
98
|
-
contents: data,
|
|
99
|
-
usage,
|
|
100
|
-
}))
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
pub fn gpu_available() -> bool {
|
|
105
|
-
pollster::block_on(async {
|
|
106
|
-
let instance = Instance::new(InstanceDescriptor {
|
|
107
|
-
backends: Backends::all(),
|
|
108
|
-
..Default::default()
|
|
109
|
-
});
|
|
110
|
-
instance.request_adapter(&RequestAdapterOptions {
|
|
111
|
-
power_preference: PowerPreference::HighPerformance,
|
|
112
|
-
compatible_surface: None,
|
|
113
|
-
force_fallback_adapter: false,
|
|
114
|
-
}).await.is_some()
|
|
115
|
-
})
|
|
116
|
-
}
|