pi-mega-compact 0.20.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/vector-cortex.js +12 -0
- package/dist/config.js +1 -1
- package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +1 -0
- package/dist/extensions/dashboard-server/routes-vector-cortex-residual.js +51 -0
- package/dist/extensions/dashboard-server/routes-vector-cortex.js +1 -0
- package/dist/extensions/dashboard-server/routes.js +1 -1
- package/dist/extensions/dashboard-server/server.js +3 -1
- package/dist/src/config/vector-cortex.js +12 -0
- package/dist/src/config.js +1 -1
- package/dist/src/vector-cortex/residual/codec.js +224 -0
- package/dist/src/vector-cortex/residual/dct.js +158 -0
- package/dist/src/vector-cortex/residual/fixture-payload.js +77 -0
- package/dist/src/vector-cortex/residual/gf256.js +182 -0
- package/dist/src/vector-cortex/residual/parity.js +245 -0
- package/dist/src/vector-cortex/residual/quantize.js +200 -0
- package/dist/src/vector-cortex/residual/stream.js +124 -0
- package/dist/src/vector-cortex/residual/types.js +55 -0
- package/dist/vector-cortex/residual/codec.js +224 -0
- package/dist/vector-cortex/residual/dct.js +158 -0
- package/dist/vector-cortex/residual/fixture-payload.js +77 -0
- package/dist/vector-cortex/residual/gf256.js +182 -0
- package/dist/vector-cortex/residual/parity.js +245 -0
- package/dist/vector-cortex/residual/quantize.js +200 -0
- package/dist/vector-cortex/residual/stream.js +124 -0
- package/dist/vector-cortex/residual/types.js +55 -0
- package/extensions/dashboard-server/api-contracts/vector-cortex.ts +29 -0
- package/extensions/dashboard-server/routes-rag-settings-helpers.ts +6 -0
- package/extensions/dashboard-server/routes-vector-cortex-residual.ts +60 -0
- package/extensions/dashboard-server/routes-vector-cortex.ts +1 -0
- package/extensions/dashboard-server/routes.ts +1 -0
- package/extensions/dashboard-server/server.ts +2 -0
- package/package.json +1 -1
- package/src/config/vector-cortex.ts +13 -0
- package/src/config.ts +1 -0
- package/src/vector-cortex/residual/codec.ts +292 -0
- package/src/vector-cortex/residual/dct.ts +166 -0
- package/src/vector-cortex/residual/fixture-payload.ts +90 -0
- package/src/vector-cortex/residual/gf256.ts +196 -0
- package/src/vector-cortex/residual/parity.ts +283 -0
- package/src/vector-cortex/residual/quantize.ts +230 -0
- package/src/vector-cortex/residual/stream.ts +135 -0
- package/src/vector-cortex/residual/types.ts +236 -0
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/residual/stream.ts — canonical PROTECTED STREAM serialization
|
|
3
|
+
* for the VC4B residual codec.
|
|
4
|
+
*
|
|
5
|
+
* The protected stream is `header + all block scales/coefficient arrays +
|
|
6
|
+
* corrections` (RESIDUAL_CODEC §erasure parity), serialized in exactly this
|
|
7
|
+
* canonical order so encode and decode are byte-symmetric:
|
|
8
|
+
*
|
|
9
|
+
* header (46 bytes) magic `VCR1` | u32 LE originalLength | 32-byte SHA-256
|
|
10
|
+
* | u16 LE blockSize | u16 LE k | u16 LE m
|
|
11
|
+
* blocks for each block in ascending index:
|
|
12
|
+
* float32 LE scale | 4096 * int16 LE coefficient
|
|
13
|
+
* corrections varint blockCount, then per non-empty block
|
|
14
|
+
* u32 LE blockIndex | varint count
|
|
15
|
+
* | count * (u16 LE offset, u8 original)
|
|
16
|
+
*
|
|
17
|
+
* Pure serialization: no storage, no console, no network (PREVENT-PI-004 /
|
|
18
|
+
* PREVENT-011).
|
|
19
|
+
*/
|
|
20
|
+
import { parseCorrections, serializeCorrections, } from "./quantize.js";
|
|
21
|
+
import { RESIDUAL_BLOCK_SIZE, RESIDUAL_HEADER_BYTES, RESIDUAL_MAGIC, RS_DATA_SHARDS, RS_PARITY_SHARDS, } from "./types.js";
|
|
22
|
+
/** Per-block serialized size: float32 scale + n int16 coefficients. */
|
|
23
|
+
export function blockBytes(blockSize = RESIDUAL_BLOCK_SIZE) {
|
|
24
|
+
return 4 + blockSize * 2;
|
|
25
|
+
}
|
|
26
|
+
/** Serialize the canonical 46-byte header. */
|
|
27
|
+
export function serializeHeader(header) {
|
|
28
|
+
const out = new Uint8Array(RESIDUAL_HEADER_BYTES);
|
|
29
|
+
const view = new DataView(out.buffer);
|
|
30
|
+
for (let i = 0; i < 4; i++)
|
|
31
|
+
out[i] = RESIDUAL_MAGIC.charCodeAt(i);
|
|
32
|
+
view.setUint32(4, header.originalLength, true);
|
|
33
|
+
const digest = Buffer.from(header.payloadDigest, "hex");
|
|
34
|
+
out.set(digest.subarray(0, 32), 8);
|
|
35
|
+
view.setUint16(40, header.blockSize, true);
|
|
36
|
+
view.setUint16(42, header.dataShards, true);
|
|
37
|
+
view.setUint16(44, header.parityShards, true);
|
|
38
|
+
return out;
|
|
39
|
+
}
|
|
40
|
+
/** Parse the canonical header. Returns null on magic/geometry mismatch. */
|
|
41
|
+
export function parseHeader(bytes) {
|
|
42
|
+
if (bytes.length < RESIDUAL_HEADER_BYTES)
|
|
43
|
+
return null;
|
|
44
|
+
for (let i = 0; i < 4; i++) {
|
|
45
|
+
if (bytes[i] !== RESIDUAL_MAGIC.charCodeAt(i))
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
49
|
+
const originalLength = view.getUint32(4, true);
|
|
50
|
+
const payloadDigest = Buffer.from(bytes.subarray(8, 40)).toString("hex");
|
|
51
|
+
const blockSize = view.getUint16(40, true);
|
|
52
|
+
const dataShards = view.getUint16(42, true);
|
|
53
|
+
const parityShards = view.getUint16(44, true);
|
|
54
|
+
if (blockSize !== RESIDUAL_BLOCK_SIZE)
|
|
55
|
+
return null;
|
|
56
|
+
if (dataShards !== RS_DATA_SHARDS || parityShards !== RS_PARITY_SHARDS)
|
|
57
|
+
return null;
|
|
58
|
+
return {
|
|
59
|
+
magic: RESIDUAL_MAGIC,
|
|
60
|
+
originalLength,
|
|
61
|
+
payloadDigest,
|
|
62
|
+
blockSize: RESIDUAL_BLOCK_SIZE,
|
|
63
|
+
dataShards: RS_DATA_SHARDS,
|
|
64
|
+
parityShards: RS_PARITY_SHARDS,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/** Serialize the full protected stream for one encoded artifact. */
|
|
68
|
+
export function serializeStream(codec) {
|
|
69
|
+
const head = serializeHeader(codec.header);
|
|
70
|
+
const perBlock = blockBytes(codec.header.blockSize);
|
|
71
|
+
const corrections = serializeCorrections(codec.corrections);
|
|
72
|
+
const out = new Uint8Array(head.length + codec.blocks.length * perBlock + corrections.length);
|
|
73
|
+
out.set(head, 0);
|
|
74
|
+
const view = new DataView(out.buffer);
|
|
75
|
+
let pos = head.length;
|
|
76
|
+
for (const block of codec.blocks) {
|
|
77
|
+
view.setFloat32(pos, block.scale, true);
|
|
78
|
+
pos += 4;
|
|
79
|
+
for (let i = 0; i < block.coefficients.length; i++) {
|
|
80
|
+
view.setInt16(pos, block.coefficients[i], true);
|
|
81
|
+
pos += 2;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
out.set(corrections, pos);
|
|
85
|
+
return out;
|
|
86
|
+
}
|
|
87
|
+
/** Parse a protected stream back into the codec artifact. Null on malformed. */
|
|
88
|
+
export function parseStream(stream) {
|
|
89
|
+
const header = parseHeader(stream);
|
|
90
|
+
if (!header)
|
|
91
|
+
return null;
|
|
92
|
+
const perBlock = blockBytes(header.blockSize);
|
|
93
|
+
const blockCount = Math.ceil(header.originalLength / header.blockSize);
|
|
94
|
+
const bodyEnd = RESIDUAL_HEADER_BYTES + blockCount * perBlock;
|
|
95
|
+
if (stream.length < bodyEnd)
|
|
96
|
+
return null;
|
|
97
|
+
const view = new DataView(stream.buffer, stream.byteOffset, stream.byteLength);
|
|
98
|
+
const blocks = [];
|
|
99
|
+
let pos = RESIDUAL_HEADER_BYTES;
|
|
100
|
+
for (let b = 0; b < blockCount; b++) {
|
|
101
|
+
const scale = view.getFloat32(pos, true);
|
|
102
|
+
pos += 4;
|
|
103
|
+
const coefficients = new Int16Array(header.blockSize);
|
|
104
|
+
for (let i = 0; i < header.blockSize; i++) {
|
|
105
|
+
coefficients[i] = view.getInt16(pos, true);
|
|
106
|
+
pos += 2;
|
|
107
|
+
}
|
|
108
|
+
blocks.push({ scale, coefficients });
|
|
109
|
+
}
|
|
110
|
+
const parsed = parseCorrections(stream, pos);
|
|
111
|
+
if (!parsed)
|
|
112
|
+
return null;
|
|
113
|
+
// Every correction must name a block that exists.
|
|
114
|
+
for (const block of parsed.blocks) {
|
|
115
|
+
if (block.blockIndex >= blockCount)
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
return {
|
|
119
|
+
schema: "residual-codec-v1",
|
|
120
|
+
header,
|
|
121
|
+
blocks,
|
|
122
|
+
corrections: parsed.blocks,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/residual/types.ts — reversible residual payload codec (VC4B).
|
|
3
|
+
*
|
|
4
|
+
* Owns `ResidualCodecV1` / `ParityShardV1` — the contract of the sprint failure
|
|
5
|
+
* triad:
|
|
6
|
+
*
|
|
7
|
+
* A = admitted residual (DCT + int16 + exact corrections) + RS(9,6) parity;
|
|
8
|
+
* B = exact compressed bytes (forced when the >95% accounting rejects A);
|
|
9
|
+
* C = ledger bytes (forced when A/B decode fails).
|
|
10
|
+
*
|
|
11
|
+
* Semantic vectors NEVER claim to recover exact text. Exact bytes come only from
|
|
12
|
+
* exact payload shards (VC4A `ExactShardV1`) or this REVERSIBLE codec: a block
|
|
13
|
+
* DCT-II analysis, int16 quantization, and a block-scoped exact correction
|
|
14
|
+
* stream that makes post-decode byte error exactly zero for admitted artifacts.
|
|
15
|
+
* Numeric erasure parity protects the codec bytes; it is not a substitute for
|
|
16
|
+
* the exact payload (RESIDUAL_CODEC.md).
|
|
17
|
+
*
|
|
18
|
+
* Consumes only reviewer-accepted predecessor contracts (VC1A EventV2 bytes via
|
|
19
|
+
* VC4A shard ranges) and the common contracts. Pure types + registered
|
|
20
|
+
* conformance IDs: no storage, no console, no network (PREVENT-PI-004 /
|
|
21
|
+
* PREVENT-011).
|
|
22
|
+
*/
|
|
23
|
+
/** Fixed transform block length (RESIDUAL_CODEC §byte scope: 4096). */
|
|
24
|
+
export const RESIDUAL_BLOCK_SIZE = 4096;
|
|
25
|
+
/** Reed–Solomon shard geometry: k=6 data shards, m=3 parity shards. */
|
|
26
|
+
export const RS_DATA_SHARDS = 6;
|
|
27
|
+
export const RS_PARITY_SHARDS = 3;
|
|
28
|
+
export const RS_TOTAL_SHARDS = RS_DATA_SHARDS + RS_PARITY_SHARDS;
|
|
29
|
+
/** GF(2^8) primitive polynomial for the parity field (0x11d). */
|
|
30
|
+
export const GF_PRIMITIVE_POLYNOMIAL = 0x11d;
|
|
31
|
+
/** Header magic bytes `VCR1`. */
|
|
32
|
+
export const RESIDUAL_MAGIC = "VCR1";
|
|
33
|
+
/**
|
|
34
|
+
* Admission ratio: residual is admitted only when its FULL encoded size is at
|
|
35
|
+
* most `floor(0.95 * exactCompressedSize)` (RESIDUAL_CODEC §admission). The
|
|
36
|
+
* accounting counts every persisted byte — header, scales, coefficients,
|
|
37
|
+
* corrections, shard metadata, all 9 shards, and digests.
|
|
38
|
+
*/
|
|
39
|
+
export const ADMISSION_NUMERATOR = 95;
|
|
40
|
+
export const ADMISSION_DENOMINATOR = 100;
|
|
41
|
+
/** Serialized header byte length: 4 + 4 + 32 + 2 + 2 + 2. */
|
|
42
|
+
export const RESIDUAL_HEADER_BYTES = 46;
|
|
43
|
+
/**
|
|
44
|
+
* Registered RES conformance ID range (RES-001..050). The acceptance test reads
|
|
45
|
+
* these rows from the v2 manifest and asserts each returns its manifest
|
|
46
|
+
* `ok`/`code`. The three named assertions (RES-DCT-001 / RES-RS-002 /
|
|
47
|
+
* RES-ADMIT-003) live alongside them.
|
|
48
|
+
*/
|
|
49
|
+
export const RES_IDS = Array.from({ length: 50 }, (_v, i) => `RES-${String(i + 1).padStart(3, "0")}`);
|
|
50
|
+
/** Named RES conformance assertions (the sprint's headline rows). */
|
|
51
|
+
export const RES_NAMED_IDS = [
|
|
52
|
+
"RES-DCT-001",
|
|
53
|
+
"RES-RS-002",
|
|
54
|
+
"RES-ADMIT-003",
|
|
55
|
+
];
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/residual/codec.ts — reversible residual payload codec (VC4B).
|
|
3
|
+
*
|
|
4
|
+
* Encode: split `EventV2.originalBytes` into 4096-byte blocks (zero-padding only
|
|
5
|
+
* the final block), map bytes to `x=(byte-127.5)/127.5`, take the orthonormal
|
|
6
|
+
* DCT-II, quantize to int16 with a per-block float32 scale, reconstruct, and
|
|
7
|
+
* append a block-scoped EXACT correction stream wherever the reconstruction
|
|
8
|
+
* differs. Post-decode byte error for an admitted artifact is therefore exactly
|
|
9
|
+
* ZERO — the digest is verified before admission, never assumed.
|
|
10
|
+
*
|
|
11
|
+
* Admission (RESIDUAL_CODEC §admission): residual is admitted only when the FULL
|
|
12
|
+
* encoded size — header, scales, coefficients, corrections, shard index/length
|
|
13
|
+
* metadata, all 9 shards and their digests — is `<= floor(0.95 *
|
|
14
|
+
* exactCompressedSize)` AND the complete decode + digest check succeeds.
|
|
15
|
+
* Otherwise the caller stores the exact compressed payload (mode B). Coefficient
|
|
16
|
+
* bytes alone are never compared.
|
|
17
|
+
*
|
|
18
|
+
* Emits `vector_cortex_residual_admitted` / `vector_cortex_parity_recovery_failed`
|
|
19
|
+
* through the flag-gated reporter and exposes AGGREGATE-ONLY residual metrics
|
|
20
|
+
* (counts/byte totals — never payload).
|
|
21
|
+
*
|
|
22
|
+
* Guardrails: local hashing only, no storage, no console, no network
|
|
23
|
+
* (PREVENT-PI-004 / PREVENT-011).
|
|
24
|
+
*/
|
|
25
|
+
import { createHash } from "node:crypto";
|
|
26
|
+
import { VC4B_ENABLED } from "../../config/vector-cortex.js";
|
|
27
|
+
import { bytesToSignal, forwardDct, inverseDct, signalToBytes, splitBlocks, } from "./dct.js";
|
|
28
|
+
import { applyCorrections, dequantizeBlock, diffBlock, quantizeBlock, } from "./quantize.js";
|
|
29
|
+
import { encodeShards, recoverStream, sha256Hex } from "./parity.js";
|
|
30
|
+
import { parseStream, serializeStream } from "./stream.js";
|
|
31
|
+
import { ADMISSION_DENOMINATOR, ADMISSION_NUMERATOR, RESIDUAL_BLOCK_SIZE, RESIDUAL_MAGIC, RS_DATA_SHARDS, RS_PARITY_SHARDS, } from "./types.js";
|
|
32
|
+
/** Per-shard persisted metadata: u8 index + u32 LE length + 32-byte digest. */
|
|
33
|
+
const SHARD_METADATA_BYTES = 1 + 4 + 32;
|
|
34
|
+
/**
|
|
35
|
+
* The inclusive admission ceiling `floor(0.95 * exactCompressedSize)`, computed
|
|
36
|
+
* in integer arithmetic so the boundary is exact (a fractional 0.95 multiply
|
|
37
|
+
* would make the "one byte above rejects" case depend on float rounding).
|
|
38
|
+
*/
|
|
39
|
+
export function admissionCeiling(exactCompressedSize) {
|
|
40
|
+
return Math.floor((exactCompressedSize * ADMISSION_NUMERATOR) / ADMISSION_DENOMINATOR);
|
|
41
|
+
}
|
|
42
|
+
/** Total persisted bytes of the shard set (payload + per-shard metadata). */
|
|
43
|
+
export function shardSetBytes(shards) {
|
|
44
|
+
return shards.reduce((n, s) => n + s.bytes.length + SHARD_METADATA_BYTES, 0);
|
|
45
|
+
}
|
|
46
|
+
/** Build the codec artifact (transform + quantize + exact corrections). */
|
|
47
|
+
export function buildArtifact(payload) {
|
|
48
|
+
const digest = sha256Hex(payload);
|
|
49
|
+
const rawBlocks = splitBlocks(payload, RESIDUAL_BLOCK_SIZE);
|
|
50
|
+
const blocks = [];
|
|
51
|
+
const corrections = [];
|
|
52
|
+
for (let b = 0; b < rawBlocks.length; b++) {
|
|
53
|
+
const original = rawBlocks[b];
|
|
54
|
+
const quantized = quantizeBlock(forwardDct(bytesToSignal(original)));
|
|
55
|
+
if (!quantized.ok)
|
|
56
|
+
return { ok: false, code: "RES_QUANTIZE_RANGE" };
|
|
57
|
+
blocks.push(quantized.block);
|
|
58
|
+
// Reconstruct and diff: any residual byte error becomes an exact correction.
|
|
59
|
+
const reconstructed = signalToBytes(inverseDct(dequantizeBlock(quantized.block)));
|
|
60
|
+
const diff = diffBlock(original, reconstructed);
|
|
61
|
+
if (diff.length > 0)
|
|
62
|
+
corrections.push({ blockIndex: b, corrections: diff });
|
|
63
|
+
}
|
|
64
|
+
return {
|
|
65
|
+
ok: true,
|
|
66
|
+
codec: {
|
|
67
|
+
schema: "residual-codec-v1",
|
|
68
|
+
header: {
|
|
69
|
+
magic: RESIDUAL_MAGIC,
|
|
70
|
+
originalLength: payload.length,
|
|
71
|
+
payloadDigest: digest,
|
|
72
|
+
blockSize: RESIDUAL_BLOCK_SIZE,
|
|
73
|
+
dataShards: RS_DATA_SHARDS,
|
|
74
|
+
parityShards: RS_PARITY_SHARDS,
|
|
75
|
+
},
|
|
76
|
+
blocks,
|
|
77
|
+
corrections,
|
|
78
|
+
},
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Encode a payload and decide admission against the competing exact compressed
|
|
83
|
+
* size. Admission requires BOTH the <=95% byte accounting AND a full decode whose
|
|
84
|
+
* digest matches the original payload.
|
|
85
|
+
*/
|
|
86
|
+
export function encodeResidual(payload, exactCompressedSize, emit) {
|
|
87
|
+
const built = buildArtifact(payload);
|
|
88
|
+
if (!built.ok)
|
|
89
|
+
return { ok: false, code: built.code };
|
|
90
|
+
const codec = built.codec;
|
|
91
|
+
const stream = serializeStream(codec);
|
|
92
|
+
const shards = encodeShards(stream);
|
|
93
|
+
const encodedSize = stream.length + shardSetBytes(shards);
|
|
94
|
+
const correctionCount = codec.corrections.reduce((n, b) => n + b.corrections.length, 0);
|
|
95
|
+
const accounting = {
|
|
96
|
+
encodedSize,
|
|
97
|
+
exactCompressedSize,
|
|
98
|
+
admissionCeiling: admissionCeiling(exactCompressedSize),
|
|
99
|
+
correctionCount,
|
|
100
|
+
blockCount: codec.blocks.length,
|
|
101
|
+
};
|
|
102
|
+
const reporter = createResidualReporter(emit);
|
|
103
|
+
if (encodedSize > accounting.admissionCeiling) {
|
|
104
|
+
return { ok: true, admitted: false, code: "RES_NOT_ADMITTED", accounting };
|
|
105
|
+
}
|
|
106
|
+
// Never admit without proving the full decode round-trips to the exact bytes.
|
|
107
|
+
const verified = decodeResidual(shards, emit);
|
|
108
|
+
if (!verified.ok)
|
|
109
|
+
return { ok: false, code: verified.code };
|
|
110
|
+
if (sha256Hex(verified.bytes) !== codec.header.payloadDigest) {
|
|
111
|
+
return { ok: false, code: "RES_PAYLOAD_DIGEST_MISMATCH" };
|
|
112
|
+
}
|
|
113
|
+
reporter.residualAdmitted({
|
|
114
|
+
encodedSize,
|
|
115
|
+
exactCompressedSize,
|
|
116
|
+
admissionCeiling: accounting.admissionCeiling,
|
|
117
|
+
blockCount: accounting.blockCount,
|
|
118
|
+
correctionCount,
|
|
119
|
+
});
|
|
120
|
+
return { ok: true, admitted: true, codec, shards, accounting };
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Decode from a (possibly partial / partially corrupt) shard set: recover the
|
|
124
|
+
* protected stream, invert the transform, apply the exact corrections, truncate
|
|
125
|
+
* to the original length, and verify the payload digest.
|
|
126
|
+
*/
|
|
127
|
+
export function decodeResidual(shards, emit) {
|
|
128
|
+
const reporter = createResidualReporter(emit);
|
|
129
|
+
const recovered = recoverStream(shards);
|
|
130
|
+
if (!recovered.ok) {
|
|
131
|
+
reporter.parityRecoveryFailed({
|
|
132
|
+
code: recovered.code,
|
|
133
|
+
shardCount: shards.length,
|
|
134
|
+
});
|
|
135
|
+
return { ok: false, code: recovered.code };
|
|
136
|
+
}
|
|
137
|
+
const codec = parseStream(recovered.stream);
|
|
138
|
+
if (!codec) {
|
|
139
|
+
reporter.parityRecoveryFailed({
|
|
140
|
+
code: "RES_HEADER_INVALID",
|
|
141
|
+
shardCount: shards.length,
|
|
142
|
+
});
|
|
143
|
+
return { ok: false, code: "RES_HEADER_INVALID" };
|
|
144
|
+
}
|
|
145
|
+
return decodeArtifact(codec);
|
|
146
|
+
}
|
|
147
|
+
/** Decode a parsed artifact directly (no parity layer). */
|
|
148
|
+
export function decodeArtifact(codec) {
|
|
149
|
+
const blockSize = codec.header.blockSize;
|
|
150
|
+
const out = new Uint8Array(codec.blocks.length * blockSize);
|
|
151
|
+
const byIndex = new Map();
|
|
152
|
+
for (const b of codec.corrections)
|
|
153
|
+
byIndex.set(b.blockIndex, b.corrections);
|
|
154
|
+
for (let b = 0; b < codec.blocks.length; b++) {
|
|
155
|
+
const reconstructed = signalToBytes(inverseDct(dequantizeBlock(codec.blocks[b])));
|
|
156
|
+
const applied = applyCorrections(reconstructed, byIndex.get(b) ?? []);
|
|
157
|
+
if (!applied.ok)
|
|
158
|
+
return { ok: false, code: applied.code };
|
|
159
|
+
out.set(reconstructed, b * blockSize);
|
|
160
|
+
}
|
|
161
|
+
const bytes = out.subarray(0, codec.header.originalLength);
|
|
162
|
+
if (sha256Hex(bytes) !== codec.header.payloadDigest) {
|
|
163
|
+
return { ok: false, code: "RES_PAYLOAD_DIGEST_MISMATCH" };
|
|
164
|
+
}
|
|
165
|
+
// Return an independent copy so the caller cannot alias the working buffer.
|
|
166
|
+
return { ok: true, bytes: Uint8Array.from(bytes) };
|
|
167
|
+
}
|
|
168
|
+
/** SHA-256 of arbitrary bytes (re-exported so callers need one import). */
|
|
169
|
+
export function payloadDigest(bytes) {
|
|
170
|
+
return createHash("sha256").update(bytes).digest("hex");
|
|
171
|
+
}
|
|
172
|
+
// ── aggregate-only metrics + reporter ───────────────────────────────────────
|
|
173
|
+
/**
|
|
174
|
+
* Accumulate AGGREGATE-ONLY residual metrics (counts/byte totals). Never
|
|
175
|
+
* payload, never prompt text: the dashboard reads this shape and nothing else.
|
|
176
|
+
*/
|
|
177
|
+
export function accumulateMetrics(previous, result) {
|
|
178
|
+
if (!result.ok) {
|
|
179
|
+
return { ...previous, encodeAttempts: previous.encodeAttempts + 1 };
|
|
180
|
+
}
|
|
181
|
+
const base = {
|
|
182
|
+
...previous,
|
|
183
|
+
encodeAttempts: previous.encodeAttempts + 1,
|
|
184
|
+
encodedByteTotal: previous.encodedByteTotal + result.accounting.encodedSize,
|
|
185
|
+
exactByteTotal: previous.exactByteTotal + result.accounting.exactCompressedSize,
|
|
186
|
+
};
|
|
187
|
+
return result.admitted
|
|
188
|
+
? { ...base, admittedCount: base.admittedCount + 1 }
|
|
189
|
+
: { ...base, rejectedCount: base.rejectedCount + 1 };
|
|
190
|
+
}
|
|
191
|
+
/** A zeroed metrics accumulator. */
|
|
192
|
+
export function emptyMetrics() {
|
|
193
|
+
return {
|
|
194
|
+
encodeAttempts: 0,
|
|
195
|
+
admittedCount: 0,
|
|
196
|
+
rejectedCount: 0,
|
|
197
|
+
recoveryFailures: 0,
|
|
198
|
+
encodedByteTotal: 0,
|
|
199
|
+
exactByteTotal: 0,
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
/** Build the flag-gated typed reporter (mirrors the VC4A shard reporter). */
|
|
203
|
+
export function createResidualReporter(emit) {
|
|
204
|
+
const fire = (event, fields) => {
|
|
205
|
+
if (!VC4B_ENABLED())
|
|
206
|
+
return;
|
|
207
|
+
if (!emit)
|
|
208
|
+
return;
|
|
209
|
+
try {
|
|
210
|
+
emit(event, fields);
|
|
211
|
+
}
|
|
212
|
+
catch {
|
|
213
|
+
/* non-fatal observability — never break the agent loop */
|
|
214
|
+
}
|
|
215
|
+
};
|
|
216
|
+
return {
|
|
217
|
+
residualAdmitted(fields) {
|
|
218
|
+
fire("vector_cortex_residual_admitted", fields);
|
|
219
|
+
},
|
|
220
|
+
parityRecoveryFailed(fields) {
|
|
221
|
+
fire("vector_cortex_parity_recovery_failed", fields);
|
|
222
|
+
},
|
|
223
|
+
};
|
|
224
|
+
}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/residual/dct.ts — orthonormal DCT-II basis V1 (VC4B).
|
|
3
|
+
*
|
|
4
|
+
* Basis V1 is the orthonormal DCT-II matrix generated ANALYTICALLY and never
|
|
5
|
+
* learned or stored (RESIDUAL_CODEC §byte scope):
|
|
6
|
+
*
|
|
7
|
+
* alpha(0) = sqrt(1/n), alpha(k>0) = sqrt(2/n)
|
|
8
|
+
* C[k][i] = alpha(k) * cos(pi * (2i + 1) * k / (2n))
|
|
9
|
+
*
|
|
10
|
+
* Forward coefficients are the dot product with each basis row (the orthonormal
|
|
11
|
+
* least-squares solve); the inverse is the transpose applied to the coefficient
|
|
12
|
+
* vector. Byte mapping is `x = (byte - 127.5) / 127.5`; the inverse maps back
|
|
13
|
+
* with round-to-nearest-even and clamps to 0..255.
|
|
14
|
+
*
|
|
15
|
+
* The full 4096x4096 matrix is ~134 MB of float64, so it is NEVER materialized.
|
|
16
|
+
* Instead the cosine argument `pi*(2i+1)*k/(2n)` is reduced modulo `2n` against
|
|
17
|
+
* a precomputed half-period table of length `2n` — this is EXACTLY the same set
|
|
18
|
+
* of cosine values the matrix would hold (cos is sampled only at the `2n`
|
|
19
|
+
* distinct arguments `pi*j/(2n)`, j = 0..2n-1), so the transform is bit-stable
|
|
20
|
+
* and table-generation is O(n) rather than O(n^2).
|
|
21
|
+
*
|
|
22
|
+
* Pure numeric transform: no storage, no console, no network (PREVENT-PI-004 /
|
|
23
|
+
* PREVENT-011).
|
|
24
|
+
*/
|
|
25
|
+
import { RESIDUAL_BLOCK_SIZE } from "./types.js";
|
|
26
|
+
/** Byte-to-signal mapping midpoint / half-range (RESIDUAL_CODEC §transform). */
|
|
27
|
+
const BYTE_MIDPOINT = 127.5;
|
|
28
|
+
/**
|
|
29
|
+
* Cosine table cache keyed by block length. `table[j] = cos(pi * j / (2n))` for
|
|
30
|
+
* j = 0..2n-1; every DCT argument `pi*(2i+1)*k/(2n)` reduces to one of these
|
|
31
|
+
* entries (with a sign) because cos has period `2*pi` = index period `4n`.
|
|
32
|
+
*/
|
|
33
|
+
const cosTables = new Map();
|
|
34
|
+
/** Build (or fetch) the `4n`-entry cosine table for block length `n`. */
|
|
35
|
+
function cosTable(n) {
|
|
36
|
+
const cached = cosTables.get(n);
|
|
37
|
+
if (cached)
|
|
38
|
+
return cached;
|
|
39
|
+
const period = 4 * n;
|
|
40
|
+
const table = new Float64Array(period);
|
|
41
|
+
for (let j = 0; j < period; j++) {
|
|
42
|
+
table[j] = Math.cos((Math.PI * j) / (2 * n));
|
|
43
|
+
}
|
|
44
|
+
cosTables.set(n, table);
|
|
45
|
+
return table;
|
|
46
|
+
}
|
|
47
|
+
/** Orthonormal DCT-II row scale: alpha(0)=sqrt(1/n), else sqrt(2/n). */
|
|
48
|
+
export function alpha(k, n) {
|
|
49
|
+
return k === 0 ? Math.sqrt(1 / n) : Math.sqrt(2 / n);
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Map a padded byte block to the signal domain: `x = (byte - 127.5) / 127.5`.
|
|
53
|
+
* The input must be exactly `n` bytes (the caller zero-pads the final block).
|
|
54
|
+
*/
|
|
55
|
+
export function bytesToSignal(block) {
|
|
56
|
+
const out = new Float64Array(block.length);
|
|
57
|
+
for (let i = 0; i < block.length; i++) {
|
|
58
|
+
out[i] = (block[i] - BYTE_MIDPOINT) / BYTE_MIDPOINT;
|
|
59
|
+
}
|
|
60
|
+
return out;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Map a reconstructed signal back to bytes: invert the affine map, round to
|
|
64
|
+
* nearest with ties-to-even, and clamp to 0..255 (RESIDUAL_CODEC §transform).
|
|
65
|
+
*/
|
|
66
|
+
export function signalToBytes(signal) {
|
|
67
|
+
const out = new Uint8Array(signal.length);
|
|
68
|
+
for (let i = 0; i < signal.length; i++) {
|
|
69
|
+
const v = signal[i] * BYTE_MIDPOINT + BYTE_MIDPOINT;
|
|
70
|
+
const r = roundHalfToEven(v);
|
|
71
|
+
out[i] = r < 0 ? 0 : r > 255 ? 255 : r;
|
|
72
|
+
}
|
|
73
|
+
return out;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Round to nearest, ties to even (banker's rounding). `Math.round` rounds ties
|
|
77
|
+
* toward +Infinity, which is NOT the rule RESIDUAL_CODEC mandates.
|
|
78
|
+
*/
|
|
79
|
+
export function roundHalfToEven(v) {
|
|
80
|
+
const floor = Math.floor(v);
|
|
81
|
+
const diff = v - floor;
|
|
82
|
+
if (diff > 0.5)
|
|
83
|
+
return floor + 1;
|
|
84
|
+
if (diff < 0.5)
|
|
85
|
+
return floor;
|
|
86
|
+
// Exact tie: pick the even neighbour.
|
|
87
|
+
return floor % 2 === 0 ? floor : floor + 1;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Forward orthonormal DCT-II over one block of length `n` (default 4096).
|
|
91
|
+
* `coefficients[k] = alpha(k) * sum_i x[i] * cos(pi*(2i+1)*k/(2n))`, emitted in
|
|
92
|
+
* ascending frequency order `k = 0..n-1` (the exact RESIDUAL_CODEC coefficient
|
|
93
|
+
* order).
|
|
94
|
+
*/
|
|
95
|
+
export function forwardDct(signal) {
|
|
96
|
+
const n = signal.length;
|
|
97
|
+
const table = cosTable(n);
|
|
98
|
+
const period = 4 * n;
|
|
99
|
+
const out = new Float64Array(n);
|
|
100
|
+
for (let k = 0; k < n; k++) {
|
|
101
|
+
let sum = 0;
|
|
102
|
+
// Argument index advances by 2k each step, starting at k (i=0 gives
|
|
103
|
+
// (2*0+1)*k = k). Reduce modulo the 4n-entry period.
|
|
104
|
+
let idx = k % period;
|
|
105
|
+
const step = (2 * k) % period;
|
|
106
|
+
for (let i = 0; i < n; i++) {
|
|
107
|
+
sum += signal[i] * table[idx];
|
|
108
|
+
idx += step;
|
|
109
|
+
if (idx >= period)
|
|
110
|
+
idx -= period;
|
|
111
|
+
}
|
|
112
|
+
out[k] = alpha(k, n) * sum;
|
|
113
|
+
}
|
|
114
|
+
return out;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Inverse orthonormal DCT (the transpose of the forward matrix):
|
|
118
|
+
* `x[i] = sum_k alpha(k) * c[k] * cos(pi*(2i+1)*k/(2n))`.
|
|
119
|
+
*/
|
|
120
|
+
export function inverseDct(coefficients) {
|
|
121
|
+
const n = coefficients.length;
|
|
122
|
+
const table = cosTable(n);
|
|
123
|
+
const period = 4 * n;
|
|
124
|
+
// Pre-scale each coefficient by its row alpha so the inner loop is a plain
|
|
125
|
+
// dot product (identical arithmetic to scaling inside the loop for k, and it
|
|
126
|
+
// keeps the per-i accumulation order stable).
|
|
127
|
+
const scaled = new Float64Array(n);
|
|
128
|
+
for (let k = 0; k < n; k++)
|
|
129
|
+
scaled[k] = alpha(k, n) * coefficients[k];
|
|
130
|
+
const out = new Float64Array(n);
|
|
131
|
+
for (let i = 0; i < n; i++) {
|
|
132
|
+
let sum = 0;
|
|
133
|
+
const step = (2 * i + 1) % period;
|
|
134
|
+
let idx = 0;
|
|
135
|
+
for (let k = 0; k < n; k++) {
|
|
136
|
+
sum += scaled[k] * table[idx];
|
|
137
|
+
idx += step;
|
|
138
|
+
if (idx >= period)
|
|
139
|
+
idx -= period;
|
|
140
|
+
}
|
|
141
|
+
out[i] = sum;
|
|
142
|
+
}
|
|
143
|
+
return out;
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Split a payload into fixed 4096-byte blocks, zero-padding ONLY the final
|
|
147
|
+
* block. The original length is retained by the caller (the header) so the
|
|
148
|
+
* padding is dropped on decode.
|
|
149
|
+
*/
|
|
150
|
+
export function splitBlocks(payload, blockSize = RESIDUAL_BLOCK_SIZE) {
|
|
151
|
+
const blocks = [];
|
|
152
|
+
for (let off = 0; off < payload.length; off += blockSize) {
|
|
153
|
+
const block = new Uint8Array(blockSize);
|
|
154
|
+
block.set(payload.subarray(off, Math.min(off + blockSize, payload.length)));
|
|
155
|
+
blocks.push(block);
|
|
156
|
+
}
|
|
157
|
+
return blocks;
|
|
158
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/residual/fixture-payload.ts — deterministic payload
|
|
3
|
+
* materialization for the VC4B conformance corpus.
|
|
4
|
+
*
|
|
5
|
+
* The residual fixtures describe their payloads GENERATIVELY (`kind` + `length`
|
|
6
|
+
* + `seed`) rather than embedding megabytes of base64, so this module is the
|
|
7
|
+
* single normative generator both the fixture producer and the acceptance test
|
|
8
|
+
* agree on. Every generator is pure and deterministic — the same descriptor
|
|
9
|
+
* always yields byte-identical output, which is what makes the committed
|
|
10
|
+
* fixtures meaningful.
|
|
11
|
+
*
|
|
12
|
+
* Pure byte generation: no storage, no console, no network (PREVENT-PI-004 /
|
|
13
|
+
* PREVENT-011).
|
|
14
|
+
*/
|
|
15
|
+
/** Linear congruential generator (numerical-recipes constants), deterministic. */
|
|
16
|
+
function lcgBytes(length, seed) {
|
|
17
|
+
const out = new Uint8Array(length);
|
|
18
|
+
let state = seed >>> 0;
|
|
19
|
+
for (let i = 0; i < length; i++) {
|
|
20
|
+
state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
|
|
21
|
+
out[i] = (state >>> 24) & 0xff;
|
|
22
|
+
}
|
|
23
|
+
return out;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Materialize a fixture payload descriptor into exact bytes.
|
|
27
|
+
*
|
|
28
|
+
* empty zero bytes
|
|
29
|
+
* zeros `length` 0x00 bytes
|
|
30
|
+
* constant `length` copies of `value`
|
|
31
|
+
* sequence `i % 256`
|
|
32
|
+
* lcg deterministic pseudorandom stream from `seed`
|
|
33
|
+
* text repeating printable ASCII (valid UTF-8)
|
|
34
|
+
* invalid-utf8 a stream containing lone continuation/overlong bytes
|
|
35
|
+
* dc-outlier all 0xff except one 0x00 at `outlierOffset` (forces the exact
|
|
36
|
+
* correction stream: a DC-dominant block whose coarse scale
|
|
37
|
+
* cannot reproduce the outlier's neighbourhood)
|
|
38
|
+
* alternating 0x00/0xff alternation (maximum Nyquist energy)
|
|
39
|
+
*/
|
|
40
|
+
export function materializePayload(d) {
|
|
41
|
+
const length = d.length ?? 0;
|
|
42
|
+
switch (d.kind) {
|
|
43
|
+
case "empty":
|
|
44
|
+
return new Uint8Array(0);
|
|
45
|
+
case "zeros":
|
|
46
|
+
return new Uint8Array(length);
|
|
47
|
+
case "constant":
|
|
48
|
+
return new Uint8Array(length).fill((d.value ?? 0) & 0xff);
|
|
49
|
+
case "sequence":
|
|
50
|
+
return Uint8Array.from({ length }, (_v, i) => i % 256);
|
|
51
|
+
case "lcg":
|
|
52
|
+
return lcgBytes(length, d.seed ?? 1);
|
|
53
|
+
case "text": {
|
|
54
|
+
const alphabet = "abcdefghijklmnopqrstuvwxyz0123456789 ";
|
|
55
|
+
return Uint8Array.from({ length }, (_v, i) => alphabet.charCodeAt(i % alphabet.length));
|
|
56
|
+
}
|
|
57
|
+
case "invalid-utf8": {
|
|
58
|
+
// Deliberately invalid: lone 0x80 continuation, 0xff (never valid), and
|
|
59
|
+
// the overlong 0xc0 0xaf encoding of "/". Never normalized by the codec.
|
|
60
|
+
const pattern = [0xff, 0x80, 0xc0, 0xaf, 0x00, 0xfe];
|
|
61
|
+
return Uint8Array.from({ length }, (_v, i) => pattern[i % pattern.length]);
|
|
62
|
+
}
|
|
63
|
+
case "dc-outlier": {
|
|
64
|
+
const out = new Uint8Array(length).fill(255);
|
|
65
|
+
const at = d.outlierOffset ?? 0;
|
|
66
|
+
if (at < length)
|
|
67
|
+
out[at] = 0;
|
|
68
|
+
return out;
|
|
69
|
+
}
|
|
70
|
+
case "alternating":
|
|
71
|
+
return Uint8Array.from({ length }, (_v, i) => (i % 2 === 0 ? 0 : 255));
|
|
72
|
+
case "literal":
|
|
73
|
+
return new Uint8Array(Buffer.from(d.bytesBase64 ?? "", "base64"));
|
|
74
|
+
default:
|
|
75
|
+
throw new Error(`unknown residual payload kind: ${d.kind}`);
|
|
76
|
+
}
|
|
77
|
+
}
|